Merge pull request #4 from Guacam-Ole/develop

1.0

.gitignore

@@ -396,3 +396,4 @@ FodyWeavers.xsd
 
 # JetBrains Rider
 *.sln.iml
+secrets.json

RssBot.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.33516.290
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RssBot", "RssBot\RssBot.csproj", "{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {73B4F06F-AEBC-4FD7-86E2-A303CFE1FCB2}
+	EndGlobalSection
+EndGlobal

RssBot/BotWork.cs

@@ -0,0 +1,39 @@
+using Microsoft.Extensions.Logging;
+
+namespace RssBot
+{
+    public class BotWork
+    {
+        private readonly ILogger<BotWork> _logger;
+        private readonly Rss _rss;
+        private readonly Toot _toot;
+
+        public BotWork(ILogger<BotWork> logger, Rss rss, Toot toot)
+        {
+            _logger = logger;
+            _rss = rss;
+            _toot = toot;
+        }
+
+        public async Task RetrieveAndSendToots()
+        {
+            var newFeedItems = await _rss.ReadFeed();
+            foreach (var botItems in newFeedItems)
+            {
+                foreach (var item in botItems.Value)
+                {
+                    try
+                    {
+                        await _toot.SendToot(botItems.Key, item);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, "failed sending toot for {key}, {item}", botItems.Key, item);
+                        throw;
+                    }
+                }
+            }
+            _logger.LogInformation("Done");
+        }
+    }
+}

RssBot/Config.cs

@@ -0,0 +1,33 @@
+namespace RssBot
+{
+    public class Config
+    {
+        public bool PrivateOnly { get; set; }   // Don'T toot public
+        public List<FeedConfig> Feeds { get; set; }
+        public List<TagReplacement> TagReplacements { get; set; }
+    }
+
+    public class TagReplacement
+    {
+        public string From { get; set; }    
+        public string To { get; set; }  
+    }
+
+    public class FeedConfig
+    {
+        public string Url { get; set; }
+        public List<BotConfig> Bots { get; set; }
+    }
+
+    public class BotConfig
+    {
+        public string Id { get; set; }
+        public string UrlFilter { get; set; }
+        public string? UrlExclude { get; set; }
+        public string? TypeFilter { get; set; }
+        public bool ShowImage { get; set; }
+        public bool ShowTags { get; set; }
+        public string? IgnoreTags { get; set; }
+        public string? AdditionalTags { get; set; }
+    }
+}

RssBot/Helpers.cs

@@ -0,0 +1,55 @@
+using CodeHollow.FeedReader;
+
+using RssBot.RssBot;
+
+using System.Xml.Linq;
+
+namespace RssBot
+{
+    public static class Helpers
+    {
+        public static RssItem ToRssItem(this FeedItem feedItem)
+        {
+            var bestImage = GetBestImage(feedItem.SpecificItem.Element);
+            var tags = GetTags(feedItem.SpecificItem.Element);
+
+            return new RssItem
+            {
+                Title = feedItem.Title,
+                Description = feedItem.Description,
+                Url = feedItem.Link,
+                ImageUrl = bestImage?.Key,
+                ImageDescription = bestImage?.Value,
+                Tags = tags
+            };
+        }
+
+        public static string? GetTags(XElement feedElement)
+        {
+            return feedElement.Descendants().Where(q => q.Name.LocalName == "keywords").FirstOrDefault()?.FirstNode?.ToString();
+        }
+
+        public static KeyValuePair<string, string>? GetBestImage(XElement feedElement)
+        {
+            KeyValuePair<string, string>? bestImage = null;
+            var images = feedElement.Descendants().Where(q => q.Name.LocalName == "image").ToList();
+            if (images.Count > 0)
+            {
+                int maxWidth = 0;
+
+                foreach (var image in images)
+                {
+                    var url = image.Elements().FirstOrDefault(q => q.Name.LocalName == "data")?.FirstNode?.ToString();
+                    var width = image.Elements().FirstOrDefault(q => q.Name.LocalName == "width")?.FirstNode?.ToString();
+                    var alt = image.Elements().FirstOrDefault(q => q.Name.LocalName == "alt")?.FirstNode?.ToString();
+                    if (alt != null && url != null && width != null && int.TryParse(width, out int intWith) && intWith > maxWidth)
+                    {
+                        maxWidth = intWith;
+                        bestImage = new KeyValuePair<string, string>(url, alt);
+                    }
+                }
+            }
+            return bestImage;
+        }
+    }
+}

RssBot/Program.cs

@@ -0,0 +1,29 @@
+// See https://aka.ms/new-console-template for more information
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+using RssBot;
+
+Console.WriteLine("Hello, World!");
+var services = new ServiceCollection();
+
+services.AddLogging(logging =>
+{
+    logging.ClearProviders();
+    logging.AddConsole();
+    logging.SetMinimumLevel(LogLevel.Debug);
+    var logFile = "rssmastodon.log";
+    logging.AddFile(logFile, append: true);
+});
+services.AddScoped<Rss>();
+services.AddScoped<Toot>();
+services.AddScoped<BotWork>();
+
+var provider = services.BuildServiceProvider();
+var botwork = provider.GetRequiredService<BotWork>();
+await botwork.RetrieveAndSendToots();
+
+
+
+
+

RssBot/Rss.cs

@@ -0,0 +1,114 @@
+using CodeHollow.FeedReader;
+
+using Microsoft.Extensions.Logging;
+
+using Newtonsoft.Json;
+
+using RssBot.RssBot;
+
+namespace RssBot
+{
+    public class Rss
+    {
+        private readonly ILogger<Rss> _logger;
+        private Config _config;
+
+        public Rss(ILogger<Rss> logger)
+        {
+            _logger = logger;
+            var config = File.ReadAllText("./config.json");
+            _config = JsonConvert.DeserializeObject<Config>(config) ?? throw new FileNotFoundException("cannot read config");
+        }
+
+        public async Task<Dictionary<BotConfig, List<RssItem>>> ReadFeed()
+        {
+            var unpublishedItems = new Dictionary<BotConfig, List<RssItem>>();
+            foreach (var bots in _config.Feeds.Select(q => q.Bots))
+            {
+                foreach (var bot in bots)
+                {
+                    unpublishedItems.Add(bot, new List<RssItem>());
+                }
+            }
+
+            foreach (var feedConfig in _config.Feeds)
+            {
+                using (var db = new LiteDB.LiteDatabase("state.db"))
+                {
+                    var states = db.GetCollection<State>();
+                    var match = states.FindById(feedConfig.Url);
+
+                    var feed = await FeedReader.ReadAsync(feedConfig.Url);
+                    if (feed.Type != FeedType.Rss_1_0)
+                    {
+                        _logger.LogError("Unexpected RSS-Type. Expecting 1.0, received '{type}'", feed.Type);
+                        return unpublishedItems;
+                    }
+
+                    if (match == null)
+                    {
+                        // first start, mark all as already sent
+                        match = new State { Id = feedConfig.Url, LastFeed = DateTime.Now };
+
+                        foreach (var item in feed.Items)
+                        {
+                            match.PostedItems.Add(new PostedItem { Id = item.Id, ReadDate = DateTime.Now });
+                        }
+                    }
+                    else
+                    {
+                        // cleanup old stuff
+                        match.PostedItems.RemoveAll(q => q.ReadDate < DateTime.Now.AddDays(-120));
+                    }
+                    var newItems = feed.Items.Where(q => !match.PostedItems.Any(m => m.Id == q.Id));
+
+                    _logger.LogInformation("Tooting '{count}' feeds since '{lastfeed}'", newItems.Count(), match.LastFeed);
+                    foreach (var item in newItems)
+                    {
+                        try
+                        {
+                            var x = item.SpecificItem.Element.Descendants().ToList();
+                            var rssItem = (item.ToRssItem());
+                            var bot = GetBotForRssItem(feedConfig, rssItem);
+                            match.PostedItems.Add(new PostedItem { Id = item.Id, ReadDate = DateTime.Now });
+                            if (bot == null) continue;
+                            unpublishedItems[bot].Add(rssItem);
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogError(ex, "Cannot toot item {item}", item);
+                        }
+                    }
+
+                    match.LastFeed = DateTime.Now;
+                    states.Upsert(match);
+                }
+            }
+            return unpublishedItems;
+        }
+
+        private static bool UrlHasExcludes(string url, string? excludes)
+        {
+            if (excludes == null) return false;
+            var excludeList = excludes.Split(" ");
+            foreach (var exclude in excludeList)
+            {
+                if (url.Contains(exclude, StringComparison.CurrentCultureIgnoreCase)) return true;
+            }
+            return false;
+        }
+
+        private static BotConfig? GetBotForRssItem(FeedConfig config, RssItem item)
+        {
+            foreach (var bot in config.Bots)
+            {
+                if (item.Url.Contains(bot.UrlFilter, StringComparison.InvariantCultureIgnoreCase))
+                {
+                    if (UrlHasExcludes(item.Url, bot.UrlExclude)) continue;
+                    return bot;
+                }
+            }
+            return null;
+        }
+    }
+}

RssBot/RssBot.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="config.json" />
+    <None Remove="secrets.json" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="config.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+    <Content Include="secrets.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CodeHollow.FeedReader" Version="1.2.6" />
+    <PackageReference Include="LiteDB" Version="5.0.17" />
+    <PackageReference Include="Mastonet" Version="2.3.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="NReco.Logging.File" Version="1.1.7" />
+  </ItemGroup>
+
+</Project>

RssBot/RssBot.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.33516.290
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RssBot", "RssBot\RssBot.csproj", "{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2F433C4-D3E7-4D95-A89A-D7B8377D7195}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {73B4F06F-AEBC-4FD7-86E2-A303CFE1FCB2}
+	EndGlobalSection
+EndGlobal

RssBot/RssItem.cs

@@ -0,0 +1,12 @@
+namespace RssBot.RssBot
+{
+    public class RssItem
+    {
+        public string Title { get; set; }
+        public string Description { get; set; }
+        public string Url { get; set; }
+        public string? ImageUrl { get; set; }
+        public string? ImageDescription { get; set; }
+        public string? Tags { get; set; }
+    }
+}