feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

2026-08-24 21:35:33 +02:00
parent 44b161d509
commit 8112598602
400 changed files with 1812 additions and 113367 deletions
@@ -0,0 +1,49 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Playwright;
+
+namespace FinlyticNews.Adapters.Scraping;
+
+/// <summary>
+/// Specialized article scraper adapter for Ariva.de.
+/// </summary>
+public class ArivaScraperAdapter : ArticleScraperAdapter
+{
+    public override string Hostname => "ariva.de";
+
+    public override string ArticleBodySelector => "#news_teaser, div.readable, div.column.twothirds";
+
+    public override async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
+    {
+        // Remove forum widgets, ad containers, and comments before extraction
+        await page.EvaluateAsync(@"() => {
+            const el = document.querySelectorAll('#forum_box, #basics-indices-modal, #nativendo-article-desktop, .action-buttons, .summary');
+            el.forEach(e => e.remove());
+        }");
+
+        var titleLocator = page.Locator("#news_title").First;
+        string title = string.Empty;
+        if (await titleLocator.CountAsync() > 0)
+        {
+            title = CleanTitle(await titleLocator.InnerTextAsync());
+        }
+
+        var teaserLocator = page.Locator("#news_teaser").First;
+        if (await teaserLocator.CountAsync() > 0)
+        {
+            var text = await teaserLocator.InnerTextAsync();
+            var cleaned = TruncateDisclaimers(text.Trim());
+
+            return new ScrapedArticleResult(
+                Title: !string.IsNullOrWhiteSpace(title) ? title : CleanTitle(await page.TitleAsync()),
+                TextContent: cleaned,
+                HtmlContent: await teaserLocator.InnerHTMLAsync(),
+                Author: "ARIVA.DE",
+                Excerpt: cleaned.Length > 250 ? cleaned[..250] + "..." : cleaned,
+                FinalUrl: page.Url
+            );
+        }
+
+        return await base.ExtractArticleContentAsync(page);
+    }
+}