feat(News): refactor news service and add sentiment tracking

2026-08-09 21:01:41 +02:00
parent 605e41cfeb
commit aff831cf58
390 changed files with 115094 additions and 9 deletions
@@ -0,0 +1,140 @@
+using System;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.Playwright;
+
+namespace FinlyticNews.Adapters.Scraping;
+
+/// <summary>
+/// Record holding structured article content extracted via Mozilla Readability.
+/// </summary>
+public record ScrapedArticleResult(
+    string Title,
+    string TextContent,
+    string HtmlContent,
+    string? Author,
+    string? Excerpt,
+    string FinalUrl
+);
+
+/// <summary>
+/// Abstract base class representing a website-specific scraping adapter.
+/// </summary>
+public abstract class ArticleScraperAdapter
+{
+    /// <summary>
+    /// Gets the target domain hostname of the news website (e.g. "finanznachrichten.de").
+    /// </summary>
+    public abstract string Hostname { get; }
+
+    /// <summary>
+    /// Gets the CSS selector to identify and click the 'Read full article' redirect button, if applicable.
+    /// </summary>
+    public virtual string? ReadMoreSelector => null;
+
+    /// <summary>
+    /// Fallback CSS selector targeting the article body text elements if Readability fails.
+    /// </summary>
+    public virtual string ArticleBodySelector => "body";
+
+    /// <summary>
+    /// Tries to resolve 'Read More' links or external redirects.
+    /// </summary>
+    public virtual async Task<string?> TryResolveRedirectUrlAsync(IPage page)
+    {
+        var selector = ReadMoreSelector;
+        if (string.IsNullOrWhiteSpace(selector)) return null;
+
+        var readMoreButton = page.Locator(selector).First;
+        if (await readMoreButton.CountAsync() > 0 && await readMoreButton.IsVisibleAsync())
+        {
+            try
+            {
+                // Playwright modern pattern for combined popup or navigation handling
+                var popupTask = page.Context.WaitForPageAsync(new() { Timeout = 6000 });
+                var navTask = page.WaitForURLAsync(url => url != page.Url, new() { Timeout = 6000 });
+
+                await readMoreButton.ClickAsync();
+
+                var completedTask = await Task.WhenAny(popupTask, navTask);
+                if (completedTask == popupTask)
+                {
+                    var popup = await popupTask;
+                    await popup.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
+                    return popup.Url;
+                }
+
+                await navTask;
+                return page.Url;
+            }
+            catch
+            {
+                // Timeout / Click failed -> Fallback to current URL
+            }
+        }
+
+        return null;
+    }
+
+    /// <summary>
+    /// Injects Mozilla's Readability.js into the Playwright page to parse article content cleanly.
+    /// </summary>
+    public virtual async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
+    {
+        try
+        {
+            // 1. Inject Mozilla Readability Standalone JS Bundle via CDN
+            await page.AddScriptTagAsync(new PageAddScriptTagOptions
+            {
+                Url = "https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js"
+            });
+
+            // 2. Execute Readability in the Browser Context
+            var jsScript = @"
+            () => {
+                if (typeof Readability === 'undefined') return null;
+                const documentClone = document.cloneNode(true);
+                const article = new Readability(documentClone).parse();
+                if (!article) return null;
+
+                return {
+                    title: article.title || '',
+                    textContent: article.textContent || '',
+                    htmlContent: article.content || '',
+                    author: article.byline || null,
+                    excerpt: article.excerpt || null
+                };
+            }";
+
+            var jsonResult = await page.EvaluateAsync<JsonElement?>(jsScript);
+
+            if (jsonResult.HasValue && jsonResult.Value.ValueKind != JsonValueKind.Null)
+            {
+                var root = jsonResult.Value;
+                return new ScrapedArticleResult(
+                    Title: root.GetProperty("title").GetString() ?? page.TitleAsync().Result,
+                    TextContent: root.GetProperty("textContent").GetString()?.Trim() ?? string.Empty,
+                    HtmlContent: root.GetProperty("htmlContent").GetString() ?? string.Empty,
+                    Author: root.GetProperty("author").GetString(),
+                    Excerpt: root.GetProperty("excerpt").GetString(),
+                    FinalUrl: page.Url
+                );
+            }
+        }
+        catch
+        {
+            // Readability Injection / Parsing failed -> Fallback to standard selector parsing
+        }
+
+        // Fallback: Manuelles Auslesen via ArticleBodySelector
+        var bodyText = await page.Locator(ArticleBodySelector).InnerTextAsync();
+        return new ScrapedArticleResult(
+            Title: await page.TitleAsync(),
+            TextContent: bodyText.Trim(),
+            HtmlContent: await page.Locator(ArticleBodySelector).InnerHTMLAsync(),
+            Author: null,
+            Excerpt: null,
+            FinalUrl: page.Url
+        );
+    }
+}