Finlytic/FinlyticNews/Adapters/Scraping/ArticleScraperAdapter.cs

using System;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.Playwright;

namespace FinlyticNews.Adapters.Scraping;

/// <summary>
/// Record holding structured article content extracted via Mozilla Readability.
/// </summary>
public record ScrapedArticleResult(
    string Title,
    string TextContent,
    string HtmlContent,
    string? Author,
    string? Excerpt,
    string FinalUrl
);

/// <summary>
/// Abstract base class representing a website-specific scraping adapter.
/// </summary>
public abstract class ArticleScraperAdapter
{
    /// <summary>
    /// Gets the target domain hostname of the news website (e.g. "finanznachrichten.de").
    /// </summary>
    public abstract string Hostname { get; }

    /// <summary>
    /// Gets the CSS selector to identify and click the 'Read full article' redirect button, if applicable.
    /// </summary>
    public virtual string? ReadMoreSelector => null;

    /// <summary>
    /// Fallback CSS selector targeting the article body text elements if Readability fails.
    /// </summary>
    public virtual string ArticleBodySelector => "body";

    /// <summary>
    /// Tries to resolve 'Read More' links or external redirects.
    /// </summary>
    public virtual async Task<string?> TryResolveRedirectUrlAsync(IPage page)
    {
        var selector = ReadMoreSelector;
        if (string.IsNullOrWhiteSpace(selector)) return null;

        var readMoreButton = page.Locator(selector).First;
        if (await readMoreButton.CountAsync() > 0 && await readMoreButton.IsVisibleAsync())
        {
            try
            {
                // Playwright modern pattern for combined popup or navigation handling
                var popupTask = page.Context.WaitForPageAsync(new() { Timeout = 6000 });
                var navTask = page.WaitForURLAsync(url => url != page.Url, new() { Timeout = 6000 });

                await readMoreButton.ClickAsync();

                var completedTask = await Task.WhenAny(popupTask, navTask);
                if (completedTask == popupTask)
                {
                    var popup = await popupTask;
                    await popup.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
                    return popup.Url;
                }

                await navTask;
                return page.Url;
            }
            catch
            {
                // Timeout / Click failed -> Fallback to current URL
            }
        }

        return null;
    }

    /// <summary>
    /// Injects Mozilla's Readability.js into the Playwright page to parse article content cleanly.
    /// </summary>
    public virtual async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
    {
        try
        {
            // 1. Inject Mozilla Readability Standalone JS Bundle via CDN
            await page.AddScriptTagAsync(new PageAddScriptTagOptions
            {
                Url = "https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js"
            });

            // 2. Execute Readability in the Browser Context
            var jsScript = @"
            () => {
                if (typeof Readability === 'undefined') return null;
                const documentClone = document.cloneNode(true);
                const article = new Readability(documentClone).parse();
                if (!article) return null;

                return {
                    title: article.title || '',
                    textContent: article.textContent || '',
                    htmlContent: article.content || '',
                    author: article.byline || null,
                    excerpt: article.excerpt || null
                };
            }";

            var jsonResult = await page.EvaluateAsync<JsonElement?>(jsScript);

            if (jsonResult.HasValue && jsonResult.Value.ValueKind != JsonValueKind.Null)
            {
                var root = jsonResult.Value;
                return new ScrapedArticleResult(
                    Title: root.GetProperty("title").GetString() ?? page.TitleAsync().Result,
                    TextContent: root.GetProperty("textContent").GetString()?.Trim() ?? string.Empty,
                    HtmlContent: root.GetProperty("htmlContent").GetString() ?? string.Empty,
                    Author: root.GetProperty("author").GetString(),
                    Excerpt: root.GetProperty("excerpt").GetString(),
                    FinalUrl: page.Url
                );
            }
        }
        catch
        {
            // Readability Injection / Parsing failed -> Fallback to standard selector parsing
        }

        // Fallback: Manuelles Auslesen via ArticleBodySelector
        var bodyText = await page.Locator(ArticleBodySelector).InnerTextAsync();
        return new ScrapedArticleResult(
            Title: await page.TitleAsync(),
            TextContent: bodyText.Trim(),
            HtmlContent: await page.Locator(ArticleBodySelector).InnerHTMLAsync(),
            Author: null,
            Excerpt: null,
            FinalUrl: page.Url
        );
    }
}