using System; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Playwright; namespace FinlyticNews.Adapters.Scraping; /// /// Record holding structured article content extracted via Mozilla Readability. /// public record ScrapedArticleResult( string Title, string TextContent, string HtmlContent, string? Author, string? Excerpt, string FinalUrl ); /// /// Abstract base class representing a website-specific scraping adapter. /// public abstract class ArticleScraperAdapter { /// /// Gets the target domain hostname of the news website (e.g. "finanznachrichten.de"). /// public abstract string Hostname { get; } /// /// Gets the CSS selector to identify and click the 'Read full article' redirect button, if applicable. /// public virtual string? ReadMoreSelector => null; /// /// Fallback CSS selector targeting the article body text elements if Readability fails. /// public virtual string ArticleBodySelector => "body"; /// /// Tries to resolve 'Read More' links or external redirects. /// public virtual async Task TryResolveRedirectUrlAsync(IPage page) { var selector = ReadMoreSelector; if (string.IsNullOrWhiteSpace(selector)) return null; var readMoreButton = page.Locator(selector).First; if (await readMoreButton.CountAsync() > 0 && await readMoreButton.IsVisibleAsync()) { try { // Playwright modern pattern for combined popup or navigation handling var popupTask = page.Context.WaitForPageAsync(new() { Timeout = 6000 }); var navTask = page.WaitForURLAsync(url => url != page.Url, new() { Timeout = 6000 }); await readMoreButton.ClickAsync(); var completedTask = await Task.WhenAny(popupTask, navTask); if (completedTask == popupTask) { var popup = await popupTask; await popup.WaitForLoadStateAsync(LoadState.DOMContentLoaded); return popup.Url; } await navTask; return page.Url; } catch { // Timeout / Click failed -> Fallback to current URL } } return null; } /// /// Injects Mozilla's Readability.js into the Playwright page to parse article content cleanly. /// public virtual async Task ExtractArticleContentAsync(IPage page) { try { // 1. Inject Mozilla Readability Standalone JS Bundle via CDN await page.AddScriptTagAsync(new PageAddScriptTagOptions { Url = "https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js" }); // 2. Execute Readability in the Browser Context var jsScript = @" () => { if (typeof Readability === 'undefined') return null; const documentClone = document.cloneNode(true); const article = new Readability(documentClone).parse(); if (!article) return null; return { title: article.title || '', textContent: article.textContent || '', htmlContent: article.content || '', author: article.byline || null, excerpt: article.excerpt || null }; }"; var jsonResult = await page.EvaluateAsync(jsScript); if (jsonResult.HasValue && jsonResult.Value.ValueKind != JsonValueKind.Null) { var root = jsonResult.Value; return new ScrapedArticleResult( Title: root.GetProperty("title").GetString() ?? page.TitleAsync().Result, TextContent: root.GetProperty("textContent").GetString()?.Trim() ?? string.Empty, HtmlContent: root.GetProperty("htmlContent").GetString() ?? string.Empty, Author: root.GetProperty("author").GetString(), Excerpt: root.GetProperty("excerpt").GetString(), FinalUrl: page.Url ); } } catch { // Readability Injection / Parsing failed -> Fallback to standard selector parsing } // Fallback: Manuelles Auslesen via ArticleBodySelector var bodyText = await page.Locator(ArticleBodySelector).InnerTextAsync(); return new ScrapedArticleResult( Title: await page.TitleAsync(), TextContent: bodyText.Trim(), HtmlContent: await page.Locator(ArticleBodySelector).InnerHTMLAsync(), Author: null, Excerpt: null, FinalUrl: page.Url ); } }