using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticNews.Adapters.Scraping; using Microsoft.Extensions.Logging; using Microsoft.Playwright; namespace FinlyticNews.Services; /// /// Defines a headless scraping service for extracting text and resolving redirects from news sites. /// public interface IPlaywrightScraperService { /// /// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters. /// /// The initial article URL. /// A tuple containing the final resolved URL and the extracted raw text content. Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url); } /// public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable { private readonly ILogger _logger; private readonly IEnumerable _scraperAdapters; private IPlaywright? _playwright; private IBrowser? _browser; private readonly SemaphoreSlim _browserLock = new(1, 1); /// /// Initializes a new instance of the class. /// public PlaywrightScraperService( ILogger logger, IEnumerable scraperAdapters) { _logger = logger; _scraperAdapters = scraperAdapters; } /// public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url) { _logger.LogInformation("[{Channel}] Launching browser context to scrape article: {Url}", "NewsChannel", url); var browser = await GetOrInitBrowserAsync(); // Fast, isolated browser context (incognito tab environment) per article await using var context = await browser.NewContextAsync(new BrowserNewContextOptions { UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", ViewportSize = new ViewportSize { Width = 1280, Height = 800 } }); var page = await context.NewPageAsync(); try { // 1. Initial Navigation with DOMContentLoaded wait var response = await page.GotoAsync(url, new PageGotoOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 30000 }); if (response == null) { throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}"); } var finalUrl = page.Url; _logger.LogDebug("[{Channel}] Navigation completed. Initial final URL: {Url}", "NewsChannel", finalUrl); // 2. Resolve Host Specific Scraper Adapter var uri = new Uri(url); var host = uri.Host; var adapter = _scraperAdapters.FirstOrDefault(a => host.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase)); IPage targetPage = page; if (adapter != null) { _logger.LogInformation("[{Channel}] Executing adapter redirect check for host: {Host}", "NewsChannel", adapter.Hostname); try { var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page); // Loop Protection: Navigate only if redirect target is a new URL if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) && !string.Equals(page.Url, resolvedRedirectUrl, StringComparison.OrdinalIgnoreCase)) { _logger.LogInformation("[{Channel}] Redirect resolved to target URL: {Url}", "NewsChannel", resolvedRedirectUrl); var refererUrl = page.Url; finalUrl = resolvedRedirectUrl; var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 30000, Referer = refererUrl }); if (redirectResponse == null) { _logger.LogWarning("[{Channel}] Failed to load response for redirect URL: {Url}", "NewsChannel", resolvedRedirectUrl); } else { finalUrl = page.Url; } // Check if redirect opened a new tab/popup var matchedPage = page.Context.Pages.FirstOrDefault(p => p.Url == resolvedRedirectUrl); if (matchedPage != null) { targetPage = matchedPage; } } } catch (Exception ex) { _logger.LogWarning(ex, "[{Channel}] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", "NewsChannel", adapter.Hostname); } } // 3. Re-resolve detail page adapter for final target page var targetUri = new Uri(targetPage.Url); var targetHost = targetUri.Host; var targetAdapter = _scraperAdapters.FirstOrDefault(a => targetHost.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase)); // Wait a brief moment for dynamic scripts / DOM settling await targetPage.WaitForTimeoutAsync(1000); // 4. Extract Content via Mozilla Readability (or Adapter Fallback) string extractedText = string.Empty; if (targetAdapter != null) { var readabilityResult = await targetAdapter.ExtractArticleContentAsync(targetPage); if (readabilityResult != null && !string.IsNullOrWhiteSpace(readabilityResult.TextContent)) { extractedText = readabilityResult.TextContent; } } // Standard Fallback: Body Text / Selector Extraction if (string.IsNullOrWhiteSpace(extractedText)) { var bodySelector = targetAdapter?.ArticleBodySelector ?? "body"; var locator = targetPage.Locator(bodySelector); if (await locator.CountAsync() > 0) { extractedText = await locator.First.InnerTextAsync(); } if (string.IsNullOrWhiteSpace(extractedText)) { extractedText = await targetPage.EvaluateAsync( $"() => document.querySelector('{bodySelector}')?.innerText ?? ''"); } } return (finalUrl, extractedText?.Trim() ?? string.Empty); } catch (Exception ex) { _logger.LogError(ex, "[{Channel}] Failed to scrape page content from URL: {Url}", "NewsChannel", url); throw; } finally { await context.CloseAsync(); } } /// /// Thread-safe singleton initialization of the Chromium browser instance. /// private async Task GetOrInitBrowserAsync() { if (_browser != null && _browser.IsConnected) { return _browser; } await _browserLock.WaitAsync(); try { if (_browser != null && _browser.IsConnected) { return _browser; } _playwright ??= await Playwright.CreateAsync(); _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions { Headless = true, Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"] }); _logger.LogInformation("[{Channel}] Initialized shared Chromium browser instance.", "NewsChannel"); return _browser; } finally { _browserLock.Release(); } } /// /// Disposes the Playwright and Browser instances cleanly during service shutdown. /// public async ValueTask DisposeAsync() { if (_browser != null) { await _browser.CloseAsync(); await _browser.DisposeAsync(); } _playwright?.Dispose(); _browserLock.Dispose(); GC.SuppressFinalize(this); } }