using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Services; using FinlyticNews.Adapters.Scraping; using FinlyticNews.Util; using Microsoft.Playwright; namespace FinlyticNews.Services; /// /// Defines a headless scraping service for extracting text, metadata, and resolving redirects from news sites. /// public interface IPlaywrightScraperService { /// /// Scrapes an article, automatically following redirects and applying site-specific scraper adapters. /// /// The initial article URL. /// A tuple containing the final resolved URL and the structured scrape result. Task<(string ResolvedUrl, ScrapedArticleResult Result)> ScrapeArticleAsync(string url); } /// public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable { private readonly IFinlyticLogger _finlyticLogger; private readonly IEnumerable _scraperAdapters; private IPlaywright? _playwright; private IBrowser? _browser; private readonly SemaphoreSlim _browserLock = new(1, 1); /// /// Initializes a new instance of the class. /// public PlaywrightScraperService( IFinlyticLogger finlyticLogger, IEnumerable scraperAdapters) { _finlyticLogger = finlyticLogger; _scraperAdapters = scraperAdapters; } /// public async Task<(string ResolvedUrl, ScrapedArticleResult Result)> ScrapeArticleAsync(string url) { await _finlyticLogger.LogInfoAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url); var browser = await GetOrInitBrowserAsync(); await using var context = await browser.NewContextAsync(new BrowserNewContextOptions { UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", ViewportSize = new ViewportSize { Width = 1280, Height = 800 } }); var page = await context.NewPageAsync(); try { var response = await page.GotoAsync(url, new PageGotoOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 30000 }); if (response == null) { throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}"); } var finalUrl = page.Url; var host = new Uri(finalUrl).Host; var adapter = _scraperAdapters.FirstOrDefault(a => host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) || a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase)); if (adapter != null) { try { var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page); if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) && !resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase)) { await _finlyticLogger.LogInfoAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl); var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions { WaitUntil = WaitUntilState.DOMContentLoaded, Timeout = 30000 }); finalUrl = page.Url; host = new Uri(finalUrl).Host; adapter = _scraperAdapters.FirstOrDefault(a => host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) || a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase)); } } catch (Exception ex) { await _finlyticLogger.LogWarningAsync(SettingKeys.ScraperChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter?.Hostname ?? host); } } ScrapedArticleResult? result = null; for (int attempt = 1; attempt <= 2; attempt++) { try { if (adapter != null) { result = await adapter.ExtractArticleContentAsync(page); } if (result == null || string.IsNullOrWhiteSpace(result.TextContent)) { var bodyText = await FallbackExtractContentAsync(page); result = new ScrapedArticleResult( Title: await page.TitleAsync(), TextContent: bodyText, HtmlContent: string.Empty, Author: null, Excerpt: null, FinalUrl: finalUrl ); } break; // Extraction succeeded without execution context getting destroyed } catch (PlaywrightException ex) when (ex.Message.Contains("Execution context was destroyed") && attempt == 1) { await _finlyticLogger.LogWarningAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Execution context destroyed (likely JS/Meta redirect). Waiting for new page load..."); try { await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 }); } catch (TimeoutException) { /* Ignored, try extracting anyway */ } finalUrl = page.Url; host = new Uri(finalUrl).Host; adapter = _scraperAdapters.FirstOrDefault(a => host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) || a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase)); } } return (finalUrl, result!); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.ScraperChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url); throw; } finally { await page.CloseAsync(); } } private async Task FallbackExtractContentAsync(IPage page) { var innerText = await page.EvaluateAsync(@"() => { const scripts = document.querySelectorAll('script, style, noscript, nav, header, footer, iframe, svg'); scripts.forEach(s => s.remove()); const main = document.querySelector('article, main, .article-content, #content, .story-body') || document.body; return main ? main.innerText : document.body.innerText; }"); return innerText?.Trim() ?? string.Empty; } private async Task GetOrInitBrowserAsync() { if (_browser != null && _browser.IsConnected) { return _browser; } await _browserLock.WaitAsync(); try { if (_browser != null && _browser.IsConnected) { return _browser; } _playwright ??= await Playwright.CreateAsync(); _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions { Headless = true, Args = new[] { "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", "--disable-gpu" } }); await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Initialized shared Chromium browser instance."); return _browser; } finally { _browserLock.Release(); } } public async ValueTask DisposeAsync() { if (_browser != null) { await _browser.CloseAsync(); await _browser.DisposeAsync(); } _playwright?.Dispose(); _browserLock.Dispose(); GC.SuppressFinalize(this); } }