using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services;
using FinlyticNews.Adapters.Scraping;
using FinlyticNews.Util;
using Microsoft.Playwright;
namespace FinlyticNews.Services;
///
/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
///
public interface IPlaywrightScraperService
{
///
/// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
///
/// The initial article URL.
/// A tuple containing the final resolved URL and the extracted raw text content.
Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
}
///
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
{
private readonly IFinlyticLogger _finlyticLogger;
private readonly IEnumerable _scraperAdapters;
private IPlaywright? _playwright;
private IBrowser? _browser;
private readonly SemaphoreSlim _browserLock = new(1, 1);
///
/// Initializes a new instance of the class.
///
public PlaywrightScraperService(
IFinlyticLogger finlyticLogger,
IEnumerable scraperAdapters)
{
_finlyticLogger = finlyticLogger;
_scraperAdapters = scraperAdapters;
}
///
public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
var browser = await GetOrInitBrowserAsync();
await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
{
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
ViewportSize = new ViewportSize { Width = 1280, Height = 800 }
});
var page = await context.NewPageAsync();
try
{
var response = await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30000
});
if (response == null)
{
throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}");
}
var finalUrl = page.Url;
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Navigation completed. Initial final URL: {Url}", finalUrl);
var host = new Uri(finalUrl).Host;
var adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
if (adapter != null)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Executing adapter redirect check for host: {Host}", adapter.Hostname);
try
{
var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) &&
!resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase))
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30000
});
if (redirectResponse == null)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Failed to load response for redirect URL: {Url}", resolvedRedirectUrl);
}
finalUrl = page.Url;
host = new Uri(finalUrl).Host;
adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
}
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter.Hostname);
}
}
string content;
if (adapter != null)
{
var result = await adapter.ExtractArticleContentAsync(page);
content = result?.TextContent ?? await FallbackExtractContentAsync(page);
}
else
{
content = await FallbackExtractContentAsync(page);
}
return (finalUrl, content);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
throw;
}
finally
{
await page.CloseAsync();
}
}
private async Task FallbackExtractContentAsync(IPage page)
{
var innerText = await page.EvaluateAsync(@"() => {
const scripts = document.querySelectorAll('script, style, noscript, nav, header, footer, iframe, svg');
scripts.forEach(s => s.remove());
const main = document.querySelector('article, main, .article-content, #content, .story-body') || document.body;
return main ? main.innerText : document.body.innerText;
}");
return innerText?.Trim() ?? string.Empty;
}
private async Task GetOrInitBrowserAsync()
{
if (_browser != null && _browser.IsConnected)
{
return _browser;
}
await _browserLock.WaitAsync();
try
{
if (_browser != null && _browser.IsConnected)
{
return _browser;
}
_playwright ??= await Playwright.CreateAsync();
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = true,
Args = new[]
{
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu"
}
});
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Initialized shared Chromium browser instance.");
return _browser;
}
finally
{
_browserLock.Release();
}
}
public async ValueTask DisposeAsync()
{
if (_browser != null)
{
await _browser.CloseAsync();
await _browser.DisposeAsync();
}
_playwright?.Dispose();
_browserLock.Dispose();
GC.SuppressFinalize(this);
}
}