Finlytic/FinlyticNews/Services/PlaywrightScraperService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services;
using FinlyticNews.Adapters.Scraping;
using FinlyticNews.Util;
using Microsoft.Playwright;

namespace FinlyticNews.Services;

/// <summary>
/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
/// </summary>
public interface IPlaywrightScraperService
{
    /// <summary>
    /// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
    /// </summary>
    /// <param name="url">The initial article URL.</param>
    /// <returns>A tuple containing the final resolved URL and the extracted raw text content.</returns>
    Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
}

/// <inheritdoc />
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
{
    private readonly IFinlyticLogger<PlaywrightScraperService> _finlyticLogger;
    private readonly IEnumerable<ArticleScraperAdapter> _scraperAdapters;

    private IPlaywright? _playwright;
    private IBrowser? _browser;
    private readonly SemaphoreSlim _browserLock = new(1, 1);

    /// <summary>
    /// Initializes a new instance of the <see cref="PlaywrightScraperService"/> class.
    /// </summary>
    public PlaywrightScraperService(
        IFinlyticLogger<PlaywrightScraperService> finlyticLogger,
        IEnumerable<ArticleScraperAdapter> scraperAdapters)
    {
        _finlyticLogger = finlyticLogger;
        _scraperAdapters = scraperAdapters;
    }

    /// <inheritdoc />
    public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
    {
        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);

        var browser = await GetOrInitBrowserAsync();
        
        await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
        {
            UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            ViewportSize = new ViewportSize { Width = 1280, Height = 800 }
        });

        var page = await context.NewPageAsync();

        try
        {
            var response = await page.GotoAsync(url, new PageGotoOptions
            {
                WaitUntil = WaitUntilState.DOMContentLoaded,
                Timeout = 30000
            });

            if (response == null)
            {
                throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}");
            }

            var finalUrl = page.Url;
            await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Navigation completed. Initial final URL: {Url}", finalUrl);

            var host = new Uri(finalUrl).Host;
            var adapter = _scraperAdapters.FirstOrDefault(a => 
                host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
                a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));

            if (adapter != null)
            {
                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Executing adapter redirect check for host: {Host}", adapter.Hostname);
                try
                {
                    var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
                    if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) && 
                        !resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase))
                    {
                        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
                        
                        var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
                        {
                            WaitUntil = WaitUntilState.DOMContentLoaded,
                            Timeout = 30000
                        });

                        if (redirectResponse == null)
                        {
                            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Failed to load response for redirect URL: {Url}", resolvedRedirectUrl);
                        }

                        finalUrl = page.Url;
                        host = new Uri(finalUrl).Host;
                        
                        adapter = _scraperAdapters.FirstOrDefault(a => 
                            host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
                            a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
                    }
                }
                catch (Exception ex)
                {
                    await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter.Hostname);
                }
            }

            string content;
            if (adapter != null)
            {
                var result = await adapter.ExtractArticleContentAsync(page);
                content = result?.TextContent ?? await FallbackExtractContentAsync(page);
            }
            else
            {
                content = await FallbackExtractContentAsync(page);
            }

            return (finalUrl, content);
        }
        catch (Exception ex)
        {
            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
            throw;
        }
        finally
        {
            await page.CloseAsync();
        }
    }

    private async Task<string> FallbackExtractContentAsync(IPage page)
    {
        var innerText = await page.EvaluateAsync<string>(@"() => {
            const scripts = document.querySelectorAll('script, style, noscript, nav, header, footer, iframe, svg');
            scripts.forEach(s => s.remove());

            const main = document.querySelector('article, main, .article-content, #content, .story-body') || document.body;
            return main ? main.innerText : document.body.innerText;
        }");

        return innerText?.Trim() ?? string.Empty;
    }

    private async Task<IBrowser> GetOrInitBrowserAsync()
    {
        if (_browser != null && _browser.IsConnected)
        {
            return _browser;
        }

        await _browserLock.WaitAsync();
        try
        {
            if (_browser != null && _browser.IsConnected)
            {
                return _browser;
            }

            _playwright ??= await Playwright.CreateAsync();
            _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
            {
                Headless = true,
                Args = new[]
                {
                    "--no-sandbox",
                    "--disable-setuid-sandbox",
                    "--disable-dev-shm-usage",
                    "--disable-gpu"
                }
            });

            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Initialized shared Chromium browser instance.");
            return _browser;
        }
        finally
        {
            _browserLock.Release();
        }
    }

    public async ValueTask DisposeAsync()
    {
        if (_browser != null)
        {
            await _browser.CloseAsync();
            await _browser.DisposeAsync();
        }

        _playwright?.Dispose();
        _browserLock.Dispose();

        GC.SuppressFinalize(this);
    }
}