Finlytic/FinlyticNews/Services/PlaywrightScraperService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticNews.Adapters.Scraping;
using Microsoft.Extensions.Logging;
using Microsoft.Playwright;

namespace FinlyticNews.Services;

/// <summary>
/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
/// </summary>
public interface IPlaywrightScraperService
{
    /// <summary>
    /// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
    /// </summary>
    /// <param name="url">The initial article URL.</param>
    /// <returns>A tuple containing the final resolved URL and the extracted raw text content.</returns>
    Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
}

/// <inheritdoc />
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
{
    private readonly ILogger<PlaywrightScraperService> _logger;
    private readonly IEnumerable<ArticleScraperAdapter> _scraperAdapters;

    private IPlaywright? _playwright;
    private IBrowser? _browser;
    private readonly SemaphoreSlim _browserLock = new(1, 1);

    /// <summary>
    /// Initializes a new instance of the <see cref="PlaywrightScraperService"/> class.
    /// </summary>
    public PlaywrightScraperService(
        ILogger<PlaywrightScraperService> logger,
        IEnumerable<ArticleScraperAdapter> scraperAdapters)
    {
        _logger = logger;
        _scraperAdapters = scraperAdapters;
    }

    /// <inheritdoc />
    public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
    {
        _logger.LogInformation("[{Channel}] Launching browser context to scrape article: {Url}", "NewsChannel", url);

        var browser = await GetOrInitBrowserAsync();
        
        // Fast, isolated browser context (incognito tab environment) per article
        await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
        {
            UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            ViewportSize = new ViewportSize { Width = 1280, Height = 800 }
        });

        var page = await context.NewPageAsync();

        try
        {
            // 1. Initial Navigation with DOMContentLoaded wait
            var response = await page.GotoAsync(url, new PageGotoOptions
            {
                WaitUntil = WaitUntilState.DOMContentLoaded,
                Timeout = 30000
            });

            if (response == null)
            {
                throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}");
            }

            var finalUrl = page.Url;
            _logger.LogDebug("[{Channel}] Navigation completed. Initial final URL: {Url}", "NewsChannel", finalUrl);

            // 2. Resolve Host Specific Scraper Adapter
            var uri = new Uri(url);
            var host = uri.Host;
            var adapter = _scraperAdapters.FirstOrDefault(a => host.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));

            IPage targetPage = page;

            if (adapter != null)
            {
                _logger.LogInformation("[{Channel}] Executing adapter redirect check for host: {Host}", "NewsChannel", adapter.Hostname);
                try
                {
                    var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);

                    // Loop Protection: Navigate only if redirect target is a new URL
                    if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) && 
                        !string.Equals(page.Url, resolvedRedirectUrl, StringComparison.OrdinalIgnoreCase))
                    {
                        _logger.LogInformation("[{Channel}] Redirect resolved to target URL: {Url}", "NewsChannel", resolvedRedirectUrl);

                        var refererUrl = page.Url;
                        finalUrl = resolvedRedirectUrl;

                        var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
                        {
                            WaitUntil = WaitUntilState.DOMContentLoaded,
                            Timeout = 30000,
                            Referer = refererUrl
                        });

                        if (redirectResponse == null)
                        {
                            _logger.LogWarning("[{Channel}] Failed to load response for redirect URL: {Url}", "NewsChannel", resolvedRedirectUrl);
                        }
                        else
                        {
                            finalUrl = page.Url;
                        }

                        // Check if redirect opened a new tab/popup
                        var matchedPage = page.Context.Pages.FirstOrDefault(p => p.Url == resolvedRedirectUrl);
                        if (matchedPage != null)
                        {
                            targetPage = matchedPage;
                        }
                    }
                }
                catch (Exception ex)
                {
                    _logger.LogWarning(ex, "[{Channel}] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", "NewsChannel", adapter.Hostname);
                }
            }

            // 3. Re-resolve detail page adapter for final target page
            var targetUri = new Uri(targetPage.Url);
            var targetHost = targetUri.Host;
            var targetAdapter = _scraperAdapters.FirstOrDefault(a => targetHost.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));

            // Wait a brief moment for dynamic scripts / DOM settling
            await targetPage.WaitForTimeoutAsync(1000);

            // 4. Extract Content via Mozilla Readability (or Adapter Fallback)
            string extractedText = string.Empty;

            if (targetAdapter != null)
            {
                var readabilityResult = await targetAdapter.ExtractArticleContentAsync(targetPage);
                if (readabilityResult != null && !string.IsNullOrWhiteSpace(readabilityResult.TextContent))
                {
                    extractedText = readabilityResult.TextContent;
                }
            }

            // Standard Fallback: Body Text / Selector Extraction
            if (string.IsNullOrWhiteSpace(extractedText))
            {
                var bodySelector = targetAdapter?.ArticleBodySelector ?? "body";
                var locator = targetPage.Locator(bodySelector);

                if (await locator.CountAsync() > 0)
                {
                    extractedText = await locator.First.InnerTextAsync();
                }

                if (string.IsNullOrWhiteSpace(extractedText))
                {
                    extractedText = await targetPage.EvaluateAsync<string>(
                        $"() => document.querySelector('{bodySelector}')?.innerText ?? ''");
                }
            }

            return (finalUrl, extractedText?.Trim() ?? string.Empty);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "[{Channel}] Failed to scrape page content from URL: {Url}", "NewsChannel", url);
            throw;
        }
        finally
        {
            await context.CloseAsync();
        }
    }

    /// <summary>
    /// Thread-safe singleton initialization of the Chromium browser instance.
    /// </summary>
    private async Task<IBrowser> GetOrInitBrowserAsync()
    {
        if (_browser != null && _browser.IsConnected)
        {
            return _browser;
        }

        await _browserLock.WaitAsync();
        try
        {
            if (_browser != null && _browser.IsConnected)
            {
                return _browser;
            }

            _playwright ??= await Playwright.CreateAsync();
            _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
            {
                Headless = true,
                Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
            });

            _logger.LogInformation("[{Channel}] Initialized shared Chromium browser instance.", "NewsChannel");
            return _browser;
        }
        finally
        {
            _browserLock.Release();
        }
    }

    /// <summary>
    /// Disposes the Playwright and Browser instances cleanly during service shutdown.
    /// </summary>
    public async ValueTask DisposeAsync()
    {
        if (_browser != null)
        {
            await _browser.CloseAsync();
            await _browser.DisposeAsync();
        }

        _playwright?.Dispose();
        _browserLock.Dispose();

        GC.SuppressFinalize(this);
    }
}