feat(News): refactor news service and add sentiment tracking
This commit is contained in:
@@ -0,0 +1,233 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticNews.Adapters.Scraping;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
|
||||
/// </summary>
|
||||
public interface IPlaywrightScraperService
|
||||
{
|
||||
/// <summary>
|
||||
/// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
|
||||
/// </summary>
|
||||
/// <param name="url">The initial article URL.</param>
|
||||
/// <returns>A tuple containing the final resolved URL and the extracted raw text content.</returns>
|
||||
Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
|
||||
{
|
||||
private readonly ILogger<PlaywrightScraperService> _logger;
|
||||
private readonly IEnumerable<ArticleScraperAdapter> _scraperAdapters;
|
||||
|
||||
private IPlaywright? _playwright;
|
||||
private IBrowser? _browser;
|
||||
private readonly SemaphoreSlim _browserLock = new(1, 1);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PlaywrightScraperService"/> class.
|
||||
/// </summary>
|
||||
public PlaywrightScraperService(
|
||||
ILogger<PlaywrightScraperService> logger,
|
||||
IEnumerable<ArticleScraperAdapter> scraperAdapters)
|
||||
{
|
||||
_logger = logger;
|
||||
_scraperAdapters = scraperAdapters;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Launching browser context to scrape article: {Url}", "NewsChannel", url);
|
||||
|
||||
var browser = await GetOrInitBrowserAsync();
|
||||
|
||||
// Fast, isolated browser context (incognito tab environment) per article
|
||||
await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
|
||||
{
|
||||
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
ViewportSize = new ViewportSize { Width = 1280, Height = 800 }
|
||||
});
|
||||
|
||||
var page = await context.NewPageAsync();
|
||||
|
||||
try
|
||||
{
|
||||
// 1. Initial Navigation with DOMContentLoaded wait
|
||||
var response = await page.GotoAsync(url, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 30000
|
||||
});
|
||||
|
||||
if (response == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}");
|
||||
}
|
||||
|
||||
var finalUrl = page.Url;
|
||||
_logger.LogDebug("[{Channel}] Navigation completed. Initial final URL: {Url}", "NewsChannel", finalUrl);
|
||||
|
||||
// 2. Resolve Host Specific Scraper Adapter
|
||||
var uri = new Uri(url);
|
||||
var host = uri.Host;
|
||||
var adapter = _scraperAdapters.FirstOrDefault(a => host.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
IPage targetPage = page;
|
||||
|
||||
if (adapter != null)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Executing adapter redirect check for host: {Host}", "NewsChannel", adapter.Hostname);
|
||||
try
|
||||
{
|
||||
var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
|
||||
|
||||
// Loop Protection: Navigate only if redirect target is a new URL
|
||||
if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) &&
|
||||
!string.Equals(page.Url, resolvedRedirectUrl, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Redirect resolved to target URL: {Url}", "NewsChannel", resolvedRedirectUrl);
|
||||
|
||||
var refererUrl = page.Url;
|
||||
finalUrl = resolvedRedirectUrl;
|
||||
|
||||
var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 30000,
|
||||
Referer = refererUrl
|
||||
});
|
||||
|
||||
if (redirectResponse == null)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Failed to load response for redirect URL: {Url}", "NewsChannel", resolvedRedirectUrl);
|
||||
}
|
||||
else
|
||||
{
|
||||
finalUrl = page.Url;
|
||||
}
|
||||
|
||||
// Check if redirect opened a new tab/popup
|
||||
var matchedPage = page.Context.Pages.FirstOrDefault(p => p.Url == resolvedRedirectUrl);
|
||||
if (matchedPage != null)
|
||||
{
|
||||
targetPage = matchedPage;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[{Channel}] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", "NewsChannel", adapter.Hostname);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Re-resolve detail page adapter for final target page
|
||||
var targetUri = new Uri(targetPage.Url);
|
||||
var targetHost = targetUri.Host;
|
||||
var targetAdapter = _scraperAdapters.FirstOrDefault(a => targetHost.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// Wait a brief moment for dynamic scripts / DOM settling
|
||||
await targetPage.WaitForTimeoutAsync(1000);
|
||||
|
||||
// 4. Extract Content via Mozilla Readability (or Adapter Fallback)
|
||||
string extractedText = string.Empty;
|
||||
|
||||
if (targetAdapter != null)
|
||||
{
|
||||
var readabilityResult = await targetAdapter.ExtractArticleContentAsync(targetPage);
|
||||
if (readabilityResult != null && !string.IsNullOrWhiteSpace(readabilityResult.TextContent))
|
||||
{
|
||||
extractedText = readabilityResult.TextContent;
|
||||
}
|
||||
}
|
||||
|
||||
// Standard Fallback: Body Text / Selector Extraction
|
||||
if (string.IsNullOrWhiteSpace(extractedText))
|
||||
{
|
||||
var bodySelector = targetAdapter?.ArticleBodySelector ?? "body";
|
||||
var locator = targetPage.Locator(bodySelector);
|
||||
|
||||
if (await locator.CountAsync() > 0)
|
||||
{
|
||||
extractedText = await locator.First.InnerTextAsync();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(extractedText))
|
||||
{
|
||||
extractedText = await targetPage.EvaluateAsync<string>(
|
||||
$"() => document.querySelector('{bodySelector}')?.innerText ?? ''");
|
||||
}
|
||||
}
|
||||
|
||||
return (finalUrl, extractedText?.Trim() ?? string.Empty);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to scrape page content from URL: {Url}", "NewsChannel", url);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await context.CloseAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe singleton initialization of the Chromium browser instance.
|
||||
/// </summary>
|
||||
private async Task<IBrowser> GetOrInitBrowserAsync()
|
||||
{
|
||||
if (_browser != null && _browser.IsConnected)
|
||||
{
|
||||
return _browser;
|
||||
}
|
||||
|
||||
await _browserLock.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (_browser != null && _browser.IsConnected)
|
||||
{
|
||||
return _browser;
|
||||
}
|
||||
|
||||
_playwright ??= await Playwright.CreateAsync();
|
||||
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
|
||||
{
|
||||
Headless = true,
|
||||
Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
|
||||
});
|
||||
|
||||
_logger.LogInformation("[{Channel}] Initialized shared Chromium browser instance.", "NewsChannel");
|
||||
return _browser;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_browserLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the Playwright and Browser instances cleanly during service shutdown.
|
||||
/// </summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_browser != null)
|
||||
{
|
||||
await _browser.CloseAsync();
|
||||
await _browser.DisposeAsync();
|
||||
}
|
||||
|
||||
_playwright?.Dispose();
|
||||
_browserLock.Dispose();
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user