Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(news): dynamic settings, IFinlyticLogger, live log streaming, and EF migration

Browse Source
This commit is contained in:
Kleidukos
2026-08-15 21:30:01 +02:00
parent a1f2b888f6
commit a94c36a878
12 changed files with 800 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticNews/Services/PlaywrightScraperService.cs
+54 -81
View File
@@ -3,8 +3,9 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services;
using FinlyticNews.Adapters.Scraping;
using Microsoft.Extensions.Logging;
using FinlyticNews.Util;
using Microsoft.Playwright;
namespace FinlyticNews.Services;
@@ -25,7 +26,7 @@ public interface IPlaywrightScraperService
/// <inheritdoc />
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
{
private readonly ILogger<PlaywrightScraperService> _logger;
private readonly IFinlyticLogger<PlaywrightScraperService> _finlyticLogger;
private readonly IEnumerable<ArticleScraperAdapter> _scraperAdapters;
private IPlaywright? _playwright;
@@ -36,21 +37,20 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
/// Initializes a new instance of the <see cref="PlaywrightScraperService"/> class.
/// </summary>
public PlaywrightScraperService(
ILogger<PlaywrightScraperService> logger,
IFinlyticLogger<PlaywrightScraperService> finlyticLogger,
IEnumerable<ArticleScraperAdapter> scraperAdapters)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_scraperAdapters = scraperAdapters;
}
/// <inheritdoc />
public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
{
_logger.LogInformation("[{Channel}] Launching browser context to scrape article: {Url}", "NewsChannel", url);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
var browser = await GetOrInitBrowserAsync();
// Fast, isolated browser context (incognito tab environment) per article
await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
{
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -61,7 +61,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
try
{
// 1. Initial Navigation with DOMContentLoaded wait
var response = await page.GotoAsync(url, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
@@ -74,115 +73,86 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
var finalUrl = page.Url;
_logger.LogDebug("[{Channel}] Navigation completed. Initial final URL: {Url}", "NewsChannel", finalUrl);
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Navigation completed. Initial final URL: {Url}", finalUrl);
// 2. Resolve Host Specific Scraper Adapter
var uri = new Uri(url);
var host = uri.Host;
var adapter = _scraperAdapters.FirstOrDefault(a => host.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
IPage targetPage = page;
var host = new Uri(finalUrl).Host;
var adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
if (adapter != null)
{
_logger.LogInformation("[{Channel}] Executing adapter redirect check for host: {Host}", "NewsChannel", adapter.Hostname);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Executing adapter redirect check for host: {Host}", adapter.Hostname);
try
{
var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
// Loop Protection: Navigate only if redirect target is a new URL
if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) &&
!string.Equals(page.Url, resolvedRedirectUrl, StringComparison.OrdinalIgnoreCase))
!resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase))
{
_logger.LogInformation("[{Channel}] Redirect resolved to target URL: {Url}", "NewsChannel", resolvedRedirectUrl);
var refererUrl = page.Url;
finalUrl = resolvedRedirectUrl;
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
{
WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = 30000,
Referer = refererUrl
Timeout = 30000
});
if (redirectResponse == null)
{
_logger.LogWarning("[{Channel}] Failed to load response for redirect URL: {Url}", "NewsChannel", resolvedRedirectUrl);
}
else
{
finalUrl = page.Url;
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Failed to load response for redirect URL: {Url}", resolvedRedirectUrl);
}
// Check if redirect opened a new tab/popup
var matchedPage = page.Context.Pages.FirstOrDefault(p => p.Url == resolvedRedirectUrl);
if (matchedPage != null)
{
targetPage = matchedPage;
}
finalUrl = page.Url;
host = new Uri(finalUrl).Host;
adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", "NewsChannel", adapter.Hostname);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter.Hostname);
}
}
// 3. Re-resolve detail page adapter for final target page
var targetUri = new Uri(targetPage.Url);
var targetHost = targetUri.Host;
var targetAdapter = _scraperAdapters.FirstOrDefault(a => targetHost.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
// Wait a brief moment for dynamic scripts / DOM settling
await targetPage.WaitForTimeoutAsync(1000);
// 4. Extract Content via Mozilla Readability (or Adapter Fallback)
string extractedText = string.Empty;
if (targetAdapter != null)
string content;
if (adapter != null)
{
var readabilityResult = await targetAdapter.ExtractArticleContentAsync(targetPage);
if (readabilityResult != null && !string.IsNullOrWhiteSpace(readabilityResult.TextContent))
{
extractedText = readabilityResult.TextContent;
}
var result = await adapter.ExtractArticleContentAsync(page);
content = result?.TextContent ?? await FallbackExtractContentAsync(page);
}
// Standard Fallback: Body Text / Selector Extraction
if (string.IsNullOrWhiteSpace(extractedText))
else
{
var bodySelector = targetAdapter?.ArticleBodySelector ?? "body";
var locator = targetPage.Locator(bodySelector);
if (await locator.CountAsync() > 0)
{
extractedText = await locator.First.InnerTextAsync();
}
if (string.IsNullOrWhiteSpace(extractedText))
{
extractedText = await targetPage.EvaluateAsync<string>(
$"() => document.querySelector('{bodySelector}')?.innerText ?? ''");
}
content = await FallbackExtractContentAsync(page);
}
return (finalUrl, extractedText?.Trim() ?? string.Empty);
return (finalUrl, content);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to scrape page content from URL: {Url}", "NewsChannel", url);
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
throw;
}
finally
{
await context.CloseAsync();
await page.CloseAsync();
}
}
/// <summary>
/// Thread-safe singleton initialization of the Chromium browser instance.
/// </summary>
private async Task<string> FallbackExtractContentAsync(IPage page)
{
var innerText = await page.EvaluateAsync<string>(@"() => {
const scripts = document.querySelectorAll('script, style, noscript, nav, header, footer, iframe, svg');
scripts.forEach(s => s.remove());
const main = document.querySelector('article, main, .article-content, #content, .story-body') || document.body;
return main ? main.innerText : document.body.innerText;
}");
return innerText?.Trim() ?? string.Empty;
}
private async Task<IBrowser> GetOrInitBrowserAsync()
{
if (_browser != null && _browser.IsConnected)
@@ -202,10 +172,16 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
{
Headless = true,
Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
Args = new[]
{
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu"
}
});
_logger.LogInformation("[{Channel}] Initialized shared Chromium browser instance.", "NewsChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Initialized shared Chromium browser instance.");
return _browser;
}
finally
@@ -214,9 +190,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
}
/// <summary>
/// Disposes the Playwright and Browser instances cleanly during service shutdown.
/// </summary>
public async ValueTask DisposeAsync()
{
if (_browser != null)