Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

Browse Source
This commit is contained in:
Kleidukos
2026-08-24 21:35:33 +02:00
parent 44b161d509
commit 8112598602
400 changed files with 1812 additions and 113367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticNews/Services/PlaywrightScraperService.cs
+51 -26
View File
@@ -11,16 +11,16 @@ using Microsoft.Playwright;
namespace FinlyticNews.Services;
/// <summary>
/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
/// Defines a headless scraping service for extracting text, metadata, and resolving redirects from news sites.
/// </summary>
public interface IPlaywrightScraperService
{
/// <summary>
/// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
/// Scrapes an article, automatically following redirects and applying site-specific scraper adapters.
/// </summary>
/// <param name="url">The initial article URL.</param>
/// <returns>A tuple containing the final resolved URL and the extracted raw text content.</returns>
Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
/// <returns>A tuple containing the final resolved URL and the structured scrape result.</returns>
Task<(string ResolvedUrl, ScrapedArticleResult Result)> ScrapeArticleAsync(string url);
}
/// <inheritdoc />
@@ -45,9 +45,9 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
/// <inheritdoc />
public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
public async Task<(string ResolvedUrl, ScrapedArticleResult Result)> ScrapeArticleAsync(string url)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
await _finlyticLogger.LogInfoAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
var browser = await GetOrInitBrowserAsync();
@@ -73,8 +73,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
var finalUrl = page.Url;
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Navigation completed. Initial final URL: {Url}", finalUrl);
var host = new Uri(finalUrl).Host;
var adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
@@ -82,14 +80,13 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
if (adapter != null)
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Executing adapter redirect check for host: {Host}", adapter.Hostname);
try
{
var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) &&
!resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase))
{
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
await _finlyticLogger.LogInfoAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
{
@@ -97,11 +94,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
Timeout = 30000
});
if (redirectResponse == null)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Failed to load response for redirect URL: {Url}", resolvedRedirectUrl);
}
finalUrl = page.Url;
host = new Uri(finalUrl).Host;
@@ -112,26 +104,59 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter.Hostname);
await _finlyticLogger.LogWarningAsync(SettingKeys.ScraperChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter?.Hostname ?? host);
}
}
string content;
if (adapter != null)
ScrapedArticleResult? result = null;
for (int attempt = 1; attempt <= 2; attempt++)
{
var result = await adapter.ExtractArticleContentAsync(page);
content = result?.TextContent ?? await FallbackExtractContentAsync(page);
}
else
{
content = await FallbackExtractContentAsync(page);
try
{
if (adapter != null)
{
result = await adapter.ExtractArticleContentAsync(page);
}
if (result == null || string.IsNullOrWhiteSpace(result.TextContent))
{
var bodyText = await FallbackExtractContentAsync(page);
result = new ScrapedArticleResult(
Title: await page.TitleAsync(),
TextContent: bodyText,
HtmlContent: string.Empty,
Author: null,
Excerpt: null,
FinalUrl: finalUrl
);
}
break; // Extraction succeeded without execution context getting destroyed
}
catch (PlaywrightException ex) when (ex.Message.Contains("Execution context was destroyed") && attempt == 1)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Execution context destroyed (likely JS/Meta redirect). Waiting for new page load...");
try
{
await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
}
catch (TimeoutException) { /* Ignored, try extracting anyway */ }
finalUrl = page.Url;
host = new Uri(finalUrl).Host;
adapter = _scraperAdapters.FirstOrDefault(a =>
host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
}
}
return (finalUrl, content);
return (finalUrl, result!);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
await _finlyticLogger.LogErrorAsync(SettingKeys.ScraperChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
throw;
}
finally