feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts
This commit is contained in:
@@ -1,12 +1,13 @@
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace FinlyticNews.Adapters.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Record holding structured article content extracted via Mozilla Readability.
|
||||
/// Record holding structured article content extracted from a news webpage.
|
||||
/// </summary>
|
||||
public record ScrapedArticleResult(
|
||||
string Title,
|
||||
@@ -14,11 +15,13 @@ public record ScrapedArticleResult(
|
||||
string HtmlContent,
|
||||
string? Author,
|
||||
string? Excerpt,
|
||||
string FinalUrl
|
||||
string FinalUrl,
|
||||
DateTime? PublishedAt = null,
|
||||
string? Language = null
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Abstract base class representing a website-specific scraping adapter.
|
||||
/// Abstract base class representing a website-specific scraping adapter with heuristic extraction and DOM cleanup.
|
||||
/// </summary>
|
||||
public abstract class ArticleScraperAdapter
|
||||
{
|
||||
@@ -33,9 +36,17 @@ public abstract class ArticleScraperAdapter
|
||||
public virtual string? ReadMoreSelector => null;
|
||||
|
||||
/// <summary>
|
||||
/// Fallback CSS selector targeting the article body text elements if Readability fails.
|
||||
/// Fallback CSS selector targeting the article body text elements if Readability/JSON-LD fails.
|
||||
/// </summary>
|
||||
public virtual string ArticleBodySelector => "body";
|
||||
public virtual string ArticleBodySelector => "article, main, div.article-content, div.article-text, div.entry-content, body";
|
||||
|
||||
private static readonly Regex RegulatoryDisclaimerRegex = new(
|
||||
@"(Interessenkonflikt[e:]|Offenlegung nach §|Risikohinweis:|Disclaimer:|Hinweis auf Interessenkonflikte|Keine Anlageberatung).*$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline);
|
||||
|
||||
private static readonly Regex TitlePortalSuffixRegex = new(
|
||||
@"\s*[-|–—]\s*(DER AKTIONÄR|ARIVA\.DE|IT-Times|onvista|wallstreet:online|Sharedeals\.de|boerse\.de|Handelsblatt|WirtschaftsWoche|finanzen\.net|Finanznachrichten|XTB|Lynx|T3n|ntg24|Moneycab).*$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Tries to resolve 'Read More' links or external redirects.
|
||||
@@ -50,7 +61,6 @@ public abstract class ArticleScraperAdapter
|
||||
{
|
||||
try
|
||||
{
|
||||
// Playwright modern pattern for combined popup or navigation handling
|
||||
var popupTask = page.Context.WaitForPageAsync(new() { Timeout = 6000 });
|
||||
var navTask = page.WaitForURLAsync(url => url != page.Url, new() { Timeout = 6000 });
|
||||
|
||||
@@ -69,7 +79,7 @@ public abstract class ArticleScraperAdapter
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Timeout / Click failed -> Fallback to current URL
|
||||
// Fallback to current URL if click/navigation times out
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,21 +87,119 @@ public abstract class ArticleScraperAdapter
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Injects Mozilla's Readability.js into the Playwright page to parse article content cleanly.
|
||||
/// Extracts structured article content using JSON-LD metadata, DOM noise pruning, and Mozilla Readability.
|
||||
/// </summary>
|
||||
public virtual async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
|
||||
{
|
||||
try
|
||||
{
|
||||
// 1. Inject Mozilla Readability Standalone JS Bundle via CDN
|
||||
// 1. Remove DOM noise and ads before extraction
|
||||
await page.EvaluateAsync(@"() => {
|
||||
const noiseSelectors = [
|
||||
'nav', 'header', 'footer', 'aside', '.cookie-banner', '#consent',
|
||||
'[id*=""ad-""], [class*=""banner""], [class*=""outbrain""], [class*=""taboola""]',
|
||||
'[class*=""share-buttons""], .social-media, .related-articles, .comments',
|
||||
'#forum_box', '.newsletter-box', '.audio-player', '.tags-container'
|
||||
];
|
||||
for (const sel of noiseSelectors) {
|
||||
document.querySelectorAll(sel).forEach(el => el.remove());
|
||||
}
|
||||
}");
|
||||
|
||||
// 2. Extract JSON-LD (Schema.org) if available
|
||||
var jsonLdResult = await page.EvaluateAsync<JsonElement?>(@"() => {
|
||||
const scripts = document.querySelectorAll('script[type=""application/ld+json""]');
|
||||
for (const script of scripts) {
|
||||
try {
|
||||
const data = JSON.parse(script.textContent);
|
||||
const items = Array.isArray(data) ? data : (data['@graph'] || [data]);
|
||||
for (const item of items) {
|
||||
if (item['@type'] === 'NewsArticle' || item['@type'] === 'Article' || item['@type'] === 'Report') {
|
||||
return {
|
||||
headline: item.headline || null,
|
||||
description: item.description || null,
|
||||
articleBody: item.articleBody || null,
|
||||
author: typeof item.author === 'object' ? item.author.name : item.author,
|
||||
datePublished: item.datePublished || item.dateCreated || null
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
return null;
|
||||
}");
|
||||
|
||||
string? title = null;
|
||||
string? excerpt = null;
|
||||
string? author = null;
|
||||
DateTime? publishedAt = null;
|
||||
|
||||
if (jsonLdResult.HasValue && jsonLdResult.Value.ValueKind != JsonValueKind.Null)
|
||||
{
|
||||
var root = jsonLdResult.Value;
|
||||
if (root.TryGetProperty("headline", out var hl) && hl.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
title = CleanTitle(hl.GetString());
|
||||
}
|
||||
if (root.TryGetProperty("description", out var desc) && desc.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
excerpt = desc.GetString();
|
||||
}
|
||||
if (root.TryGetProperty("author", out var auth) && auth.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
author = auth.GetString();
|
||||
}
|
||||
if (root.TryGetProperty("datePublished", out var dp) && dp.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
if (DateTime.TryParse(dp.GetString(), out var dt)) publishedAt = dt;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Fallback to OpenGraph and Meta tags for missing metadata
|
||||
if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(excerpt))
|
||||
{
|
||||
var metaResult = await page.EvaluateAsync<JsonElement?>(@"() => {
|
||||
const getMeta = (prop) => {
|
||||
const el = document.querySelector(`meta[property=""${prop}""], meta[name=""${prop}""]`);
|
||||
return el ? el.getAttribute('content') : null;
|
||||
};
|
||||
return {
|
||||
ogTitle: getMeta('og:title') || getMeta('twitter:title'),
|
||||
ogDesc: getMeta('og:description') || getMeta('description') || getMeta('twitter:description'),
|
||||
author: getMeta('author') || getMeta('article:author'),
|
||||
pubDate: getMeta('article:published_time') || getMeta('date')
|
||||
};
|
||||
}");
|
||||
|
||||
if (metaResult.HasValue && metaResult.Value.ValueKind != JsonValueKind.Null)
|
||||
{
|
||||
var m = metaResult.Value;
|
||||
if (string.IsNullOrWhiteSpace(title) && m.TryGetProperty("ogTitle", out var ogT) && ogT.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
title = CleanTitle(ogT.GetString());
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(excerpt) && m.TryGetProperty("ogDesc", out var ogD) && ogD.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
excerpt = ogD.GetString();
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(author) && m.TryGetProperty("author", out var ogA) && ogA.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
author = ogA.GetString();
|
||||
}
|
||||
if (!publishedAt.HasValue && m.TryGetProperty("pubDate", out var ogP) && ogP.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
if (DateTime.TryParse(ogP.GetString(), out var dt)) publishedAt = dt;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Inject Mozilla Readability to extract main content
|
||||
await page.AddScriptTagAsync(new PageAddScriptTagOptions
|
||||
{
|
||||
Url = "https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js"
|
||||
});
|
||||
|
||||
// 2. Execute Readability in the Browser Context
|
||||
var jsScript = @"
|
||||
() => {
|
||||
var readabilityResult = await page.EvaluateAsync<JsonElement?>(@"() => {
|
||||
if (typeof Readability === 'undefined') return null;
|
||||
const documentClone = document.cloneNode(true);
|
||||
const article = new Readability(documentClone).parse();
|
||||
@@ -104,37 +212,61 @@ public abstract class ArticleScraperAdapter
|
||||
author: article.byline || null,
|
||||
excerpt: article.excerpt || null
|
||||
};
|
||||
}";
|
||||
}");
|
||||
|
||||
var jsonResult = await page.EvaluateAsync<JsonElement?>(jsScript);
|
||||
|
||||
if (jsonResult.HasValue && jsonResult.Value.ValueKind != JsonValueKind.Null)
|
||||
if (readabilityResult.HasValue && readabilityResult.Value.ValueKind != JsonValueKind.Null)
|
||||
{
|
||||
var root = jsonResult.Value;
|
||||
var r = readabilityResult.Value;
|
||||
var rawText = r.GetProperty("textContent").GetString()?.Trim() ?? string.Empty;
|
||||
var cleanedText = TruncateDisclaimers(rawText);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
title = CleanTitle(r.GetProperty("title").GetString() ?? await page.TitleAsync());
|
||||
}
|
||||
|
||||
return new ScrapedArticleResult(
|
||||
Title: root.GetProperty("title").GetString() ?? page.TitleAsync().Result,
|
||||
TextContent: root.GetProperty("textContent").GetString()?.Trim() ?? string.Empty,
|
||||
HtmlContent: root.GetProperty("htmlContent").GetString() ?? string.Empty,
|
||||
Author: root.GetProperty("author").GetString(),
|
||||
Excerpt: root.GetProperty("excerpt").GetString(),
|
||||
FinalUrl: page.Url
|
||||
Title: title ?? string.Empty,
|
||||
TextContent: cleanedText,
|
||||
HtmlContent: r.GetProperty("htmlContent").GetString() ?? string.Empty,
|
||||
Author: author ?? r.GetProperty("author").GetString(),
|
||||
Excerpt: excerpt ?? r.GetProperty("excerpt").GetString(),
|
||||
FinalUrl: page.Url,
|
||||
PublishedAt: publishedAt
|
||||
);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Readability Injection / Parsing failed -> Fallback to standard selector parsing
|
||||
// Readability/Script injection failed -> Fallback to selector parsing
|
||||
}
|
||||
|
||||
// Fallback: Manuelles Auslesen via ArticleBodySelector
|
||||
var bodyText = await page.Locator(ArticleBodySelector).InnerTextAsync();
|
||||
// 5. Selector-based fallback extraction
|
||||
var bodyLocator = page.Locator(ArticleBodySelector).First;
|
||||
var bodyText = await bodyLocator.CountAsync() > 0 ? await bodyLocator.InnerTextAsync() : await page.InnerTextAsync("body");
|
||||
var fallbackCleaned = TruncateDisclaimers(bodyText.Trim());
|
||||
|
||||
return new ScrapedArticleResult(
|
||||
Title: await page.TitleAsync(),
|
||||
TextContent: bodyText.Trim(),
|
||||
HtmlContent: await page.Locator(ArticleBodySelector).InnerHTMLAsync(),
|
||||
Title: CleanTitle(await page.TitleAsync()),
|
||||
TextContent: fallbackCleaned,
|
||||
HtmlContent: string.Empty,
|
||||
Author: null,
|
||||
Excerpt: null,
|
||||
FinalUrl: page.Url
|
||||
);
|
||||
}
|
||||
|
||||
protected static string CleanTitle(string? title)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(title)) return string.Empty;
|
||||
var cleaned = TitlePortalSuffixRegex.Replace(title.Trim(), "");
|
||||
return cleaned.Trim();
|
||||
}
|
||||
|
||||
protected static string TruncateDisclaimers(string text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text)) return string.Empty;
|
||||
var truncated = RegulatoryDisclaimerRegex.Replace(text, "");
|
||||
return truncated.Trim();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user