272 lines
12 KiB
C#
272 lines
12 KiB
C#
using System;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading.Tasks;
|
|
using Microsoft.Playwright;
|
|
|
|
namespace FinlyticNews.Adapters.Scraping;
|
|
|
|
/// <summary>
|
|
/// Record holding structured article content extracted from a news webpage.
|
|
/// </summary>
|
|
public record ScrapedArticleResult(
|
|
string Title,
|
|
string TextContent,
|
|
string HtmlContent,
|
|
string? Author,
|
|
string? Excerpt,
|
|
string FinalUrl,
|
|
DateTime? PublishedAt = null,
|
|
string? Language = null
|
|
);
|
|
|
|
/// <summary>
|
|
/// Abstract base class representing a website-specific scraping adapter with heuristic extraction and DOM cleanup.
|
|
/// </summary>
|
|
public abstract class ArticleScraperAdapter
|
|
{
|
|
/// <summary>
|
|
/// Gets the target domain hostname of the news website (e.g. "finanznachrichten.de").
|
|
/// </summary>
|
|
public abstract string Hostname { get; }
|
|
|
|
/// <summary>
|
|
/// Gets the CSS selector to identify and click the 'Read full article' redirect button, if applicable.
|
|
/// </summary>
|
|
public virtual string? ReadMoreSelector => null;
|
|
|
|
/// <summary>
|
|
/// Fallback CSS selector targeting the article body text elements if Readability/JSON-LD fails.
|
|
/// </summary>
|
|
public virtual string ArticleBodySelector => "article, main, div.article-content, div.article-text, div.entry-content, body";
|
|
|
|
private static readonly Regex RegulatoryDisclaimerRegex = new(
|
|
@"(Interessenkonflikt[e:]|Offenlegung nach §|Risikohinweis:|Disclaimer:|Hinweis auf Interessenkonflikte|Keine Anlageberatung).*$",
|
|
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline);
|
|
|
|
private static readonly Regex TitlePortalSuffixRegex = new(
|
|
@"\s*[-|–—]\s*(DER AKTIONÄR|ARIVA\.DE|IT-Times|onvista|wallstreet:online|Sharedeals\.de|boerse\.de|Handelsblatt|WirtschaftsWoche|finanzen\.net|Finanznachrichten|XTB|Lynx|T3n|ntg24|Moneycab).*$",
|
|
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
|
|
|
/// <summary>
|
|
/// Tries to resolve 'Read More' links or external redirects.
|
|
/// </summary>
|
|
public virtual async Task<string?> TryResolveRedirectUrlAsync(IPage page)
|
|
{
|
|
var selector = ReadMoreSelector;
|
|
if (string.IsNullOrWhiteSpace(selector)) return null;
|
|
|
|
var readMoreButton = page.Locator(selector).First;
|
|
if (await readMoreButton.CountAsync() > 0 && await readMoreButton.IsVisibleAsync())
|
|
{
|
|
try
|
|
{
|
|
var popupTask = page.Context.WaitForPageAsync(new() { Timeout = 6000 });
|
|
var navTask = page.WaitForURLAsync(url => url != page.Url, new() { Timeout = 6000 });
|
|
|
|
await readMoreButton.ClickAsync();
|
|
|
|
var completedTask = await Task.WhenAny(popupTask, navTask);
|
|
if (completedTask == popupTask)
|
|
{
|
|
var popup = await popupTask;
|
|
await popup.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
|
|
return popup.Url;
|
|
}
|
|
|
|
await navTask;
|
|
return page.Url;
|
|
}
|
|
catch
|
|
{
|
|
// Fallback to current URL if click/navigation times out
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Extracts structured article content using JSON-LD metadata, DOM noise pruning, and Mozilla Readability.
|
|
/// </summary>
|
|
public virtual async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
|
|
{
|
|
try
|
|
{
|
|
// 1. Remove DOM noise and ads before extraction
|
|
await page.EvaluateAsync(@"() => {
|
|
const noiseSelectors = [
|
|
'nav', 'header', 'footer', 'aside', '.cookie-banner', '#consent',
|
|
'[id*=""ad-""], [class*=""banner""], [class*=""outbrain""], [class*=""taboola""]',
|
|
'[class*=""share-buttons""], .social-media, .related-articles, .comments',
|
|
'#forum_box', '.newsletter-box', '.audio-player', '.tags-container'
|
|
];
|
|
for (const sel of noiseSelectors) {
|
|
document.querySelectorAll(sel).forEach(el => el.remove());
|
|
}
|
|
}");
|
|
|
|
// 2. Extract JSON-LD (Schema.org) if available
|
|
var jsonLdResult = await page.EvaluateAsync<JsonElement?>(@"() => {
|
|
const scripts = document.querySelectorAll('script[type=""application/ld+json""]');
|
|
for (const script of scripts) {
|
|
try {
|
|
const data = JSON.parse(script.textContent);
|
|
const items = Array.isArray(data) ? data : (data['@graph'] || [data]);
|
|
for (const item of items) {
|
|
if (item['@type'] === 'NewsArticle' || item['@type'] === 'Article' || item['@type'] === 'Report') {
|
|
return {
|
|
headline: item.headline || null,
|
|
description: item.description || null,
|
|
articleBody: item.articleBody || null,
|
|
author: typeof item.author === 'object' ? item.author.name : item.author,
|
|
datePublished: item.datePublished || item.dateCreated || null
|
|
};
|
|
}
|
|
}
|
|
} catch (e) {}
|
|
}
|
|
return null;
|
|
}");
|
|
|
|
string? title = null;
|
|
string? excerpt = null;
|
|
string? author = null;
|
|
DateTime? publishedAt = null;
|
|
|
|
if (jsonLdResult.HasValue && jsonLdResult.Value.ValueKind != JsonValueKind.Null)
|
|
{
|
|
var root = jsonLdResult.Value;
|
|
if (root.TryGetProperty("headline", out var hl) && hl.ValueKind == JsonValueKind.String)
|
|
{
|
|
title = CleanTitle(hl.GetString());
|
|
}
|
|
if (root.TryGetProperty("description", out var desc) && desc.ValueKind == JsonValueKind.String)
|
|
{
|
|
excerpt = desc.GetString();
|
|
}
|
|
if (root.TryGetProperty("author", out var auth) && auth.ValueKind == JsonValueKind.String)
|
|
{
|
|
author = auth.GetString();
|
|
}
|
|
if (root.TryGetProperty("datePublished", out var dp) && dp.ValueKind == JsonValueKind.String)
|
|
{
|
|
if (DateTime.TryParse(dp.GetString(), out var dt)) publishedAt = dt;
|
|
}
|
|
}
|
|
|
|
// 3. Fallback to OpenGraph and Meta tags for missing metadata
|
|
if (string.IsNullOrWhiteSpace(title) || string.IsNullOrWhiteSpace(excerpt))
|
|
{
|
|
var metaResult = await page.EvaluateAsync<JsonElement?>(@"() => {
|
|
const getMeta = (prop) => {
|
|
const el = document.querySelector(`meta[property=""${prop}""], meta[name=""${prop}""]`);
|
|
return el ? el.getAttribute('content') : null;
|
|
};
|
|
return {
|
|
ogTitle: getMeta('og:title') || getMeta('twitter:title'),
|
|
ogDesc: getMeta('og:description') || getMeta('description') || getMeta('twitter:description'),
|
|
author: getMeta('author') || getMeta('article:author'),
|
|
pubDate: getMeta('article:published_time') || getMeta('date')
|
|
};
|
|
}");
|
|
|
|
if (metaResult.HasValue && metaResult.Value.ValueKind != JsonValueKind.Null)
|
|
{
|
|
var m = metaResult.Value;
|
|
if (string.IsNullOrWhiteSpace(title) && m.TryGetProperty("ogTitle", out var ogT) && ogT.ValueKind == JsonValueKind.String)
|
|
{
|
|
title = CleanTitle(ogT.GetString());
|
|
}
|
|
if (string.IsNullOrWhiteSpace(excerpt) && m.TryGetProperty("ogDesc", out var ogD) && ogD.ValueKind == JsonValueKind.String)
|
|
{
|
|
excerpt = ogD.GetString();
|
|
}
|
|
if (string.IsNullOrWhiteSpace(author) && m.TryGetProperty("author", out var ogA) && ogA.ValueKind == JsonValueKind.String)
|
|
{
|
|
author = ogA.GetString();
|
|
}
|
|
if (!publishedAt.HasValue && m.TryGetProperty("pubDate", out var ogP) && ogP.ValueKind == JsonValueKind.String)
|
|
{
|
|
if (DateTime.TryParse(ogP.GetString(), out var dt)) publishedAt = dt;
|
|
}
|
|
}
|
|
}
|
|
|
|
// 4. Inject Mozilla Readability to extract main content
|
|
await page.AddScriptTagAsync(new PageAddScriptTagOptions
|
|
{
|
|
Url = "https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js"
|
|
});
|
|
|
|
var readabilityResult = await page.EvaluateAsync<JsonElement?>(@"() => {
|
|
if (typeof Readability === 'undefined') return null;
|
|
const documentClone = document.cloneNode(true);
|
|
const article = new Readability(documentClone).parse();
|
|
if (!article) return null;
|
|
|
|
return {
|
|
title: article.title || '',
|
|
textContent: article.textContent || '',
|
|
htmlContent: article.content || '',
|
|
author: article.byline || null,
|
|
excerpt: article.excerpt || null
|
|
};
|
|
}");
|
|
|
|
if (readabilityResult.HasValue && readabilityResult.Value.ValueKind != JsonValueKind.Null)
|
|
{
|
|
var r = readabilityResult.Value;
|
|
var rawText = r.GetProperty("textContent").GetString()?.Trim() ?? string.Empty;
|
|
var cleanedText = TruncateDisclaimers(rawText);
|
|
|
|
if (string.IsNullOrWhiteSpace(title))
|
|
{
|
|
title = CleanTitle(r.GetProperty("title").GetString() ?? await page.TitleAsync());
|
|
}
|
|
|
|
return new ScrapedArticleResult(
|
|
Title: title ?? string.Empty,
|
|
TextContent: cleanedText,
|
|
HtmlContent: r.GetProperty("htmlContent").GetString() ?? string.Empty,
|
|
Author: author ?? r.GetProperty("author").GetString(),
|
|
Excerpt: excerpt ?? r.GetProperty("excerpt").GetString(),
|
|
FinalUrl: page.Url,
|
|
PublishedAt: publishedAt
|
|
);
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
// Readability/Script injection failed -> Fallback to selector parsing
|
|
}
|
|
|
|
// 5. Selector-based fallback extraction
|
|
var bodyLocator = page.Locator(ArticleBodySelector).First;
|
|
var bodyText = await bodyLocator.CountAsync() > 0 ? await bodyLocator.InnerTextAsync() : await page.InnerTextAsync("body");
|
|
var fallbackCleaned = TruncateDisclaimers(bodyText.Trim());
|
|
|
|
return new ScrapedArticleResult(
|
|
Title: CleanTitle(await page.TitleAsync()),
|
|
TextContent: fallbackCleaned,
|
|
HtmlContent: string.Empty,
|
|
Author: null,
|
|
Excerpt: null,
|
|
FinalUrl: page.Url
|
|
);
|
|
}
|
|
|
|
protected static string CleanTitle(string? title)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(title)) return string.Empty;
|
|
var cleaned = TitlePortalSuffixRegex.Replace(title.Trim(), "");
|
|
return cleaned.Trim();
|
|
}
|
|
|
|
protected static string TruncateDisclaimers(string text)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(text)) return string.Empty;
|
|
var truncated = RegulatoryDisclaimerRegex.Replace(text, "");
|
|
return truncated.Trim();
|
|
}
|
|
} |