Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(News): refactor news service and add sentiment tracking

Browse Source
This commit is contained in:
Kleidukos
2026-08-09 21:01:41 +02:00
parent 605e41cfeb
commit aff831cf58
390 changed files with 115094 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticNews/Adapters/Discovery/ArticleDiscoveryAdapter.cs
+22
View File
@@ -0,0 +1,22 @@
using FinlyticCore.Dtos.News;
namespace FinlyticNews.Adapters.Discovery;
/// <summary>
/// Abstract base class representing an adapter capable of extracting article URLs from web content.
/// </summary>
public abstract class ArticleDiscoveryAdapter
{
/// <summary>
/// Gets the unique adapter keyword name used for matching (e.g. "rss", "html").
/// </summary>
public abstract string Name { get; }
/// <summary>
/// Extracts a list of absolute article URLs from the retrieved web/feed page content.
/// </summary>
/// <param name="content">The raw text or XML content loaded from the source.</param>
/// <param name="sourceUrl">The original URL of the source page (used to resolve relative links).</param>
/// <returns>A list of resolved absolute URLs.</returns>
public abstract List<DiscoveredArticle> ExtractUrls(string content, string sourceUrl);
}
FinlyticNews/Adapters/Discovery/FinanznachrichtenRssDiscoveryAdapter.cs
+93
View File
@@ -0,0 +1,93 @@
using System.Text.RegularExpressions;
using System.Xml.Linq;
using FinlyticCore.Dtos.News;
namespace FinlyticNews.Adapters.Discovery;
/// <summary>
/// Specialized RSS discovery adapter for Finanznachrichten.de, capable of parsing custom fn:isin elements and standard RSS metadata.
/// </summary>
public partial class FinanznachrichtenRssDiscoveryAdapter : ArticleDiscoveryAdapter
{
private static readonly XNamespace FnNamespace = "http://www.finanznachrichten.de/service/rss";
/// <inheritdoc />
public override string Name => "finanznachrichten_rss";
/// <inheritdoc />
public override List<DiscoveredArticle> ExtractUrls(string content, string sourceUrl)
{
if (string.IsNullOrWhiteSpace(content)) return [];
var articles = new List<DiscoveredArticle>();
if (!Uri.TryCreate(sourceUrl, UriKind.Absolute, out var baseUri)) return articles;
try
{
var doc = XDocument.Parse(content);
var channel = doc.Root?.Element("channel");
var channelTitle = channel?.Element("title")?.Value?.Trim();
var channelLanguage = channel?.Element("language")?.Value?.Trim();
var items = doc.Descendants("item");
foreach (var item in items)
{
var link = item.Element("link")?.Value?.Trim() ?? item.Element("guid")?.Value?.Trim();
if (string.IsNullOrWhiteSpace(link)) continue;
// ISIN Extraktion
var isins = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var isinEl in item.Elements(FnNamespace + "isin"))
{
if (string.IsNullOrWhiteSpace(isinEl.Value)) continue;
var parts = isinEl.Value.Split(new[] { ',', ';', ' ' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var part in parts)
{
var trimmed = part.Trim().ToUpperInvariant();
if (IsValidIsin(trimmed))
{
isins.Add(trimmed);
}
}
}
var title = item.Element("title")?.Value?.Trim();
var description = item.Element("description")?.Value?.Trim();
DateTime? pubDate = null;
var pubDateValue = item.Element("pubDate")?.Value;
if (!string.IsNullOrWhiteSpace(pubDateValue) && DateTime.TryParse(pubDateValue, out var parsedDate))
{
pubDate = parsedDate.ToUniversalTime();
}
if (Uri.TryCreate(baseUri, link, out var absoluteUri))
{
articles.Add(new DiscoveredArticle(
Url: absoluteUri.ToString(),
Isins: isins.Count > 0 ? isins.ToList() : null,
Title: title,
Summary: description,
PublishedAt: pubDate,
Language: channelLanguage,
SourceName: channelTitle
));
}
}
}
catch
{
// XML Malformed / Parsing Errors ignorieren
}
// Deduplizierung nach URL
return articles.GroupBy(a => a.Url, StringComparer.OrdinalIgnoreCase)
.Select(g => g.First())
.ToList();
}
private static bool IsValidIsin(string input)
=> input.Length == 12 && char.IsLetter(input[0]) && char.IsLetter(input[1]);
}
FinlyticNews/Adapters/Discovery/RssDiscoveryAdapter.cs
+73
View File
@@ -0,0 +1,73 @@
using System.ServiceModel.Syndication;
using System.Xml;
using FinlyticCore.Dtos.News;
namespace FinlyticNews.Adapters.Discovery;
/// <summary>
/// Discovery adapter targeting standard RSS/Atom syndication feeds.
/// </summary>
public class RssDiscoveryAdapter : ArticleDiscoveryAdapter
{
/// <inheritdoc />
public override string Name => "rss";
/// <inheritdoc />
public override List<DiscoveredArticle> ExtractUrls(string content, string sourceUrl)
{
if (string.IsNullOrWhiteSpace(content)) return [];
var articles = new List<DiscoveredArticle>();
if (!Uri.TryCreate(sourceUrl, UriKind.Absolute, out var baseUri)) return articles;
try
{
using var stringReader = new StringReader(content);
using var xmlReader = XmlReader.Create(stringReader, new XmlReaderSettings
{
DtdProcessing = DtdProcessing.Ignore,
IgnoreWhitespace = true
});
var feed = SyndicationFeed.Load(xmlReader);
var feedLanguage = feed?.Language;
var feedTitle = feed?.Title?.Text;
if (feed?.Items != null)
{
foreach (var item in feed.Items)
{
var link = item.Links.FirstOrDefault()?.Uri?.ToString() ?? item.Id;
if (!string.IsNullOrWhiteSpace(link) && Uri.TryCreate(baseUri, link, out var absoluteUri))
{
var title = item.Title?.Text?.Trim();
var summary = item.Summary?.Text?.Trim();
DateTime? pubDate = item.PublishDate != DateTimeOffset.MinValue
? item.PublishDate.UtcDateTime
: (item.LastUpdatedTime != DateTimeOffset.MinValue ? item.LastUpdatedTime.UtcDateTime : null);
articles.Add(new DiscoveredArticle(
Url: absoluteUri.ToString(),
Isins: null,
Title: title,
Summary: summary,
PublishedAt: pubDate,
Language: feedLanguage,
SourceName: feedTitle
));
}
}
}
}
catch
{
// Graceful handling fehlerhafter Feeds
}
return articles.GroupBy(u => u.Url, StringComparer.OrdinalIgnoreCase)
.Select(g => g.First())
.ToList();
}
}
FinlyticNews/Adapters/Scraping/ArticleScraperAdapter.cs
+140
View File
@@ -0,0 +1,140 @@
using System;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.Playwright;
namespace FinlyticNews.Adapters.Scraping;
/// <summary>
/// Record holding structured article content extracted via Mozilla Readability.
/// </summary>
public record ScrapedArticleResult(
string Title,
string TextContent,
string HtmlContent,
string? Author,
string? Excerpt,
string FinalUrl
);
/// <summary>
/// Abstract base class representing a website-specific scraping adapter.
/// </summary>
public abstract class ArticleScraperAdapter
{
/// <summary>
/// Gets the target domain hostname of the news website (e.g. "finanznachrichten.de").
/// </summary>
public abstract string Hostname { get; }
/// <summary>
/// Gets the CSS selector to identify and click the 'Read full article' redirect button, if applicable.
/// </summary>
public virtual string? ReadMoreSelector => null;
/// <summary>
/// Fallback CSS selector targeting the article body text elements if Readability fails.
/// </summary>
public virtual string ArticleBodySelector => "body";
/// <summary>
/// Tries to resolve 'Read More' links or external redirects.
/// </summary>
public virtual async Task<string?> TryResolveRedirectUrlAsync(IPage page)
{
var selector = ReadMoreSelector;
if (string.IsNullOrWhiteSpace(selector)) return null;
var readMoreButton = page.Locator(selector).First;
if (await readMoreButton.CountAsync() > 0 && await readMoreButton.IsVisibleAsync())
{
try
{
// Playwright modern pattern for combined popup or navigation handling
var popupTask = page.Context.WaitForPageAsync(new() { Timeout = 6000 });
var navTask = page.WaitForURLAsync(url => url != page.Url, new() { Timeout = 6000 });
await readMoreButton.ClickAsync();
var completedTask = await Task.WhenAny(popupTask, navTask);
if (completedTask == popupTask)
{
var popup = await popupTask;
await popup.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
return popup.Url;
}
await navTask;
return page.Url;
}
catch
{
// Timeout / Click failed -> Fallback to current URL
}
}
return null;
}
/// <summary>
/// Injects Mozilla's Readability.js into the Playwright page to parse article content cleanly.
/// </summary>
public virtual async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
{
try
{
// 1. Inject Mozilla Readability Standalone JS Bundle via CDN
await page.AddScriptTagAsync(new PageAddScriptTagOptions
{
Url = "https://cdn.jsdelivr.net/npm/@mozilla/readability@0.5.0/Readability.min.js"
});
// 2. Execute Readability in the Browser Context
var jsScript = @"
() => {
if (typeof Readability === 'undefined') return null;
const documentClone = document.cloneNode(true);
const article = new Readability(documentClone).parse();
if (!article) return null;
return {
title: article.title || '',
textContent: article.textContent || '',
htmlContent: article.content || '',
author: article.byline || null,
excerpt: article.excerpt || null
};
}";
var jsonResult = await page.EvaluateAsync<JsonElement?>(jsScript);
if (jsonResult.HasValue && jsonResult.Value.ValueKind != JsonValueKind.Null)
{
var root = jsonResult.Value;
return new ScrapedArticleResult(
Title: root.GetProperty("title").GetString() ?? page.TitleAsync().Result,
TextContent: root.GetProperty("textContent").GetString()?.Trim() ?? string.Empty,
HtmlContent: root.GetProperty("htmlContent").GetString() ?? string.Empty,
Author: root.GetProperty("author").GetString(),
Excerpt: root.GetProperty("excerpt").GetString(),
FinalUrl: page.Url
);
}
}
catch
{
// Readability Injection / Parsing failed -> Fallback to standard selector parsing
}
// Fallback: Manuelles Auslesen via ArticleBodySelector
var bodyText = await page.Locator(ArticleBodySelector).InnerTextAsync();
return new ScrapedArticleResult(
Title: await page.TitleAsync(),
TextContent: bodyText.Trim(),
HtmlContent: await page.Locator(ArticleBodySelector).InnerHTMLAsync(),
Author: null,
Excerpt: null,
FinalUrl: page.Url
);
}
}
FinlyticNews/Adapters/Scraping/FinanznachrichtenScraperAdapter.cs
+40
View File
@@ -0,0 +1,40 @@
using System.Threading.Tasks;
using Microsoft.Playwright;
namespace FinlyticNews.Adapters.Scraping;
/// <summary>
/// Specialized article detail scraper adapter for Finanznachrichten.de.
/// </summary>
public class FinanznachrichtenScraperAdapter : ArticleScraperAdapter
{
/// <inheritdoc />
public override string Hostname => "finanznachrichten.de";
/// <inheritdoc />
public override string ArticleBodySelector => "#article-content, div.article-text, div.headlineBody";
/// <inheritdoc />
public override async Task<string?> TryResolveRedirectUrlAsync(IPage page)
{
// 1. Bereits auf der Nachricht-Komplett Seite -> Kein Redirect nötig
if (page.Url.Contains("/ext/nachricht-komplett"))
{
return null;
}
// 2. Extraktion der nachrichtid aus dem Rating Widget (sehr zuverlässig bei FN)
var locator = page.Locator("#article--rating-data").First;
if (await locator.CountAsync() > 0)
{
var nachrichtId = await locator.GetAttributeAsync("data-nachrichtid");
if (!string.IsNullOrEmpty(nachrichtId))
{
return $"https://www.finanznachrichten.de/ext/nachricht-komplett-{nachrichtId}-0.htm";
}
}
// 3. Fallback auf Standard "Weiterlesen"-Button Verhalten der Basisklasse
return await base.TryResolveRedirectUrlAsync(page);
}
}