feat(News): refactor news service and add sentiment tracking
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
using FinlyticCore.Dtos.News;
|
||||
using FinlyticNews.Adapters.Discovery;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines operations for discovering article links from various news feeds and web pages.
|
||||
/// </summary>
|
||||
public interface IArticleDiscoveryService
|
||||
{
|
||||
/// <summary>
|
||||
/// Discovers article links from a given source URL using configured adapters.
|
||||
/// </summary>
|
||||
/// <param name="url">The URL of the source news page or feed.</param>
|
||||
/// <param name="adapterType">The type identifier of the adapter to use (e.g. "rss", "html").</param>
|
||||
/// <param name="ct">The token to monitor for cancellation requests.</param>
|
||||
/// <returns>A list of discovered absolute article URLs (optionally carrying ISINs), or null if the URL was invalid.</returns>
|
||||
Task<List<DiscoveredArticle>?> DiscoverLinksAsync(string url, string adapterType, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public class ArticleDiscoveryService : IArticleDiscoveryService
|
||||
{
|
||||
private readonly IEnumerable<ArticleDiscoveryAdapter> _adapters;
|
||||
private readonly ILogger<ArticleDiscoveryService> _logger;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ArticleDiscoveryService"/> class.
|
||||
/// </summary>
|
||||
/// <param name="adapters">Registered list of specialized adapters.</param>
|
||||
/// <param name="logger">The application logging channel.</param>
|
||||
/// <param name="httpClient">The HTTP client to fetch feeds.</param>
|
||||
public ArticleDiscoveryService(
|
||||
IEnumerable<ArticleDiscoveryAdapter> adapters,
|
||||
ILogger<ArticleDiscoveryService> logger,
|
||||
HttpClient httpClient)
|
||||
{
|
||||
_adapters = adapters;
|
||||
_logger = logger;
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<DiscoveredArticle>?> DiscoverLinksAsync(string url, string adapterType, CancellationToken ct = default)
|
||||
{
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Invalid source URL passed for discovery: {Url}", "NewsChannel", url);
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("Fetching content from discovery source: {Url}", uri);
|
||||
var content = await _httpClient.GetStringAsync(uri, ct);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(content)) return [];
|
||||
|
||||
var trimmedContent = content.TrimStart();
|
||||
|
||||
// 1. Zuerst gezielt nach registriertem Adapter suchen (z. B. finanznachrichten_rss)
|
||||
var adapter = _adapters.FirstOrDefault(a =>
|
||||
a.Name.Equals(adapterType, StringComparison.OrdinalIgnoreCase) ||
|
||||
uri.Host.Contains(a.Name, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (adapter != null)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Using specialized adapter {AdapterName} for source: {Url}", "NewsChannel", adapter.Name, url);
|
||||
return adapter.ExtractUrls(content, url);
|
||||
}
|
||||
|
||||
// 2. Fallback: Automatische Erkennung für generische RSS/Atom-Feeds
|
||||
if (adapterType.Equals("rss", StringComparison.OrdinalIgnoreCase) ||
|
||||
trimmedContent.StartsWith("<?xml", StringComparison.OrdinalIgnoreCase) ||
|
||||
trimmedContent.StartsWith("<rss", StringComparison.OrdinalIgnoreCase) ||
|
||||
trimmedContent.StartsWith("<feed", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Syndication (RSS) format detected for source: {Url}", "NewsChannel", url);
|
||||
|
||||
var rssAdapter = _adapters.FirstOrDefault(a => a.Name.Equals("rss", StringComparison.OrdinalIgnoreCase))
|
||||
?? new RssDiscoveryAdapter();
|
||||
|
||||
return rssAdapter.ExtractUrls(content, url);
|
||||
}
|
||||
|
||||
_logger.LogWarning("[{Channel}] No suitable discovery adapter found for type '{Type}' and URL: {Url}", "NewsChannel", adapterType, url);
|
||||
return [];
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to run article discovery on URL: {Url}", "NewsChannel", url);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using FinlyticCore.Util;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
using FinlyticCore.Dtos.News;
|
||||
|
||||
/// <summary>
|
||||
/// Defines integration operations with the external n8n AI workflow webhook.
|
||||
/// </summary>
|
||||
public interface IN8nService
|
||||
{
|
||||
/// <summary>
|
||||
/// Submits raw article content and pre-filtered assets to n8n, returning the parsed response metadata.
|
||||
/// </summary>
|
||||
Task<N8nResponsePayload?> AnalyzeArticleAsync(string content, List<FilteredAssetPayload> filteredAssets, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public class N8nService : IN8nService
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IConfiguration _configuration;
|
||||
private readonly ILogger<N8nService> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="N8nService"/> class.
|
||||
/// </summary>
|
||||
public N8nService(
|
||||
HttpClient httpClient,
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IConfiguration configuration,
|
||||
ILogger<N8nService> logger)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
_scopeFactory = scopeFactory;
|
||||
_configuration = configuration;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<N8nResponsePayload?> AnalyzeArticleAsync(string content, List<FilteredAssetPayload> filteredAssets, CancellationToken ct = default)
|
||||
{
|
||||
string? targetUrl = null;
|
||||
|
||||
// 1. Dynamic Settings Resolution (DB Scope -> AppSettings Fallback)
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
{
|
||||
var settingsDb = scope.ServiceProvider.GetService<ISettingsDbService>();
|
||||
if (settingsDb != null)
|
||||
{
|
||||
var settings = await settingsDb.GetSettingsAsync();
|
||||
targetUrl = settings?.N8nWebhookUrl;
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(targetUrl))
|
||||
{
|
||||
targetUrl = _configuration["N8N__WebhookUrl"]
|
||||
?? _configuration["N8N:WebhookUrl"];
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(targetUrl))
|
||||
{
|
||||
_logger.LogError("[{Channel}] N8nWebhookUrl is not configured in DB or application settings.", "NewsChannel");
|
||||
return null;
|
||||
}
|
||||
|
||||
_logger.LogInformation("[{Channel}] Posting article to n8n webhook pipeline at: {Url}", "NewsChannel", targetUrl);
|
||||
|
||||
var payload = new N8nRequestPayload(content, filteredAssets ?? []);
|
||||
|
||||
try
|
||||
{
|
||||
// Zero-Allocation / Source-Generated Request Serialization
|
||||
var jsonContent = JsonSerializer.Serialize(payload, FinlyticJsonSerializerContext.Default.N8nRequestPayload);
|
||||
using var requestContent = new StringContent(jsonContent, System.Text.Encoding.UTF8, "application/json");
|
||||
|
||||
using var response = await _httpClient.PostAsync(targetUrl, requestContent, ct);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var errorMsg = await response.Content.ReadAsStringAsync(ct);
|
||||
_logger.LogError("[{Channel}] n8n webhook returned status code {StatusCode}. Error payload: {Error}", "NewsChannel", response.StatusCode, errorMsg);
|
||||
return null;
|
||||
}
|
||||
|
||||
using var responseStream = await response.Content.ReadAsStreamAsync(ct);
|
||||
using var doc = await JsonDocument.ParseAsync(responseStream, cancellationToken: ct);
|
||||
|
||||
var root = doc.RootElement;
|
||||
|
||||
// 2. Robust n8n Array-Unwrapping (Handles [{ "json": { ... } }])
|
||||
if (root.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
if (root.GetArrayLength() == 0)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] n8n webhook returned an empty array.", "NewsChannel");
|
||||
return null;
|
||||
}
|
||||
root = root[0];
|
||||
}
|
||||
|
||||
// 3. Dynamic Node Wrapper Unwrapping ("json", "output", "data", "body")
|
||||
if (root.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
if (root.TryGetProperty("json", out var jsonChild) && jsonChild.ValueKind == JsonValueKind.Object)
|
||||
root = jsonChild;
|
||||
else if (root.TryGetProperty("output", out var outChild) && outChild.ValueKind == JsonValueKind.Object)
|
||||
root = outChild;
|
||||
else if (root.TryGetProperty("data", out var dataChild) && dataChild.ValueKind == JsonValueKind.Object)
|
||||
root = dataChild;
|
||||
else if (root.TryGetProperty("body", out var bodyChild) && bodyChild.ValueKind == JsonValueKind.Object)
|
||||
root = bodyChild;
|
||||
}
|
||||
|
||||
// 4. Source-Generated Deserialization directly from JsonElement
|
||||
var result = root.Deserialize(FinlyticJsonSerializerContext.Default.N8nResponsePayload);
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to communicate with or parse response from n8n webhook workflow.", "NewsChannel");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
using FinlyticCore.Dtos.News;
|
||||
using FinlyticNews.Database;
|
||||
using FinlyticNews.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines database persistence operations for news articles and sources following the FinlyticNews lifecycle pipeline.
|
||||
/// </summary>
|
||||
public interface INewsDbService
|
||||
{
|
||||
Task<List<ArticleSourceEntity>> GetSourcesAsync();
|
||||
Task<bool> IsUrlDuplicateAsync(string url);
|
||||
|
||||
/// <summary>
|
||||
/// Phase 1: Discovers and locks a new article URL by setting its status to "Pending".
|
||||
/// </summary>
|
||||
Task<NewsArticleEntity> CreatePendingArticleAsync(
|
||||
string url,
|
||||
List<string>? discoveredIsins = null,
|
||||
string? title = null,
|
||||
string? summary = null,
|
||||
DateTime? publishedAt = null,
|
||||
string? language = null);
|
||||
|
||||
/// <summary>
|
||||
/// Transitions the lifecycle state of an article (e.g. Pending -> Processing -> Scraping / Failed / Completed / Analyzed).
|
||||
/// </summary>
|
||||
Task UpdateArticleStatusAsync(Guid id, string status);
|
||||
|
||||
/// <summary>
|
||||
/// Updates the target URL of an article if a redirect is resolved during the Processing phase.
|
||||
/// </summary>
|
||||
Task UpdateArticleUrlAsync(Guid id, string resolvedUrl);
|
||||
|
||||
/// <summary>
|
||||
/// Phase 4: Saves AI classification from n8n and sets the lifecycle status to "Completed".
|
||||
/// </summary>
|
||||
Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves articles ready for historical sync or sentiment processing.
|
||||
/// </summary>
|
||||
Task<List<NewsArticleEntity>> GetCompletedArticlesAsync(int limit, int offset, string? isin = null);
|
||||
|
||||
/// <summary>
|
||||
/// Fetches articles matching specific lifecycle statuses (e.g. "Pending", "Scraping" for Phase 1 retry).
|
||||
/// </summary>
|
||||
Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(params string[] statuses);
|
||||
|
||||
/// <summary>
|
||||
/// Fetches public daily news for API endpoints, filtering out intermediate or failed lifecycle states by default.
|
||||
/// </summary>
|
||||
Task<List<NewsArticleEntity>> GetFilteredNewsAsync(
|
||||
int limit = 20,
|
||||
int offset = 0,
|
||||
string? isin = null,
|
||||
DateTime? date = null,
|
||||
string? status = null,
|
||||
string? searchQuery = null);
|
||||
|
||||
Task<NewsArticleEntity?> GetArticleByIdAsync(Guid id);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public class NewsDbService : INewsDbService
|
||||
{
|
||||
private readonly NewsDbContext _context;
|
||||
private readonly ILogger<NewsDbService> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="NewsDbService"/> class.
|
||||
/// </summary>
|
||||
public NewsDbService(NewsDbContext context, ILogger<NewsDbService> logger)
|
||||
{
|
||||
_context = context;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<ArticleSourceEntity>> GetSourcesAsync()
|
||||
{
|
||||
return await _context.ArticleSources.AsNoTracking().ToListAsync();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<NewsArticleEntity?> GetArticleByIdAsync(Guid id)
|
||||
{
|
||||
return await _context.NewsArticles
|
||||
.Include(a => a.MatchedAssets)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(a => a.Id == id);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> IsUrlDuplicateAsync(string url)
|
||||
{
|
||||
var trimmedUrl = url.Trim();
|
||||
return await _context.NewsArticles
|
||||
.AsNoTracking()
|
||||
.AnyAsync(a => a.SourceUrl == trimmedUrl);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Lifecycle Step 1: Creates a new article in 'Pending' state as an immediate lock.
|
||||
/// </summary>
|
||||
public async Task<NewsArticleEntity> CreatePendingArticleAsync(
|
||||
string url,
|
||||
List<string>? discoveredIsins = null,
|
||||
string? title = null,
|
||||
string? summary = null,
|
||||
DateTime? publishedAt = null,
|
||||
string? language = null)
|
||||
{
|
||||
var trimmedUrl = url.Trim();
|
||||
|
||||
var existingArticle = await _context.NewsArticles
|
||||
.FirstOrDefaultAsync(a => a.SourceUrl == trimmedUrl);
|
||||
|
||||
if (existingArticle != null)
|
||||
{
|
||||
_logger.LogDebug("[Lifecycle] Article URL already exists (Duplicate hit): {Url}", trimmedUrl);
|
||||
return existingArticle;
|
||||
}
|
||||
|
||||
var finalPublishedAt = publishedAt.HasValue
|
||||
? (publishedAt.Value.Kind == DateTimeKind.Unspecified
|
||||
? DateTime.SpecifyKind(publishedAt.Value, DateTimeKind.Utc)
|
||||
: publishedAt.Value.ToUniversalTime())
|
||||
: DateTime.UtcNow;
|
||||
|
||||
if (finalPublishedAt > DateTime.UtcNow)
|
||||
{
|
||||
finalPublishedAt = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
var article = new NewsArticleEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Title = !string.IsNullOrWhiteSpace(title) ? title : "Pending Discovery",
|
||||
Summary = !string.IsNullOrWhiteSpace(summary) ? summary : "Extraction in progress...",
|
||||
ContentRaw = "Extraction in progress...",
|
||||
SourceUrl = trimmedUrl,
|
||||
Language = language,
|
||||
ScrapedAt = DateTime.UtcNow,
|
||||
PublishedAt = finalPublishedAt,
|
||||
Status = "Pending" // 1. Pending State
|
||||
};
|
||||
|
||||
if (discoveredIsins != null && discoveredIsins.Count > 0)
|
||||
{
|
||||
foreach (var isin in discoveredIsins)
|
||||
{
|
||||
article.MatchedAssets.Add(new MatchedAssetEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
NewsArticleId = article.Id,
|
||||
Isin = isin,
|
||||
Name = isin
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_context.NewsArticles.Add(article);
|
||||
await _context.SaveChangesAsync();
|
||||
_logger.LogDebug("[Lifecycle] Registered new article with status 'Pending'. ID: {Id}, Url: {Url}", article.Id, trimmedUrl);
|
||||
}
|
||||
catch (DbUpdateConcurrencyException)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Concurrency hit during insert for URL: {Url}. Fetching existing fallback.", "NewsChannel", trimmedUrl);
|
||||
return await _context.NewsArticles.FirstAsync(a => a.SourceUrl == trimmedUrl);
|
||||
}
|
||||
|
||||
return article;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Lifecycle Step 2 & 5: Updates state (e.g. Pending -> Processing -> Scraping / Failed / Analyzed).
|
||||
/// </summary>
|
||||
public async Task UpdateArticleStatusAsync(Guid id, string status)
|
||||
{
|
||||
var rowsAffected = await _context.NewsArticles
|
||||
.Where(a => a.Id == id)
|
||||
.ExecuteUpdateAsync(s => s.SetProperty(a => a.Status, status));
|
||||
|
||||
if (rowsAffected == 0)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Attempted status transition for non-existing article. ID: {Id}", "NewsChannel", id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("[Lifecycle] Transitioned article {Id} to status '{Status}'", id, status);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateArticleUrlAsync(Guid id, string resolvedUrl)
|
||||
{
|
||||
var rowsAffected = await _context.NewsArticles
|
||||
.Where(a => a.Id == id)
|
||||
.ExecuteUpdateAsync(s => s.SetProperty(a => a.SourceUrl, resolvedUrl));
|
||||
|
||||
if (rowsAffected == 0)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Attempted URL update for non-existing article. ID: {Id}", "NewsChannel", id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("[Lifecycle] Resolved redirect for article {Id} -> New URL: {Url}", id, resolvedUrl);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Lifecycle Step 4: Persists n8n classification and transitions status to 'Completed'.
|
||||
/// </summary>
|
||||
public async Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets)
|
||||
{
|
||||
var article = await _context.NewsArticles
|
||||
.FirstOrDefaultAsync(a => a.Id == id);
|
||||
|
||||
if (article == null)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Article with ID {Id} not found for classification update.", "NewsChannel", id);
|
||||
return null;
|
||||
}
|
||||
|
||||
article.Title = payload.Title;
|
||||
article.Author = payload.Author;
|
||||
article.Summary = payload.Summary;
|
||||
article.ContentRaw = payload.ContentRaw;
|
||||
article.Language = payload.Language;
|
||||
|
||||
// 🎯 Step 4: Classification finished -> Transition to 'Completed' (triggers MQTT broadcast)
|
||||
article.Status = "Completed";
|
||||
|
||||
if (DateTime.TryParse(payload.PublishedAt, out var publishedDate))
|
||||
{
|
||||
article.PublishedAt = publishedDate.Kind == DateTimeKind.Unspecified
|
||||
? DateTime.SpecifyKind(publishedDate, DateTimeKind.Utc)
|
||||
: publishedDate.ToUniversalTime();
|
||||
}
|
||||
|
||||
if (DateTime.TryParse(payload.ScrapedAt, out var scrapedDate))
|
||||
{
|
||||
article.ScrapedAt = scrapedDate.ToUniversalTime();
|
||||
}
|
||||
|
||||
// Clean up previous temporary assets
|
||||
await _context.MatchedAssets.Where(m => m.NewsArticleId == id).ExecuteDeleteAsync();
|
||||
|
||||
article.MatchedAssets = new List<MatchedAssetEntity>();
|
||||
foreach (var asset in matchedAssets)
|
||||
{
|
||||
if (asset.Id == Guid.Empty)
|
||||
{
|
||||
asset.Id = Guid.NewGuid();
|
||||
}
|
||||
|
||||
_context.MatchedAssets.Add(asset);
|
||||
article.MatchedAssets.Add(asset);
|
||||
}
|
||||
|
||||
await _context.SaveChangesAsync();
|
||||
_logger.LogInformation("[Lifecycle] Article {Id} successfully classified and marked 'Completed'. Title: '{Title}'", article.Id, article.Title);
|
||||
|
||||
return article;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<NewsArticleEntity>> GetCompletedArticlesAsync(int limit, int offset, string? isin = null)
|
||||
{
|
||||
var query = _context.NewsArticles
|
||||
.Include(a => a.MatchedAssets)
|
||||
.Where(a => a.Status == "Completed" || a.Status == "Analyzed");
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(isin))
|
||||
{
|
||||
var cleanIsin = isin.Trim();
|
||||
query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin || m.Name == cleanIsin));
|
||||
}
|
||||
|
||||
return await query
|
||||
.OrderByDescending(a => a.PublishedAt)
|
||||
.ThenByDescending(a => a.ScrapedAt)
|
||||
.ThenByDescending(a => a.Id)
|
||||
.Skip(offset)
|
||||
.Take(limit)
|
||||
.AsNoTracking()
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Lifecycle Helper: Retrieves articles by target lifecycle status (e.g. "Pending", "Scraping" for Phase 1 Retry).
|
||||
/// </summary>
|
||||
public async Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(params string[] statuses)
|
||||
{
|
||||
IQueryable<NewsArticleEntity> query = _context.NewsArticles
|
||||
.Include(a => a.MatchedAssets)
|
||||
.AsNoTracking();
|
||||
|
||||
if (statuses != null && statuses.Length > 0)
|
||||
{
|
||||
var cleanStatuses = statuses.Select(s => s.Trim()).ToList();
|
||||
query = query.Where(a => cleanStatuses.Contains(a.Status));
|
||||
}
|
||||
else
|
||||
{
|
||||
query = query.Where(a => a.Status != "Failed" && a.Status != "Duplicate");
|
||||
}
|
||||
|
||||
return await query.ToListAsync();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// API Gateway Helper: Retrieves articles for UI rendering, excluding intermediate/failed states by default.
|
||||
/// </summary>
|
||||
public async Task<List<NewsArticleEntity>> GetFilteredNewsAsync(
|
||||
int limit = 20,
|
||||
int offset = 0,
|
||||
string? isin = null,
|
||||
DateTime? date = null,
|
||||
string? status = null,
|
||||
string? searchQuery = null)
|
||||
{
|
||||
IQueryable<NewsArticleEntity> query = _context.NewsArticles
|
||||
.Include(a => a.MatchedAssets)
|
||||
.AsNoTracking();
|
||||
|
||||
// 1. Status Filter
|
||||
if (!string.IsNullOrWhiteSpace(status))
|
||||
{
|
||||
var targetStatus = status.Trim();
|
||||
query = query.Where(a => a.Status == targetStatus);
|
||||
}
|
||||
else
|
||||
{
|
||||
// By default, only show fully processed articles to the API/UI
|
||||
query = query.Where(a => a.Status == "Completed" || a.Status == "Analyzed");
|
||||
}
|
||||
|
||||
// 2. Date Filter
|
||||
if (date.HasValue)
|
||||
{
|
||||
var targetDate = DateTime.SpecifyKind(date.Value.Date, DateTimeKind.Utc);
|
||||
var nextDate = targetDate.AddDays(1);
|
||||
query = query.Where(a => a.PublishedAt >= targetDate && a.PublishedAt < nextDate);
|
||||
}
|
||||
|
||||
// 3. ISIN / Symbol Filter
|
||||
if (!string.IsNullOrWhiteSpace(isin))
|
||||
{
|
||||
var cleanIsin = isin.Trim();
|
||||
query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin || m.Name == cleanIsin));
|
||||
}
|
||||
|
||||
// 4. Search Term
|
||||
if (!string.IsNullOrWhiteSpace(searchQuery))
|
||||
{
|
||||
var q = searchQuery.Trim();
|
||||
query = query.Where(a => EF.Functions.ILike(a.Title, $"%{q}%") || (a.Summary != null && EF.Functions.ILike(a.Summary, $"%{q}%")));
|
||||
}
|
||||
|
||||
return await query
|
||||
.OrderByDescending(a => a.PublishedAt)
|
||||
.ThenByDescending(a => a.ScrapedAt)
|
||||
.ThenByDescending(a => a.Id)
|
||||
.Skip(offset)
|
||||
.Take(limit)
|
||||
.ToListAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.RegularExpressions;
|
||||
using FinlyticAssets.Models;
|
||||
using FinlyticAssets.Util;
|
||||
using FinlyticCore.Dtos.News;
|
||||
using FinlyticCore.Util;
|
||||
using FinlyticNews.Entities;
|
||||
using FinlyticNews.Util;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// A background worker service that orchestrates link discovery, Playwright scraping,
|
||||
/// pre-filtering, n8n AI enrichment, database persistence, and MQTT broadcasts.
|
||||
/// </summary>
|
||||
public class NewsScraperBackgroundService : BackgroundService
|
||||
{
|
||||
/// <summary>
|
||||
/// Internal wrapper to associate compiled regex patterns with the unmodified AssetIndex record.
|
||||
/// </summary>
|
||||
private record CompiledAssetMatcher(
|
||||
AssetIndex Asset,
|
||||
string CoreName,
|
||||
Regex? WordRegex,
|
||||
Regex? CoreWordRegex
|
||||
);
|
||||
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<NewsScraperBackgroundService> _logger;
|
||||
private readonly NewsMqttClient _mqttClient;
|
||||
private readonly int _intervalMinutes;
|
||||
private readonly string _indexPath;
|
||||
|
||||
// In-Memory Cache for compiled asset matchers to prevent re-reading & re-compiling Regex
|
||||
private List<CompiledAssetMatcher>? _cachedAssetMatchers;
|
||||
private DateTime _lastIndexLoadTime = DateTime.MinValue;
|
||||
|
||||
public NewsScraperBackgroundService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<NewsScraperBackgroundService> logger,
|
||||
NewsMqttClient mqttClient,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
_mqttClient = mqttClient;
|
||||
|
||||
_intervalMinutes = configuration.GetValue<int>("ScrapingSettings:IntervalMinutes", 15);
|
||||
_indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json");
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] NewsScraperBackgroundService started. Interval: {Minutes} minutes.", "NewsChannel", _intervalMinutes);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RunScrapingCycleAsync(stoppingToken);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] An unhandled exception occurred during news scraping cycle.", "NewsChannel");
|
||||
}
|
||||
|
||||
int intervalMinutes = _intervalMinutes;
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var settingsDb = scope.ServiceProvider.GetService<ISettingsDbService>();
|
||||
if (settingsDb != null)
|
||||
{
|
||||
var settings = await settingsDb.GetSettingsAsync();
|
||||
if (settings?.ScrapingIntervalMinutes > 0)
|
||||
{
|
||||
intervalMinutes = settings.ScrapingIntervalMinutes;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { /* Ignore settings DB lookup failures */ }
|
||||
|
||||
var jitterSeconds = Random.Shared.Next(0, 300);
|
||||
var nextRunDelay = TimeSpan.FromMinutes(intervalMinutes) + TimeSpan.FromSeconds(jitterSeconds);
|
||||
_logger.LogInformation("[{Channel}] Scraping cycle completed. Next cycle in {Delay} (interval: {Minutes}m).", "NewsChannel", nextRunDelay, intervalMinutes);
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(nextRunDelay, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("[{Channel}] NewsScraperBackgroundService stopping.", "NewsChannel");
|
||||
}
|
||||
|
||||
private async Task RunScrapingCycleAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var dbService = scope.ServiceProvider.GetRequiredService<INewsDbService>();
|
||||
var discoveryService = scope.ServiceProvider.GetRequiredService<IArticleDiscoveryService>();
|
||||
var scraperService = scope.ServiceProvider.GetRequiredService<IPlaywrightScraperService>();
|
||||
var n8nService = scope.ServiceProvider.GetRequiredService<IN8nService>();
|
||||
|
||||
// Load pre-compiled asset index matchers for zero-latency pre-filtering
|
||||
var assetMatchers = await GetOrLoadAssetMatchersAsync();
|
||||
_logger.LogInformation("[{Channel}] Loaded {Count} asset index items for text pre-filtering.", "NewsChannel", assetMatchers.Count);
|
||||
|
||||
// 1. Scraping Retry Phase: query articles in status "Scraping" (failed Playwright runs)
|
||||
var failedArticles = await dbService.GetArticlesByStatusAsync("Scraping");
|
||||
if (failedArticles.Count > 0)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Found {Count} articles in status 'Scraping' that failed to scrape previously. Retrying...", "NewsChannel", failedArticles.Count);
|
||||
foreach (var article in failedArticles)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
await ProcessSingleArticleAsync(article, dbService, scraperService, n8nService, assetMatchers, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Link Discovery Phase: query RSS feeds and listing pages
|
||||
var sources = await dbService.GetSourcesAsync();
|
||||
if (sources.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] No article sources configured in database. Skipping cycle.", "NewsChannel");
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var source in sources)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
|
||||
_logger.LogInformation("[{Channel}] Starting article link discovery for source: {SourceName} ({Url})", "NewsChannel", source.Name, source.Source);
|
||||
var discoveredArticles = await discoveryService.DiscoverLinksAsync(source.Source, source.Type, stoppingToken);
|
||||
|
||||
if (discoveredArticles == null || discoveredArticles.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No links discovered from source: {SourceName}", source.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogInformation("[{Channel}] Discovered {Count} potential article links from {SourceName}.", "NewsChannel", discoveredArticles.Count, source.Name);
|
||||
|
||||
foreach (var discovered in discoveredArticles)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested) break;
|
||||
|
||||
// Idempotency Check & Deduplication
|
||||
var isDuplicate = await dbService.IsUrlDuplicateAsync(discovered.Url);
|
||||
if (isDuplicate)
|
||||
{
|
||||
_logger.LogDebug("Skipping duplicate article URL: {Url}", discovered.Url);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Register Initial Lock State in the database ("Pending")
|
||||
NewsArticleEntity? article;
|
||||
try
|
||||
{
|
||||
article = await dbService.CreatePendingArticleAsync(
|
||||
discovered.Url,
|
||||
discovered.Isins,
|
||||
discovered.Title,
|
||||
discovered.Summary,
|
||||
discovered.PublishedAt,
|
||||
discovered.Language);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to register initial pending state for URL: {Url}. Skipping.", "NewsChannel", discovered.Url);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (article == null || article.Id == Guid.Empty)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Created pending article has invalid/empty ID for URL: {Url}. Skipping.", "NewsChannel", discovered.Url);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process single article pipeline
|
||||
await ProcessSingleArticleAsync(article, dbService, scraperService, n8nService, assetMatchers, stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessSingleArticleAsync(
|
||||
NewsArticleEntity article,
|
||||
INewsDbService dbService,
|
||||
IPlaywrightScraperService scraperService,
|
||||
IN8nService n8nService,
|
||||
List<CompiledAssetMatcher> assetMatchers,
|
||||
CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
// 3. Extraction with Headless Browser (Transitions to "Processing")
|
||||
await dbService.UpdateArticleStatusAsync(article.Id, "Processing");
|
||||
var (resolvedUrl, rawText) = await scraperService.ScrapeArticleAsync(article.SourceUrl);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(rawText))
|
||||
{
|
||||
throw new InvalidOperationException("Scraping returned empty text body content.");
|
||||
}
|
||||
|
||||
// Update resolved URL if redirect occurred
|
||||
if (!string.Equals(resolvedUrl, article.SourceUrl, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Redirect detected. Initial: {OldUrl} -> Resolved: {NewUrl}", "NewsChannel", article.SourceUrl, resolvedUrl);
|
||||
if (await dbService.IsUrlDuplicateAsync(resolvedUrl))
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Redirected URL {ResolvedUrl} is a duplicate. Terminating processing.", "NewsChannel", resolvedUrl);
|
||||
await dbService.UpdateArticleStatusAsync(article.Id, "Duplicate");
|
||||
return;
|
||||
}
|
||||
|
||||
await dbService.UpdateArticleUrlAsync(article.Id, resolvedUrl);
|
||||
article.SourceUrl = resolvedUrl;
|
||||
}
|
||||
|
||||
// 4. Pre-filtering Assets (Optimized with Pre-Compiled Regex Patterns)
|
||||
var title = article.Title ?? string.Empty;
|
||||
var summary = article.Summary ?? string.Empty;
|
||||
|
||||
var preFilteredAssets = assetMatchers.Where(matcher =>
|
||||
{
|
||||
var asset = matcher.Asset;
|
||||
|
||||
// ISIN direct match
|
||||
if (rawText.Contains(asset.Isin, StringComparison.OrdinalIgnoreCase) ||
|
||||
title.Contains(asset.Isin, StringComparison.OrdinalIgnoreCase) ||
|
||||
summary.Contains(asset.Isin, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Fast regex word boundary check on Full Name
|
||||
if (matcher.WordRegex != null && (matcher.WordRegex.IsMatch(rawText) || matcher.WordRegex.IsMatch(title)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Fast regex word boundary check on Core Name
|
||||
if (matcher.CoreName.Length >= 3 && matcher.CoreWordRegex != null &&
|
||||
(matcher.CoreWordRegex.IsMatch(rawText) || matcher.CoreWordRegex.IsMatch(title)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
})
|
||||
.Select(m => new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin))
|
||||
.ToList();
|
||||
|
||||
if (preFilteredAssets.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Pre-filtering: Article {Id} does not reference any known assets. Terminating pipeline.", "NewsChannel", article.Id);
|
||||
await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("[{Channel}] Pre-filtering matched {Count} assets for article {Id}.", "NewsChannel", preFilteredAssets.Count, article.Id);
|
||||
|
||||
// 5. Send to n8n Webhook Pipeline
|
||||
var n8nResponse = await n8nService.AnalyzeArticleAsync(rawText, preFilteredAssets, stoppingToken);
|
||||
if (n8nResponse == null)
|
||||
{
|
||||
throw new InvalidOperationException("n8n AI webhook execution returned null or failed.");
|
||||
}
|
||||
|
||||
// 6. Map and Save Completed Classification
|
||||
var matchedEntities = new List<MatchedAssetEntity>();
|
||||
foreach (var n8nAsset in n8nResponse.MatchedAssets)
|
||||
{
|
||||
var n8nCoreName = ExtractCoreAssetName(n8nAsset.Name);
|
||||
|
||||
var matchedIsin = preFilteredAssets.FirstOrDefault(fa =>
|
||||
fa.Name.Equals(n8nAsset.Name, StringComparison.OrdinalIgnoreCase) ||
|
||||
n8nAsset.Name.Contains(fa.Name, StringComparison.OrdinalIgnoreCase) ||
|
||||
(n8nCoreName.Length >= 3 && ExtractCoreAssetName(fa.Name).Equals(n8nCoreName, StringComparison.OrdinalIgnoreCase)))?.Isin;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(matchedIsin))
|
||||
{
|
||||
matchedIsin = assetMatchers.FirstOrDefault(m =>
|
||||
m.Asset.Name.Equals(n8nAsset.Name, StringComparison.OrdinalIgnoreCase) ||
|
||||
n8nAsset.Name.Contains(m.Asset.Name, StringComparison.OrdinalIgnoreCase) ||
|
||||
(n8nCoreName.Length >= 3 && m.CoreName.Equals(n8nCoreName, StringComparison.OrdinalIgnoreCase)))?.Asset.Isin;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(matchedIsin))
|
||||
{
|
||||
matchedEntities.Add(new MatchedAssetEntity
|
||||
{
|
||||
NewsArticleId = article.Id,
|
||||
Name = n8nAsset.Name,
|
||||
Isin = matchedIsin
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var completedArticle = await dbService.SaveArticleClassificationAsync(article.Id, n8nResponse, matchedEntities);
|
||||
|
||||
if (completedArticle != null)
|
||||
{
|
||||
// 7. MQTT Broadcast (Sends completed article to downstream services)
|
||||
var dto = new NewsArticleDto
|
||||
{
|
||||
Id = completedArticle.Id,
|
||||
Title = completedArticle.Title,
|
||||
Author = completedArticle.Author,
|
||||
Summary = completedArticle.Summary,
|
||||
ContentRaw = completedArticle.ContentRaw,
|
||||
Language = completedArticle.Language,
|
||||
SourceUrl = completedArticle.SourceUrl,
|
||||
ScrapedAt = completedArticle.ScrapedAt,
|
||||
PublishedAt = completedArticle.PublishedAt,
|
||||
MatchedAssets = completedArticle.MatchedAssets.Select(m => new MatchedAssetDto
|
||||
{
|
||||
Name = m.Name,
|
||||
Isin = m.Isin
|
||||
}).ToList(),
|
||||
Status = completedArticle.Status
|
||||
};
|
||||
|
||||
await _mqttClient.BroadcastArticleAsync(dto);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to complete processing pipeline for article: {Url}. Transitioning to 'Scraping' for next cycle retry.", "NewsChannel", article.SourceUrl);
|
||||
try
|
||||
{
|
||||
await dbService.UpdateArticleStatusAsync(article.Id, "Scraping");
|
||||
}
|
||||
catch { /* Suppress database secondary errors */ }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns cached compiled asset matchers or parses the index file from disk if stale/missing.
|
||||
/// </summary>
|
||||
private async Task<List<CompiledAssetMatcher>> GetOrLoadAssetMatchersAsync()
|
||||
{
|
||||
if (_cachedAssetMatchers != null && (DateTime.UtcNow - _lastIndexLoadTime).TotalMinutes < 30)
|
||||
{
|
||||
return _cachedAssetMatchers;
|
||||
}
|
||||
|
||||
if (!File.Exists(_indexPath))
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Asset index file not found at: {Path}. Pre-filtering will match 0 assets.", "NewsChannel", _indexPath);
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await using var stream = File.OpenRead(_indexPath);
|
||||
|
||||
// Standard Deserialization for AssetIndex list
|
||||
var rawList = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream);
|
||||
|
||||
if (rawList != null)
|
||||
{
|
||||
_cachedAssetMatchers = rawList.Select(asset =>
|
||||
{
|
||||
var coreName = ExtractCoreAssetName(asset.Name);
|
||||
return new CompiledAssetMatcher(
|
||||
Asset: asset,
|
||||
CoreName: coreName,
|
||||
WordRegex: BuildWordRegex(asset.Name),
|
||||
CoreWordRegex: BuildWordRegex(coreName)
|
||||
);
|
||||
}).ToList();
|
||||
|
||||
_lastIndexLoadTime = DateTime.UtcNow;
|
||||
return _cachedAssetMatchers;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to read or parse asset index file from {Path}.", "NewsChannel", _indexPath);
|
||||
}
|
||||
|
||||
return _cachedAssetMatchers ?? [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper to pre-compile Word Boundary Regex for an asset name.
|
||||
/// </summary>
|
||||
private static Regex? BuildWordRegex(string name)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name)) return null;
|
||||
try
|
||||
{
|
||||
return new Regex($@"\b{Regex.Escape(name)}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the core name of an asset by removing parenthetical metadata and corporate suffixes.
|
||||
/// </summary>
|
||||
private static string ExtractCoreAssetName(string name)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name)) return string.Empty;
|
||||
|
||||
int parenIndex = name.IndexOf('(');
|
||||
if (parenIndex >= 0)
|
||||
{
|
||||
name = name[..parenIndex];
|
||||
}
|
||||
|
||||
name = name.Trim();
|
||||
|
||||
var suffixes = new[] { "Inc.", "Inc", "AG", "SE", "Co.", "Co", "Corp.", "Corp", "Ltd.", "Ltd", "plc", "GmbH", "SA", "NV", "Group" };
|
||||
foreach (var suffix in suffixes)
|
||||
{
|
||||
if (name.EndsWith(" " + suffix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
name = name[..^suffix.Length].Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticNews.Adapters.Scraping;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
|
||||
/// </summary>
|
||||
public interface IPlaywrightScraperService
|
||||
{
|
||||
/// <summary>
|
||||
/// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
|
||||
/// </summary>
|
||||
/// <param name="url">The initial article URL.</param>
|
||||
/// <returns>A tuple containing the final resolved URL and the extracted raw text content.</returns>
|
||||
Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposable
|
||||
{
|
||||
private readonly ILogger<PlaywrightScraperService> _logger;
|
||||
private readonly IEnumerable<ArticleScraperAdapter> _scraperAdapters;
|
||||
|
||||
private IPlaywright? _playwright;
|
||||
private IBrowser? _browser;
|
||||
private readonly SemaphoreSlim _browserLock = new(1, 1);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PlaywrightScraperService"/> class.
|
||||
/// </summary>
|
||||
public PlaywrightScraperService(
|
||||
ILogger<PlaywrightScraperService> logger,
|
||||
IEnumerable<ArticleScraperAdapter> scraperAdapters)
|
||||
{
|
||||
_logger = logger;
|
||||
_scraperAdapters = scraperAdapters;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Launching browser context to scrape article: {Url}", "NewsChannel", url);
|
||||
|
||||
var browser = await GetOrInitBrowserAsync();
|
||||
|
||||
// Fast, isolated browser context (incognito tab environment) per article
|
||||
await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
|
||||
{
|
||||
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
ViewportSize = new ViewportSize { Width = 1280, Height = 800 }
|
||||
});
|
||||
|
||||
var page = await context.NewPageAsync();
|
||||
|
||||
try
|
||||
{
|
||||
// 1. Initial Navigation with DOMContentLoaded wait
|
||||
var response = await page.GotoAsync(url, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 30000
|
||||
});
|
||||
|
||||
if (response == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to load HTTP response for URL: {url}");
|
||||
}
|
||||
|
||||
var finalUrl = page.Url;
|
||||
_logger.LogDebug("[{Channel}] Navigation completed. Initial final URL: {Url}", "NewsChannel", finalUrl);
|
||||
|
||||
// 2. Resolve Host Specific Scraper Adapter
|
||||
var uri = new Uri(url);
|
||||
var host = uri.Host;
|
||||
var adapter = _scraperAdapters.FirstOrDefault(a => host.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
IPage targetPage = page;
|
||||
|
||||
if (adapter != null)
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Executing adapter redirect check for host: {Host}", "NewsChannel", adapter.Hostname);
|
||||
try
|
||||
{
|
||||
var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
|
||||
|
||||
// Loop Protection: Navigate only if redirect target is a new URL
|
||||
if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) &&
|
||||
!string.Equals(page.Url, resolvedRedirectUrl, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("[{Channel}] Redirect resolved to target URL: {Url}", "NewsChannel", resolvedRedirectUrl);
|
||||
|
||||
var refererUrl = page.Url;
|
||||
finalUrl = resolvedRedirectUrl;
|
||||
|
||||
var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
|
||||
{
|
||||
WaitUntil = WaitUntilState.DOMContentLoaded,
|
||||
Timeout = 30000,
|
||||
Referer = refererUrl
|
||||
});
|
||||
|
||||
if (redirectResponse == null)
|
||||
{
|
||||
_logger.LogWarning("[{Channel}] Failed to load response for redirect URL: {Url}", "NewsChannel", resolvedRedirectUrl);
|
||||
}
|
||||
else
|
||||
{
|
||||
finalUrl = page.Url;
|
||||
}
|
||||
|
||||
// Check if redirect opened a new tab/popup
|
||||
var matchedPage = page.Context.Pages.FirstOrDefault(p => p.Url == resolvedRedirectUrl);
|
||||
if (matchedPage != null)
|
||||
{
|
||||
targetPage = matchedPage;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "[{Channel}] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", "NewsChannel", adapter.Hostname);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Re-resolve detail page adapter for final target page
|
||||
var targetUri = new Uri(targetPage.Url);
|
||||
var targetHost = targetUri.Host;
|
||||
var targetAdapter = _scraperAdapters.FirstOrDefault(a => targetHost.Contains(a.Hostname, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
// Wait a brief moment for dynamic scripts / DOM settling
|
||||
await targetPage.WaitForTimeoutAsync(1000);
|
||||
|
||||
// 4. Extract Content via Mozilla Readability (or Adapter Fallback)
|
||||
string extractedText = string.Empty;
|
||||
|
||||
if (targetAdapter != null)
|
||||
{
|
||||
var readabilityResult = await targetAdapter.ExtractArticleContentAsync(targetPage);
|
||||
if (readabilityResult != null && !string.IsNullOrWhiteSpace(readabilityResult.TextContent))
|
||||
{
|
||||
extractedText = readabilityResult.TextContent;
|
||||
}
|
||||
}
|
||||
|
||||
// Standard Fallback: Body Text / Selector Extraction
|
||||
if (string.IsNullOrWhiteSpace(extractedText))
|
||||
{
|
||||
var bodySelector = targetAdapter?.ArticleBodySelector ?? "body";
|
||||
var locator = targetPage.Locator(bodySelector);
|
||||
|
||||
if (await locator.CountAsync() > 0)
|
||||
{
|
||||
extractedText = await locator.First.InnerTextAsync();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(extractedText))
|
||||
{
|
||||
extractedText = await targetPage.EvaluateAsync<string>(
|
||||
$"() => document.querySelector('{bodySelector}')?.innerText ?? ''");
|
||||
}
|
||||
}
|
||||
|
||||
return (finalUrl, extractedText?.Trim() ?? string.Empty);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[{Channel}] Failed to scrape page content from URL: {Url}", "NewsChannel", url);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await context.CloseAsync();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe singleton initialization of the Chromium browser instance.
|
||||
/// </summary>
|
||||
private async Task<IBrowser> GetOrInitBrowserAsync()
|
||||
{
|
||||
if (_browser != null && _browser.IsConnected)
|
||||
{
|
||||
return _browser;
|
||||
}
|
||||
|
||||
await _browserLock.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (_browser != null && _browser.IsConnected)
|
||||
{
|
||||
return _browser;
|
||||
}
|
||||
|
||||
_playwright ??= await Playwright.CreateAsync();
|
||||
_browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
|
||||
{
|
||||
Headless = true,
|
||||
Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
|
||||
});
|
||||
|
||||
_logger.LogInformation("[{Channel}] Initialized shared Chromium browser instance.", "NewsChannel");
|
||||
return _browser;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_browserLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the Playwright and Browser instances cleanly during service shutdown.
|
||||
/// </summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_browser != null)
|
||||
{
|
||||
await _browser.CloseAsync();
|
||||
await _browser.DisposeAsync();
|
||||
}
|
||||
|
||||
_playwright?.Dispose();
|
||||
_browserLock.Dispose();
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
using FinlyticNews.Database;
|
||||
using FinlyticNews.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Service interface for retrieving and persisting runtime settings in PostgreSQL for FinlyticNews.
|
||||
/// </summary>
|
||||
public interface ISettingsDbService
|
||||
{
|
||||
/// <summary>
|
||||
/// Retrieves current news settings from database, initializing default values if empty.
|
||||
/// </summary>
|
||||
Task<NewsSettingsEntity> GetSettingsAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Persists updated settings to PostgreSQL.
|
||||
/// </summary>
|
||||
Task<NewsSettingsEntity> SaveSettingsAsync(NewsSettingsEntity settings);
|
||||
|
||||
/// <summary>
|
||||
/// Updates settings from a key-value dictionary received via Admin Panel MQTT events.
|
||||
/// </summary>
|
||||
Task UpdateSettingsFromDictionaryAsync(Dictionary<string, string> dictionary);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EF Core implementation of <see cref="ISettingsDbService"/>.
|
||||
/// </summary>
|
||||
public class SettingsDbService : ISettingsDbService
|
||||
{
|
||||
private readonly NewsDbContext _context;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SettingsDbService"/> class.
|
||||
/// </summary>
|
||||
/// <param name="context">The database context mapping database tables.</param>
|
||||
public SettingsDbService(NewsDbContext context)
|
||||
{
|
||||
_context = context;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<NewsSettingsEntity> GetSettingsAsync()
|
||||
{
|
||||
var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync();
|
||||
if (settings == null)
|
||||
{
|
||||
settings = new NewsSettingsEntity { Id = Guid.NewGuid() };
|
||||
_context.Settings.Add(settings);
|
||||
await _context.SaveChangesAsync();
|
||||
_context.ChangeTracker.Clear();
|
||||
}
|
||||
return settings;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<NewsSettingsEntity> SaveSettingsAsync(NewsSettingsEntity settings)
|
||||
{
|
||||
var existing = await _context.Settings.FirstOrDefaultAsync();
|
||||
if (existing == null)
|
||||
{
|
||||
if (settings.Id == Guid.Empty) settings.Id = Guid.NewGuid();
|
||||
_context.Settings.Add(settings);
|
||||
}
|
||||
else
|
||||
{
|
||||
existing.PollingFrequencyMinutes = settings.PollingFrequencyMinutes;
|
||||
existing.ArticleRetentionDays = settings.ArticleRetentionDays;
|
||||
existing.DefaultPageSize = settings.DefaultPageSize;
|
||||
existing.ScrapingIntervalMinutes = settings.ScrapingIntervalMinutes;
|
||||
existing.N8nWebhookUrl = settings.N8nWebhookUrl;
|
||||
existing.UpdatedAt = settings.UpdatedAt;
|
||||
_context.Settings.Update(existing);
|
||||
}
|
||||
await _context.SaveChangesAsync();
|
||||
return settings;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateSettingsFromDictionaryAsync(Dictionary<string, string> dictionary)
|
||||
{
|
||||
var settings = await GetSettingsAsync();
|
||||
|
||||
foreach (var (key, value) in dictionary)
|
||||
{
|
||||
if (string.Equals(key, "PollingFrequencyMinutes", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var pfm))
|
||||
settings.PollingFrequencyMinutes = pfm;
|
||||
else if (string.Equals(key, "ArticleRetentionDays", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ard))
|
||||
settings.ArticleRetentionDays = ard;
|
||||
else if (string.Equals(key, "DefaultPageSize", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var dps))
|
||||
settings.DefaultPageSize = dps;
|
||||
else if (string.Equals(key, "ScrapingIntervalMinutes", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var sim))
|
||||
settings.ScrapingIntervalMinutes = sim;
|
||||
else if (string.Equals(key, "N8nWebhookUrl", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(value))
|
||||
settings.N8nWebhookUrl = value.Trim();
|
||||
}
|
||||
|
||||
settings.UpdatedAt = DateTime.UtcNow;
|
||||
await SaveSettingsAsync(settings);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user