feat(News): refactor news service and add sentiment tracking

2026-08-09 21:01:41 +02:00
parent 605e41cfeb
commit aff831cf58
390 changed files with 115094 additions and 9 deletions
@@ -0,0 +1,379 @@
+using FinlyticCore.Dtos.News;
+using FinlyticNews.Database;
+using FinlyticNews.Entities;
+using Microsoft.EntityFrameworkCore;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Defines database persistence operations for news articles and sources following the FinlyticNews lifecycle pipeline.
+/// </summary>
+public interface INewsDbService
+{
+    Task<List<ArticleSourceEntity>> GetSourcesAsync();
+    Task<bool> IsUrlDuplicateAsync(string url);
+
+    /// <summary>
+    /// Phase 1: Discovers and locks a new article URL by setting its status to "Pending".
+    /// </summary>
+    Task<NewsArticleEntity> CreatePendingArticleAsync(
+        string url,
+        List<string>? discoveredIsins = null,
+        string? title = null,
+        string? summary = null,
+        DateTime? publishedAt = null,
+        string? language = null);
+
+    /// <summary>
+    /// Transitions the lifecycle state of an article (e.g. Pending -> Processing -> Scraping / Failed / Completed / Analyzed).
+    /// </summary>
+    Task UpdateArticleStatusAsync(Guid id, string status);
+
+    /// <summary>
+    /// Updates the target URL of an article if a redirect is resolved during the Processing phase.
+    /// </summary>
+    Task UpdateArticleUrlAsync(Guid id, string resolvedUrl);
+
+    /// <summary>
+    /// Phase 4: Saves AI classification from n8n and sets the lifecycle status to "Completed".
+    /// </summary>
+    Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets);
+
+    /// <summary>
+    /// Retrieves articles ready for historical sync or sentiment processing.
+    /// </summary>
+    Task<List<NewsArticleEntity>> GetCompletedArticlesAsync(int limit, int offset, string? isin = null);
+
+    /// <summary>
+    /// Fetches articles matching specific lifecycle statuses (e.g. "Pending", "Scraping" for Phase 1 retry).
+    /// </summary>
+    Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(params string[] statuses);
+
+    /// <summary>
+    /// Fetches public daily news for API endpoints, filtering out intermediate or failed lifecycle states by default.
+    /// </summary>
+    Task<List<NewsArticleEntity>> GetFilteredNewsAsync(
+        int limit = 20,
+        int offset = 0,
+        string? isin = null,
+        DateTime? date = null,
+        string? status = null,
+        string? searchQuery = null);
+
+    Task<NewsArticleEntity?> GetArticleByIdAsync(Guid id);
+}
+
+/// <inheritdoc />
+public class NewsDbService : INewsDbService
+{
+    private readonly NewsDbContext _context;
+    private readonly ILogger<NewsDbService> _logger;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="NewsDbService"/> class.
+    /// </summary>
+    public NewsDbService(NewsDbContext context, ILogger<NewsDbService> logger)
+    {
+        _context = context;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public async Task<List<ArticleSourceEntity>> GetSourcesAsync()
+    {
+        return await _context.ArticleSources.AsNoTracking().ToListAsync();
+    }
+
+    /// <inheritdoc />
+    public async Task<NewsArticleEntity?> GetArticleByIdAsync(Guid id)
+    {
+        return await _context.NewsArticles
+            .Include(a => a.MatchedAssets)
+            .AsNoTracking()
+            .FirstOrDefaultAsync(a => a.Id == id);
+    }
+
+    /// <inheritdoc />
+    public async Task<bool> IsUrlDuplicateAsync(string url)
+    {
+        var trimmedUrl = url.Trim();
+        return await _context.NewsArticles
+            .AsNoTracking()
+            .AnyAsync(a => a.SourceUrl == trimmedUrl);
+    }
+
+    /// <inheritdoc />
+    /// <summary>
+    /// Lifecycle Step 1: Creates a new article in 'Pending' state as an immediate lock.
+    /// </summary>
+    public async Task<NewsArticleEntity> CreatePendingArticleAsync(
+        string url,
+        List<string>? discoveredIsins = null,
+        string? title = null,
+        string? summary = null,
+        DateTime? publishedAt = null,
+        string? language = null)
+    {
+        var trimmedUrl = url.Trim();
+
+        var existingArticle = await _context.NewsArticles
+            .FirstOrDefaultAsync(a => a.SourceUrl == trimmedUrl);
+
+        if (existingArticle != null)
+        {
+            _logger.LogDebug("[Lifecycle] Article URL already exists (Duplicate hit): {Url}", trimmedUrl);
+            return existingArticle;
+        }
+
+        var finalPublishedAt = publishedAt.HasValue
+            ? (publishedAt.Value.Kind == DateTimeKind.Unspecified
+                ? DateTime.SpecifyKind(publishedAt.Value, DateTimeKind.Utc)
+                : publishedAt.Value.ToUniversalTime())
+            : DateTime.UtcNow;
+
+        if (finalPublishedAt > DateTime.UtcNow)
+        {
+            finalPublishedAt = DateTime.UtcNow;
+        }
+
+        var article = new NewsArticleEntity
+        {
+            Id = Guid.NewGuid(),
+            Title = !string.IsNullOrWhiteSpace(title) ? title : "Pending Discovery",
+            Summary = !string.IsNullOrWhiteSpace(summary) ? summary : "Extraction in progress...",
+            ContentRaw = "Extraction in progress...",
+            SourceUrl = trimmedUrl,
+            Language = language,
+            ScrapedAt = DateTime.UtcNow,
+            PublishedAt = finalPublishedAt,
+            Status = "Pending" // 1. Pending State
+        };
+
+        if (discoveredIsins != null && discoveredIsins.Count > 0)
+        {
+            foreach (var isin in discoveredIsins)
+            {
+                article.MatchedAssets.Add(new MatchedAssetEntity
+                {
+                    Id = Guid.NewGuid(),
+                    NewsArticleId = article.Id,
+                    Isin = isin,
+                    Name = isin
+                });
+            }
+        }
+
+        try
+        {
+            _context.NewsArticles.Add(article);
+            await _context.SaveChangesAsync();
+            _logger.LogDebug("[Lifecycle] Registered new article with status 'Pending'. ID: {Id}, Url: {Url}", article.Id, trimmedUrl);
+        }
+        catch (DbUpdateConcurrencyException)
+        {
+            _logger.LogWarning("[{Channel}] Concurrency hit during insert for URL: {Url}. Fetching existing fallback.", "NewsChannel", trimmedUrl);
+            return await _context.NewsArticles.FirstAsync(a => a.SourceUrl == trimmedUrl);
+        }
+
+        return article;
+    }
+
+    /// <inheritdoc />
+    /// <summary>
+    /// Lifecycle Step 2 & 5: Updates state (e.g. Pending -> Processing -> Scraping / Failed / Analyzed).
+    /// </summary>
+    public async Task UpdateArticleStatusAsync(Guid id, string status)
+    {
+        var rowsAffected = await _context.NewsArticles
+            .Where(a => a.Id == id)
+            .ExecuteUpdateAsync(s => s.SetProperty(a => a.Status, status));
+
+        if (rowsAffected == 0)
+        {
+            _logger.LogWarning("[{Channel}] Attempted status transition for non-existing article. ID: {Id}", "NewsChannel", id);
+        }
+        else
+        {
+            _logger.LogDebug("[Lifecycle] Transitioned article {Id} to status '{Status}'", id, status);
+        }
+    }
+
+    /// <inheritdoc />
+    public async Task UpdateArticleUrlAsync(Guid id, string resolvedUrl)
+    {
+        var rowsAffected = await _context.NewsArticles
+            .Where(a => a.Id == id)
+            .ExecuteUpdateAsync(s => s.SetProperty(a => a.SourceUrl, resolvedUrl));
+
+        if (rowsAffected == 0)
+        {
+            _logger.LogWarning("[{Channel}] Attempted URL update for non-existing article. ID: {Id}", "NewsChannel", id);
+        }
+        else
+        {
+            _logger.LogDebug("[Lifecycle] Resolved redirect for article {Id} -> New URL: {Url}", id, resolvedUrl);
+        }
+    }
+
+    /// <inheritdoc />
+    /// <summary>
+    /// Lifecycle Step 4: Persists n8n classification and transitions status to 'Completed'.
+    /// </summary>
+    public async Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets)
+    {
+        var article = await _context.NewsArticles
+            .FirstOrDefaultAsync(a => a.Id == id);
+
+        if (article == null)
+        {
+            _logger.LogWarning("[{Channel}] Article with ID {Id} not found for classification update.", "NewsChannel", id);
+            return null;
+        }
+
+        article.Title = payload.Title;
+        article.Author = payload.Author;
+        article.Summary = payload.Summary;
+        article.ContentRaw = payload.ContentRaw;
+        article.Language = payload.Language;
+        
+        // 🎯 Step 4: Classification finished -> Transition to 'Completed' (triggers MQTT broadcast)
+        article.Status = "Completed";
+
+        if (DateTime.TryParse(payload.PublishedAt, out var publishedDate))
+        {
+            article.PublishedAt = publishedDate.Kind == DateTimeKind.Unspecified
+                ? DateTime.SpecifyKind(publishedDate, DateTimeKind.Utc)
+                : publishedDate.ToUniversalTime();
+        }
+
+        if (DateTime.TryParse(payload.ScrapedAt, out var scrapedDate))
+        {
+            article.ScrapedAt = scrapedDate.ToUniversalTime();
+        }
+
+        // Clean up previous temporary assets
+        await _context.MatchedAssets.Where(m => m.NewsArticleId == id).ExecuteDeleteAsync();
+
+        article.MatchedAssets = new List<MatchedAssetEntity>();
+        foreach (var asset in matchedAssets)
+        {
+            if (asset.Id == Guid.Empty)
+            {
+                asset.Id = Guid.NewGuid();
+            }
+
+            _context.MatchedAssets.Add(asset);
+            article.MatchedAssets.Add(asset);
+        }
+
+        await _context.SaveChangesAsync();
+        _logger.LogInformation("[Lifecycle] Article {Id} successfully classified and marked 'Completed'. Title: '{Title}'", article.Id, article.Title);
+
+        return article;
+    }
+
+    /// <inheritdoc />
+    public async Task<List<NewsArticleEntity>> GetCompletedArticlesAsync(int limit, int offset, string? isin = null)
+    {
+        var query = _context.NewsArticles
+            .Include(a => a.MatchedAssets)
+            .Where(a => a.Status == "Completed" || a.Status == "Analyzed");
+
+        if (!string.IsNullOrWhiteSpace(isin))
+        {
+            var cleanIsin = isin.Trim();
+            query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin || m.Name == cleanIsin));
+        }
+
+        return await query
+            .OrderByDescending(a => a.PublishedAt)
+            .ThenByDescending(a => a.ScrapedAt)
+            .ThenByDescending(a => a.Id)
+            .Skip(offset)
+            .Take(limit)
+            .AsNoTracking()
+            .ToListAsync();
+    }
+
+    /// <inheritdoc />
+    /// <summary>
+    /// Lifecycle Helper: Retrieves articles by target lifecycle status (e.g. "Pending", "Scraping" for Phase 1 Retry).
+    /// </summary>
+    public async Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(params string[] statuses)
+    {
+        IQueryable<NewsArticleEntity> query = _context.NewsArticles
+            .Include(a => a.MatchedAssets)
+            .AsNoTracking();
+
+        if (statuses != null && statuses.Length > 0)
+        {
+            var cleanStatuses = statuses.Select(s => s.Trim()).ToList();
+            query = query.Where(a => cleanStatuses.Contains(a.Status));
+        }
+        else
+        {
+            query = query.Where(a => a.Status != "Failed" && a.Status != "Duplicate");
+        }
+
+        return await query.ToListAsync();
+    }
+
+    /// <inheritdoc />
+    /// <summary>
+    /// API Gateway Helper: Retrieves articles for UI rendering, excluding intermediate/failed states by default.
+    /// </summary>
+    public async Task<List<NewsArticleEntity>> GetFilteredNewsAsync(
+        int limit = 20,
+        int offset = 0,
+        string? isin = null,
+        DateTime? date = null,
+        string? status = null,
+        string? searchQuery = null)
+    {
+        IQueryable<NewsArticleEntity> query = _context.NewsArticles
+            .Include(a => a.MatchedAssets)
+            .AsNoTracking();
+
+        // 1. Status Filter
+        if (!string.IsNullOrWhiteSpace(status))
+        {
+            var targetStatus = status.Trim();
+            query = query.Where(a => a.Status == targetStatus);
+        }
+        else
+        {
+            // By default, only show fully processed articles to the API/UI
+            query = query.Where(a => a.Status == "Completed" || a.Status == "Analyzed");
+        }
+
+        // 2. Date Filter
+        if (date.HasValue)
+        {
+            var targetDate = DateTime.SpecifyKind(date.Value.Date, DateTimeKind.Utc);
+            var nextDate = targetDate.AddDays(1);
+            query = query.Where(a => a.PublishedAt >= targetDate && a.PublishedAt < nextDate);
+        }
+
+        // 3. ISIN / Symbol Filter
+        if (!string.IsNullOrWhiteSpace(isin))
+        {
+            var cleanIsin = isin.Trim();
+            query = query.Where(a => a.MatchedAssets.Any(m => m.Isin == cleanIsin || m.Name == cleanIsin));
+        }
+
+        // 4. Search Term
+        if (!string.IsNullOrWhiteSpace(searchQuery))
+        {
+            var q = searchQuery.Trim();
+            query = query.Where(a => EF.Functions.ILike(a.Title, $"%{q}%") || (a.Summary != null && EF.Functions.ILike(a.Summary, $"%{q}%")));
+        }
+
+        return await query
+            .OrderByDescending(a => a.PublishedAt)
+            .ThenByDescending(a => a.ScrapedAt)
+            .ThenByDescending(a => a.Id)
+            .Skip(offset)
+            .Take(limit)
+            .ToListAsync();
+    }
+}