feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

FinlyticNews/Services/NewsBlocklistService.cs

@@ -0,0 +1,126 @@
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Services;
+using FinlyticNews.Database;
+using FinlyticNews.Entities;
+using FinlyticNews.Util;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Service interface for managing and querying blocked news article URLs (duplicates, missing content, or zero matched assets).
+/// </summary>
+public interface INewsBlocklistService
+{
+    /// <summary>
+    /// Checks whether the given URL is already recorded on the blocklist.
+    /// </summary>
+    bool IsBlocked(string url);
+
+    /// <summary>
+    /// Adds a URL to the database blocklist and in-memory cache.
+    /// </summary>
+    Task BlockUrlAsync(string url, string reason, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Loads all existing blocked URLs from the database into memory on startup.
+    /// </summary>
+    Task InitializeAsync(CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// High-performance in-memory and database-backed blocklist service.
+/// </summary>
+public class NewsBlocklistService : INewsBlocklistService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly IFinlyticLogger<NewsBlocklistService> _finlyticLogger;
+    private readonly ConcurrentDictionary<string, string> _blockedUrls = new(StringComparer.OrdinalIgnoreCase);
+    private bool _initialized;
+    private readonly SemaphoreSlim _initLock = new(1, 1);
+
+    public NewsBlocklistService(
+        IServiceScopeFactory scopeFactory,
+        IFinlyticLogger<NewsBlocklistService> finlyticLogger)
+    {
+        _scopeFactory = scopeFactory;
+        _finlyticLogger = finlyticLogger;
+    }
+
+    public async Task InitializeAsync(CancellationToken cancellationToken = default)
+    {
+        if (_initialized) return;
+
+        await _initLock.WaitAsync(cancellationToken);
+        try
+        {
+            if (_initialized) return;
+
+            using var scope = _scopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();
+
+            var urls = await dbContext.BlockedNewsUrls
+                .AsNoTracking()
+                .Select(b => new { b.Url, b.Reason })
+                .ToListAsync(cancellationToken);
+
+            foreach (var item in urls)
+            {
+                if (!string.IsNullOrWhiteSpace(item.Url))
+                {
+                    _blockedUrls[item.Url.Trim()] = item.Reason ?? string.Empty;
+                }
+            }
+
+            _initialized = true;
+            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsBlocklistService] Initialized with {Count} blocked URLs in memory cache.", _blockedUrls.Count);
+        }
+        finally
+        {
+            _initLock.Release();
+        }
+    }
+
+    public bool IsBlocked(string url)
+    {
+        if (string.IsNullOrWhiteSpace(url)) return true;
+        return _blockedUrls.ContainsKey(url.Trim());
+    }
+
+    public async Task BlockUrlAsync(string url, string reason, CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(url)) return;
+        var cleanUrl = url.Trim();
+
+        _blockedUrls[cleanUrl] = reason;
+
+        try
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();
+
+            bool exists = await dbContext.BlockedNewsUrls.AnyAsync(b => b.Url == cleanUrl, cancellationToken);
+            if (!exists)
+            {
+                dbContext.BlockedNewsUrls.Add(new BlockedNewsUrlEntity
+                {
+                    Id = Guid.NewGuid(),
+                    Url = cleanUrl,
+                    Reason = reason,
+                    BlockedAtUtc = DateTime.UtcNow
+                });
+                await dbContext.SaveChangesAsync(cancellationToken);
+                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsBlocklistService] Added URL to blocklist: {Url} (Reason: {Reason})", cleanUrl, reason);
+            }
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[NewsBlocklistService] Failed to persist blocked URL: {Url}", cleanUrl);
+        }
+    }
+}