feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

2026-08-24 21:35:33 +02:00
parent 44b161d509
commit 8112598602
400 changed files with 1812 additions and 113367 deletions
@@ -0,0 +1,151 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Dtos.Assets;
+using FinlyticCore.Models.Assets;
+using FinlyticCore.Services;
+using FinlyticNews.Util;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Service interface for validating candidate asset matches against FinlyticAssets metadata and sector clustering.
+/// </summary>
+public interface IAssetValidationService
+{
+    /// <summary>
+    /// Validates and filters candidate asset matches using FinlyticAssets metadata, sector clustering, and occurrence rules.
+    /// </summary>
+    Task<List<MatchedCandidateAsset>> ValidateCandidateAssetsAsync(
+        List<MatchedCandidateAsset> candidates,
+        string title,
+        string content,
+        CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Heuristic asset validation engine verifying asset existence, title priority, mention density, and sector coherence.
+/// </summary>
+public class AssetValidationService : IAssetValidationService
+{
+    private readonly IFinlyticLogger<AssetValidationService> _finlyticLogger;
+    private readonly NewsMqttClient _mqttClient;
+    private readonly ISettingsService _settingsService;
+
+    public AssetValidationService(
+        IFinlyticLogger<AssetValidationService> finlyticLogger,
+        NewsMqttClient mqttClient,
+        ISettingsService settingsService)
+    {
+        _finlyticLogger = finlyticLogger;
+        _mqttClient = mqttClient;
+        _settingsService = settingsService;
+    }
+
+    public async Task<List<MatchedCandidateAsset>> ValidateCandidateAssetsAsync(
+        List<MatchedCandidateAsset> candidates,
+        string title,
+        string content,
+        CancellationToken cancellationToken = default)
+    {
+        if (candidates.Count == 0) return [];
+
+        var verifiedAssets = new List<MatchedCandidateAsset>();
+        var assetDetails = new Dictionary<string, AssetDto>(StringComparer.OrdinalIgnoreCase);
+
+        // 1. Fetch Asset Details from FinlyticAssets via MQTT RPC
+        foreach (var candidate in candidates)
+        {
+            try
+            {
+                var req = new GetValidAssetRequest(candidate.Isin);
+                var assets = await _mqttClient.RequestAsync<GetValidAssetRequest, List<AssetDto>>("services/request/assets_Get", req);
+
+                if (assets != null && assets.Count > 0)
+                {
+                    var match = assets.FirstOrDefault(a => a.Isin.Equals(candidate.Isin, StringComparison.OrdinalIgnoreCase));
+                    if (match != null)
+                    {
+                        assetDetails[candidate.Isin] = match;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, ex, "[AssetValidationService] Failed to query FinlyticAssets for ISIN {Isin}. Falling back to index data.", candidate.Isin);
+            }
+        }
+
+        // 2. Tag & Sector Frequency Analysis (Clustering)
+        var sectorCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+        foreach (var (_, dto) in assetDetails)
+        {
+            if (dto.Tags != null)
+            {
+                foreach (var tag in dto.Tags)
+                {
+                    if (tag.Type.Equals("sector", StringComparison.OrdinalIgnoreCase) ||
+                        tag.Type.Equals("industry", StringComparison.OrdinalIgnoreCase))
+                    {
+                        sectorCounts[tag.Name] = sectorCounts.GetValueOrDefault(tag.Name, 0) + 1;
+                    }
+                }
+            }
+        }
+
+        bool hasDominantSector = sectorCounts.Values.Any(count => count >= 2);
+        string? dominantSector = sectorCounts.OrderByDescending(kv => kv.Value).FirstOrDefault().Key;
+
+        // 3. Evaluate Each Candidate
+        foreach (var candidate in candidates)
+        {
+            // Rule A: Direct Title Match or URL Match -> Guaranteed Valid
+            if (candidate.InTitle || candidate.InUrl)
+            {
+                verifiedAssets.Add(candidate);
+                continue;
+            }
+
+            // Rule B: Multiple Mentions in Article Body
+            if (candidate.MentionCount >= 2)
+            {
+                verifiedAssets.Add(candidate);
+                continue;
+            }
+
+            // Rule C: Single Mention Validation with Sector Clustering
+            if (assetDetails.TryGetValue(candidate.Isin, out var dto))
+            {
+                bool matchesDominantSector = false;
+                if (hasDominantSector && dto.Tags != null)
+                {
+                    matchesDominantSector = dto.Tags.Any(t => t.Name.Equals(dominantSector, StringComparison.OrdinalIgnoreCase));
+                }
+
+                if (matchesDominantSector)
+                {
+                    // Reinforced by cluster
+                    verifiedAssets.Add(candidate);
+                }
+                else if (candidate.InitialConfidence >= 0.85)
+                {
+                    // High-confidence exact unique name match
+                    verifiedAssets.Add(candidate);
+                }
+                else
+                {
+                    await _finlyticLogger.LogDebugAsync(SettingKeys.MatcherChannel, "[AssetValidationService] Pruned low-confidence outlier candidate: {Name} ({Isin})", candidate.Name, candidate.Isin);
+                }
+            }
+            else
+            {
+                // If not found in FinlyticAssets, discard
+                await _finlyticLogger.LogDebugAsync(SettingKeys.MatcherChannel, "[AssetValidationService] Discarded candidate {Isin} as it does not exist in FinlyticAssets.", candidate.Isin);
+            }
+        }
+
+        return verifiedAssets;
+    }
+}