Finlytic/FinlyticNews/Services/AssetValidationService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.Assets;
using FinlyticCore.Models.Assets;
using FinlyticCore.Services;
using FinlyticNews.Util;

namespace FinlyticNews.Services;

/// <summary>
/// Service interface for validating candidate asset matches against FinlyticAssets metadata and sector clustering.
/// </summary>
public interface IAssetValidationService
{
    /// <summary>
    /// Validates and filters candidate asset matches using FinlyticAssets metadata, sector clustering, and occurrence rules.
    /// </summary>
    Task<List<MatchedCandidateAsset>> ValidateCandidateAssetsAsync(
        List<MatchedCandidateAsset> candidates,
        string title,
        string content,
        CancellationToken cancellationToken = default);
}

/// <summary>
/// Heuristic asset validation engine verifying asset existence, title priority, mention density, and sector coherence.
/// </summary>
public class AssetValidationService : IAssetValidationService
{
    private readonly IFinlyticLogger<AssetValidationService> _finlyticLogger;
    private readonly NewsMqttClient _mqttClient;
    private readonly ISettingsService _settingsService;

    public AssetValidationService(
        IFinlyticLogger<AssetValidationService> finlyticLogger,
        NewsMqttClient mqttClient,
        ISettingsService settingsService)
    {
        _finlyticLogger = finlyticLogger;
        _mqttClient = mqttClient;
        _settingsService = settingsService;
    }

    public async Task<List<MatchedCandidateAsset>> ValidateCandidateAssetsAsync(
        List<MatchedCandidateAsset> candidates,
        string title,
        string content,
        CancellationToken cancellationToken = default)
    {
        if (candidates.Count == 0) return [];

        var verifiedAssets = new List<MatchedCandidateAsset>();
        var assetDetails = new Dictionary<string, AssetDto>(StringComparer.OrdinalIgnoreCase);

        // 1. Fetch Asset Details from FinlyticAssets via MQTT RPC
        foreach (var candidate in candidates)
        {
            try
            {
                var req = new GetValidAssetRequest(candidate.Isin);
                var assets = await _mqttClient.RequestAsync<GetValidAssetRequest, List<AssetDto>>("services/request/assets_Get", req);

                if (assets != null && assets.Count > 0)
                {
                    var match = assets.FirstOrDefault(a => a.Isin.Equals(candidate.Isin, StringComparison.OrdinalIgnoreCase));
                    if (match != null)
                    {
                        assetDetails[candidate.Isin] = match;
                    }
                }
            }
            catch (Exception ex)
            {
                await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, ex, "[AssetValidationService] Failed to query FinlyticAssets for ISIN {Isin}. Falling back to index data.", candidate.Isin);
            }
        }

        // 2. Tag & Sector Frequency Analysis (Clustering)
        var sectorCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
        foreach (var (_, dto) in assetDetails)
        {
            if (dto.Tags != null)
            {
                foreach (var tag in dto.Tags)
                {
                    if (tag.Type.Equals("sector", StringComparison.OrdinalIgnoreCase) ||
                        tag.Type.Equals("industry", StringComparison.OrdinalIgnoreCase))
                    {
                        sectorCounts[tag.Name] = sectorCounts.GetValueOrDefault(tag.Name, 0) + 1;
                    }
                }
            }
        }

        bool hasDominantSector = sectorCounts.Values.Any(count => count >= 2);
        string? dominantSector = sectorCounts.OrderByDescending(kv => kv.Value).FirstOrDefault().Key;

        // 3. Evaluate Each Candidate
        foreach (var candidate in candidates)
        {
            // Rule A: Direct Title Match or URL Match -> Guaranteed Valid
            if (candidate.InTitle || candidate.InUrl)
            {
                verifiedAssets.Add(candidate);
                continue;
            }

            // Rule B: Multiple Mentions in Article Body
            if (candidate.MentionCount >= 2)
            {
                verifiedAssets.Add(candidate);
                continue;
            }

            // Rule C: Single Mention Validation with Sector Clustering
            if (assetDetails.TryGetValue(candidate.Isin, out var dto))
            {
                bool matchesDominantSector = false;
                if (hasDominantSector && dto.Tags != null)
                {
                    matchesDominantSector = dto.Tags.Any(t => t.Name.Equals(dominantSector, StringComparison.OrdinalIgnoreCase));
                }

                if (matchesDominantSector)
                {
                    // Reinforced by cluster
                    verifiedAssets.Add(candidate);
                }
                else if (candidate.InitialConfidence >= 0.85)
                {
                    // High-confidence exact unique name match
                    verifiedAssets.Add(candidate);
                }
                else
                {
                    await _finlyticLogger.LogDebugAsync(SettingKeys.MatcherChannel, "[AssetValidationService] Pruned low-confidence outlier candidate: {Name} ({Isin})", candidate.Name, candidate.Isin);
                }
            }
            else
            {
                // If not found in FinlyticAssets, discard
                await _finlyticLogger.LogDebugAsync(SettingKeys.MatcherChannel, "[AssetValidationService] Discarded candidate {Isin} as it does not exist in FinlyticAssets.", candidate.Isin);
            }
        }

        return verifiedAssets;
    }
}