Finlytic/FinlyticNews/Services/ArticleDeduplicationService.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Services;
using FinlyticNews.Database;
using FinlyticNews.Util;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;

namespace FinlyticNews.Services;

/// <summary>
/// Result of a duplicate check on a candidate news article.
/// </summary>
public record DuplicateCheckResult(
    bool IsDuplicate,
    Guid? DuplicateOfArticleId,
    string? Reason,
    string TitleHash,
    long SimHash
);

/// <summary>
/// Service interface for detecting syndicated or near-identical duplicate news articles without AI.
/// </summary>
public interface IArticleDeduplicationService
{
    /// <summary>
    /// Evaluates whether an article is a duplicate based on normalized title similarity and 64-bit text SimHash fingerprinting.
    /// </summary>
    DuplicateCheckResult CheckDuplicate(string title, string content, DateTime publishedAt);

    /// <summary>
    /// Registers a newly processed article into the in-memory deduplication cache.
    /// </summary>
    void RegisterArticle(Guid articleId, string titleHash, long simHash, string title, DateTime publishedAt);

    /// <summary>
    /// Loads recent articles from the database into the deduplication cache on startup.
    /// </summary>
    Task InitializeAsync(CancellationToken cancellationToken = default);

    /// <summary>
    /// Computes the normalized SHA-256 hash of an article title.
    /// </summary>
    string ComputeTitleHash(string title);

    /// <summary>
    /// Computes the 64-bit SimHash content fingerprint of article text.
    /// </summary>
    long ComputeSimHash(string content);
}

/// <summary>
/// High-performance non-AI deduplication engine using 64-bit SimHash and N-Gram title similarity.
/// </summary>
public class ArticleDeduplicationService : IArticleDeduplicationService
{
    private record CachedArticleEntry(
        Guid ArticleId,
        string TitleHash,
        long SimHash,
        string NormalizedTitle,
        DateTime PublishedAt
    );

    private readonly IServiceScopeFactory _scopeFactory;
    private readonly IFinlyticLogger<ArticleDeduplicationService> _finlyticLogger;
    private readonly ISettingsService _settingsService;

    // Rolling in-memory cache of recent articles
    private readonly ConcurrentDictionary<string, Guid> _titleHashIndex = new(StringComparer.OrdinalIgnoreCase);
    private readonly List<CachedArticleEntry> _simHashIndex = [];
    private readonly ReaderWriterLockSlim _simHashLock = new();

    private bool _initialized;
    private readonly SemaphoreSlim _initLock = new(1, 1);

    private static readonly Regex TitlePortalSuffixRegex = new(
        @"\s*[-|–—]\s*(DER AKTIONÄR|ARIVA\.DE|IT-Times|onvista|wallstreet:online|Sharedeals\.de|boerse\.de|Handelsblatt|WirtschaftsWoche|finanzen\.net|Finanznachrichten|XTB|Lynx|T3n|ntg24|Moneycab).*$",
        RegexOptions.Compiled | RegexOptions.IgnoreCase);

    private static readonly Regex NonAlphanumericRegex = new(@"[^\w\s]", RegexOptions.Compiled);
    private static readonly Regex MultipleSpacesRegex = new(@"\s+", RegexOptions.Compiled);

    public ArticleDeduplicationService(
        IServiceScopeFactory scopeFactory,
        IFinlyticLogger<ArticleDeduplicationService> finlyticLogger,
        ISettingsService settingsService)
    {
        _scopeFactory = scopeFactory;
        _finlyticLogger = finlyticLogger;
        _settingsService = settingsService;
    }

    public async Task InitializeAsync(CancellationToken cancellationToken = default)
    {
        if (_initialized) return;

        await _initLock.WaitAsync(cancellationToken);
        try
        {
            if (_initialized) return;

            using var scope = _scopeFactory.CreateScope();
            var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();

            var windowDays = await _settingsService.GetSettingAsync(SettingKeys.DeduplicationWindowDays, cancellationToken);
            var cutoff = DateTime.UtcNow.AddDays(-Math.Max(windowDays, 7));

            var articles = await dbContext.NewsArticles
                .AsNoTracking()
                .Where(a => a.PublishedAt >= cutoff && a.Status == "Completed")
                .Select(a => new
                {
                    a.Id,
                    a.Title,
                    a.TitleHash,
                    a.SimHash,
                    a.PublishedAt
                })
                .ToListAsync(cancellationToken);

            _simHashLock.EnterWriteLock();
            try
            {
                foreach (var a in articles)
                {
                    var tHash = a.TitleHash ?? ComputeTitleHash(a.Title);
                    var sHash = a.SimHash ?? 0L;
                    var normTitle = NormalizeTitle(a.Title);

                    _titleHashIndex[tHash] = a.Id;

                    if (sHash != 0L)
                    {
                        _simHashIndex.Add(new CachedArticleEntry(a.Id, tHash, sHash, normTitle, a.PublishedAt));
                    }
                }
            }
            finally
            {
                _simHashLock.ExitWriteLock();
            }

            _initialized = true;
            await _finlyticLogger.LogInfoAsync(SettingKeys.DeduplicationChannel, "[ArticleDeduplicationService] Initialized with {Count} recent articles for duplicate detection.", articles.Count);
        }
        finally
        {
            _initLock.Release();
        }
    }

    public DuplicateCheckResult CheckDuplicate(string title, string content, DateTime publishedAt)
    {
        var titleHash = ComputeTitleHash(title);
        var simHash = ComputeSimHash(content);
        var normalizedTitle = NormalizeTitle(title);

        // 1. Exact Title Hash Match
        if (_titleHashIndex.TryGetValue(titleHash, out var exactMatchId))
        {
            return new DuplicateCheckResult(
                IsDuplicate: true,
                DuplicateOfArticleId: exactMatchId,
                Reason: "ExactTitleHashMatch",
                TitleHash: titleHash,
                SimHash: simHash
            );
        }

        _simHashLock.EnterReadLock();
        try
        {
            var windowCutoff = publishedAt.AddDays(-7);

            foreach (var entry in _simHashIndex)
            {
                if (entry.PublishedAt < windowCutoff) continue;

                // 2. SimHash Content Similarity (Hamming Distance <= 3 bits -> > 90% identical text)
                if (simHash != 0L && entry.SimHash != 0L)
                {
                    int distance = HammingDistance(simHash, entry.SimHash);
                    if (distance <= 3)
                    {
                        return new DuplicateCheckResult(
                            IsDuplicate: true,
                            DuplicateOfArticleId: entry.ArticleId,
                            Reason: $"SimHashSimilarity (HammingDistance: {distance})",
                            TitleHash: titleHash,
                            SimHash: simHash
                        );
                    }
                }

                // 3. Fuzzy Title Similarity (3-gram Jaccard Index >= 0.85)
                double titleSim = Calculate3GramJaccard(normalizedTitle, entry.NormalizedTitle);
                if (titleSim >= 0.85)
                {
                    return new DuplicateCheckResult(
                        IsDuplicate: true,
                        DuplicateOfArticleId: entry.ArticleId,
                        Reason: $"FuzzyTitleSimilarity ({titleSim:P0})",
                        TitleHash: titleHash,
                        SimHash: simHash
                    );
                }
            }
        }
        finally
        {
            _simHashLock.ExitReadLock();
        }

        return new DuplicateCheckResult(
            IsDuplicate: false,
            DuplicateOfArticleId: null,
            Reason: null,
            TitleHash: titleHash,
            SimHash: simHash
        );
    }

    public void RegisterArticle(Guid articleId, string titleHash, long simHash, string title, DateTime publishedAt)
    {
        _titleHashIndex[titleHash] = articleId;

        _simHashLock.EnterWriteLock();
        try
        {
            _simHashIndex.Add(new CachedArticleEntry(articleId, titleHash, simHash, NormalizeTitle(title), publishedAt));
        }
        finally
        {
            _simHashLock.ExitWriteLock();
        }
    }

    public string ComputeTitleHash(string title)
    {
        var normalized = NormalizeTitle(title);
        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(normalized));
        return Convert.ToHexString(bytes).ToLowerInvariant();
    }

    public long ComputeSimHash(string content)
    {
        if (string.IsNullOrWhiteSpace(content)) return 0L;

        // Clean and tokenize content into words
        var clean = NonAlphanumericRegex.Replace(content.ToLowerInvariant(), " ");
        var words = clean.Split(' ', StringSplitOptions.RemoveEmptyEntries);
        if (words.Length < 10) return 0L;

        // Generate 3-word shingles
        var v = new int[64];
        for (int i = 0; i < words.Length - 2; i++)
        {
            var shingle = $"{words[i]} {words[i + 1]} {words[i + 2]}";
            var hash = Hash64(shingle);

            for (int bit = 0; bit < 64; bit++)
            {
                if (((hash >> bit) & 1L) == 1L)
                {
                    v[bit]++;
                }
                else
                {
                    v[bit]--;
                }
            }
        }

        long simHash = 0L;
        for (int bit = 0; bit < 64; bit++)
        {
            if (v[bit] > 0)
            {
                simHash |= (1L << bit);
            }
        }

        return simHash;
    }

    private static string NormalizeTitle(string title)
    {
        if (string.IsNullOrWhiteSpace(title)) return string.Empty;
        var stripped = TitlePortalSuffixRegex.Replace(title, "");
        var cleaned = NonAlphanumericRegex.Replace(stripped.ToLowerInvariant(), " ");
        return MultipleSpacesRegex.Replace(cleaned, " ").Trim();
    }

    private static double Calculate3GramJaccard(string a, string b)
    {
        if (string.IsNullOrEmpty(a) || string.IsNullOrEmpty(b)) return 0.0;
        if (a == b) return 1.0;

        var gramsA = Get3Grams(a);
        var gramsB = Get3Grams(b);

        if (gramsA.Count == 0 || gramsB.Count == 0) return 0.0;

        int intersection = 0;
        foreach (var g in gramsA)
        {
            if (gramsB.Contains(g)) intersection++;
        }

        int union = gramsA.Count + gramsB.Count - intersection;
        return union == 0 ? 0.0 : (double)intersection / union;
    }

    private static HashSet<string> Get3Grams(string text)
    {
        var set = new HashSet<string>(StringComparer.Ordinal);
        if (text.Length < 3)
        {
            set.Add(text);
            return set;
        }

        for (int i = 0; i <= text.Length - 3; i++)
        {
            set.Add(text.Substring(i, 3));
        }
        return set;
    }

    private static int HammingDistance(long a, long b)
    {
        return BitOperations.PopCount((ulong)(a ^ b));
    }

    private static long Hash64(string text)
    {
        // 64-bit FNV-1a Hash
        ulong hash = 14695981039346656037UL;
        var bytes = Encoding.UTF8.GetBytes(text);
        foreach (var b in bytes)
        {
            hash ^= b;
            hash *= 1099511628211UL;
        }
        return (long)hash;
    }
}