feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts
This commit is contained in:
@@ -0,0 +1,357 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticNews.Database;
|
||||
using FinlyticNews.Util;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Result of a duplicate check on a candidate news article.
|
||||
/// </summary>
|
||||
public record DuplicateCheckResult(
|
||||
bool IsDuplicate,
|
||||
Guid? DuplicateOfArticleId,
|
||||
string? Reason,
|
||||
string TitleHash,
|
||||
long SimHash
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Service interface for detecting syndicated or near-identical duplicate news articles without AI.
|
||||
/// </summary>
|
||||
public interface IArticleDeduplicationService
|
||||
{
|
||||
/// <summary>
|
||||
/// Evaluates whether an article is a duplicate based on normalized title similarity and 64-bit text SimHash fingerprinting.
|
||||
/// </summary>
|
||||
DuplicateCheckResult CheckDuplicate(string title, string content, DateTime publishedAt);
|
||||
|
||||
/// <summary>
|
||||
/// Registers a newly processed article into the in-memory deduplication cache.
|
||||
/// </summary>
|
||||
void RegisterArticle(Guid articleId, string titleHash, long simHash, string title, DateTime publishedAt);
|
||||
|
||||
/// <summary>
|
||||
/// Loads recent articles from the database into the deduplication cache on startup.
|
||||
/// </summary>
|
||||
Task InitializeAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Computes the normalized SHA-256 hash of an article title.
|
||||
/// </summary>
|
||||
string ComputeTitleHash(string title);
|
||||
|
||||
/// <summary>
|
||||
/// Computes the 64-bit SimHash content fingerprint of article text.
|
||||
/// </summary>
|
||||
long ComputeSimHash(string content);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// High-performance non-AI deduplication engine using 64-bit SimHash and N-Gram title similarity.
|
||||
/// </summary>
|
||||
public class ArticleDeduplicationService : IArticleDeduplicationService
|
||||
{
|
||||
private record CachedArticleEntry(
|
||||
Guid ArticleId,
|
||||
string TitleHash,
|
||||
long SimHash,
|
||||
string NormalizedTitle,
|
||||
DateTime PublishedAt
|
||||
);
|
||||
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IFinlyticLogger<ArticleDeduplicationService> _finlyticLogger;
|
||||
private readonly ISettingsService _settingsService;
|
||||
|
||||
// Rolling in-memory cache of recent articles
|
||||
private readonly ConcurrentDictionary<string, Guid> _titleHashIndex = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly List<CachedArticleEntry> _simHashIndex = [];
|
||||
private readonly ReaderWriterLockSlim _simHashLock = new();
|
||||
|
||||
private bool _initialized;
|
||||
private readonly SemaphoreSlim _initLock = new(1, 1);
|
||||
|
||||
private static readonly Regex TitlePortalSuffixRegex = new(
|
||||
@"\s*[-|–—]\s*(DER AKTIONÄR|ARIVA\.DE|IT-Times|onvista|wallstreet:online|Sharedeals\.de|boerse\.de|Handelsblatt|WirtschaftsWoche|finanzen\.net|Finanznachrichten|XTB|Lynx|T3n|ntg24|Moneycab).*$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly Regex NonAlphanumericRegex = new(@"[^\w\s]", RegexOptions.Compiled);
|
||||
private static readonly Regex MultipleSpacesRegex = new(@"\s+", RegexOptions.Compiled);
|
||||
|
||||
public ArticleDeduplicationService(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IFinlyticLogger<ArticleDeduplicationService> finlyticLogger,
|
||||
ISettingsService settingsService)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_finlyticLogger = finlyticLogger;
|
||||
_settingsService = settingsService;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_initialized) return;
|
||||
|
||||
await _initLock.WaitAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
if (_initialized) return;
|
||||
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();
|
||||
|
||||
var windowDays = await _settingsService.GetSettingAsync(SettingKeys.DeduplicationWindowDays, cancellationToken);
|
||||
var cutoff = DateTime.UtcNow.AddDays(-Math.Max(windowDays, 7));
|
||||
|
||||
var articles = await dbContext.NewsArticles
|
||||
.AsNoTracking()
|
||||
.Where(a => a.PublishedAt >= cutoff && a.Status == "Completed")
|
||||
.Select(a => new
|
||||
{
|
||||
a.Id,
|
||||
a.Title,
|
||||
a.TitleHash,
|
||||
a.SimHash,
|
||||
a.PublishedAt
|
||||
})
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
_simHashLock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
foreach (var a in articles)
|
||||
{
|
||||
var tHash = a.TitleHash ?? ComputeTitleHash(a.Title);
|
||||
var sHash = a.SimHash ?? 0L;
|
||||
var normTitle = NormalizeTitle(a.Title);
|
||||
|
||||
_titleHashIndex[tHash] = a.Id;
|
||||
|
||||
if (sHash != 0L)
|
||||
{
|
||||
_simHashIndex.Add(new CachedArticleEntry(a.Id, tHash, sHash, normTitle, a.PublishedAt));
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_simHashLock.ExitWriteLock();
|
||||
}
|
||||
|
||||
_initialized = true;
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.DeduplicationChannel, "[ArticleDeduplicationService] Initialized with {Count} recent articles for duplicate detection.", articles.Count);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public DuplicateCheckResult CheckDuplicate(string title, string content, DateTime publishedAt)
|
||||
{
|
||||
var titleHash = ComputeTitleHash(title);
|
||||
var simHash = ComputeSimHash(content);
|
||||
var normalizedTitle = NormalizeTitle(title);
|
||||
|
||||
// 1. Exact Title Hash Match
|
||||
if (_titleHashIndex.TryGetValue(titleHash, out var exactMatchId))
|
||||
{
|
||||
return new DuplicateCheckResult(
|
||||
IsDuplicate: true,
|
||||
DuplicateOfArticleId: exactMatchId,
|
||||
Reason: "ExactTitleHashMatch",
|
||||
TitleHash: titleHash,
|
||||
SimHash: simHash
|
||||
);
|
||||
}
|
||||
|
||||
_simHashLock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
var windowCutoff = publishedAt.AddDays(-7);
|
||||
|
||||
foreach (var entry in _simHashIndex)
|
||||
{
|
||||
if (entry.PublishedAt < windowCutoff) continue;
|
||||
|
||||
// 2. SimHash Content Similarity (Hamming Distance <= 3 bits -> > 90% identical text)
|
||||
if (simHash != 0L && entry.SimHash != 0L)
|
||||
{
|
||||
int distance = HammingDistance(simHash, entry.SimHash);
|
||||
if (distance <= 3)
|
||||
{
|
||||
return new DuplicateCheckResult(
|
||||
IsDuplicate: true,
|
||||
DuplicateOfArticleId: entry.ArticleId,
|
||||
Reason: $"SimHashSimilarity (HammingDistance: {distance})",
|
||||
TitleHash: titleHash,
|
||||
SimHash: simHash
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Fuzzy Title Similarity (3-gram Jaccard Index >= 0.85)
|
||||
double titleSim = Calculate3GramJaccard(normalizedTitle, entry.NormalizedTitle);
|
||||
if (titleSim >= 0.85)
|
||||
{
|
||||
return new DuplicateCheckResult(
|
||||
IsDuplicate: true,
|
||||
DuplicateOfArticleId: entry.ArticleId,
|
||||
Reason: $"FuzzyTitleSimilarity ({titleSim:P0})",
|
||||
TitleHash: titleHash,
|
||||
SimHash: simHash
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_simHashLock.ExitReadLock();
|
||||
}
|
||||
|
||||
return new DuplicateCheckResult(
|
||||
IsDuplicate: false,
|
||||
DuplicateOfArticleId: null,
|
||||
Reason: null,
|
||||
TitleHash: titleHash,
|
||||
SimHash: simHash
|
||||
);
|
||||
}
|
||||
|
||||
public void RegisterArticle(Guid articleId, string titleHash, long simHash, string title, DateTime publishedAt)
|
||||
{
|
||||
_titleHashIndex[titleHash] = articleId;
|
||||
|
||||
_simHashLock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
_simHashIndex.Add(new CachedArticleEntry(articleId, titleHash, simHash, NormalizeTitle(title), publishedAt));
|
||||
}
|
||||
finally
|
||||
{
|
||||
_simHashLock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
public string ComputeTitleHash(string title)
|
||||
{
|
||||
var normalized = NormalizeTitle(title);
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(normalized));
|
||||
return Convert.ToHexString(bytes).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public long ComputeSimHash(string content)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(content)) return 0L;
|
||||
|
||||
// Clean and tokenize content into words
|
||||
var clean = NonAlphanumericRegex.Replace(content.ToLowerInvariant(), " ");
|
||||
var words = clean.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (words.Length < 10) return 0L;
|
||||
|
||||
// Generate 3-word shingles
|
||||
var v = new int[64];
|
||||
for (int i = 0; i < words.Length - 2; i++)
|
||||
{
|
||||
var shingle = $"{words[i]} {words[i + 1]} {words[i + 2]}";
|
||||
var hash = Hash64(shingle);
|
||||
|
||||
for (int bit = 0; bit < 64; bit++)
|
||||
{
|
||||
if (((hash >> bit) & 1L) == 1L)
|
||||
{
|
||||
v[bit]++;
|
||||
}
|
||||
else
|
||||
{
|
||||
v[bit]--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
long simHash = 0L;
|
||||
for (int bit = 0; bit < 64; bit++)
|
||||
{
|
||||
if (v[bit] > 0)
|
||||
{
|
||||
simHash |= (1L << bit);
|
||||
}
|
||||
}
|
||||
|
||||
return simHash;
|
||||
}
|
||||
|
||||
private static string NormalizeTitle(string title)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(title)) return string.Empty;
|
||||
var stripped = TitlePortalSuffixRegex.Replace(title, "");
|
||||
var cleaned = NonAlphanumericRegex.Replace(stripped.ToLowerInvariant(), " ");
|
||||
return MultipleSpacesRegex.Replace(cleaned, " ").Trim();
|
||||
}
|
||||
|
||||
private static double Calculate3GramJaccard(string a, string b)
|
||||
{
|
||||
if (string.IsNullOrEmpty(a) || string.IsNullOrEmpty(b)) return 0.0;
|
||||
if (a == b) return 1.0;
|
||||
|
||||
var gramsA = Get3Grams(a);
|
||||
var gramsB = Get3Grams(b);
|
||||
|
||||
if (gramsA.Count == 0 || gramsB.Count == 0) return 0.0;
|
||||
|
||||
int intersection = 0;
|
||||
foreach (var g in gramsA)
|
||||
{
|
||||
if (gramsB.Contains(g)) intersection++;
|
||||
}
|
||||
|
||||
int union = gramsA.Count + gramsB.Count - intersection;
|
||||
return union == 0 ? 0.0 : (double)intersection / union;
|
||||
}
|
||||
|
||||
private static HashSet<string> Get3Grams(string text)
|
||||
{
|
||||
var set = new HashSet<string>(StringComparer.Ordinal);
|
||||
if (text.Length < 3)
|
||||
{
|
||||
set.Add(text);
|
||||
return set;
|
||||
}
|
||||
|
||||
for (int i = 0; i <= text.Length - 3; i++)
|
||||
{
|
||||
set.Add(text.Substring(i, 3));
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
private static int HammingDistance(long a, long b)
|
||||
{
|
||||
return BitOperations.PopCount((ulong)(a ^ b));
|
||||
}
|
||||
|
||||
private static long Hash64(string text)
|
||||
{
|
||||
// 64-bit FNV-1a Hash
|
||||
ulong hash = 14695981039346656037UL;
|
||||
var bytes = Encoding.UTF8.GetBytes(text);
|
||||
foreach (var b in bytes)
|
||||
{
|
||||
hash ^= b;
|
||||
hash *= 1099511628211UL;
|
||||
}
|
||||
return (long)hash;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user