feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Models.Assets;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticCore.Util;
|
||||
using FinlyticNews.Util;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace FinlyticNews.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Candidate asset match identified in an article.
|
||||
/// </summary>
|
||||
public record MatchedCandidateAsset(
|
||||
string Isin,
|
||||
string Name,
|
||||
int MentionCount,
|
||||
bool InTitle,
|
||||
bool InUrl,
|
||||
double InitialConfidence
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Service interface for fast in-memory asset detection from news text, titles, and URLs without AI.
|
||||
/// </summary>
|
||||
public interface IAssetMatcherService
|
||||
{
|
||||
/// <summary>
|
||||
/// Matches financial assets in the given text, title, and URL against the in-memory asset index.
|
||||
/// </summary>
|
||||
List<MatchedCandidateAsset> MatchAssets(string title, string content, string? url = null);
|
||||
|
||||
/// <summary>
|
||||
/// Forces a reload of the index.json from disk into memory.
|
||||
/// </summary>
|
||||
Task ReloadIndexAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory asset matching engine supporting ISIN regex, normalized name matching, legal suffix stripping, and stopword defenses.
|
||||
/// </summary>
|
||||
public class AssetMatcherService : IAssetMatcherService, IHostedService
|
||||
{
|
||||
private record CompiledNameMatcher(
|
||||
AssetIndex Asset,
|
||||
string CleanName,
|
||||
Regex WordRegex,
|
||||
bool IsShortOrCommon
|
||||
);
|
||||
|
||||
private readonly IFinlyticLogger<AssetMatcherService> _finlyticLogger;
|
||||
private readonly string _indexPath;
|
||||
|
||||
private readonly Dictionary<string, AssetIndex> _isinLookup = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly List<CompiledNameMatcher> _nameMatchers = [];
|
||||
private readonly ReaderWriterLockSlim _indexLock = new();
|
||||
|
||||
private static readonly Regex IsinRegex = new(@"\b[A-Z]{2}[A-Z0-9]{9}[0-9]\b", RegexOptions.Compiled);
|
||||
private static readonly Regex FinancialContextRegex = new(
|
||||
@"(aktie|aktien|kurs|kurse|börse|finanz|umsatz|gewinn|quartal|prognose|dividende|kgv|analyst|kursziel|konzern|shares|stock|stocks|revenue|earnings|ebitda|ceo|cfo|nasdaq|dow|dax|s&p|\$|€)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly Regex LegalSuffixRegex = new(
|
||||
@"\s+(AG|SE|GmbH|KGaA|Inc\.?|Incorporated|Corp\.?|Corporation|Ltd\.?|Limited|PLC|Plc|SA|NV|Holdings?|Group|Co\.?|Class\s+[A-Z]|ADR|SpA|Oyj|AB|A\/S|N\.V\.|S\.A\.|S\.E\.)\b",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
private static readonly HashSet<string> CommonWordStopwords = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"CAN", "IT", "ALL", "BE", "NEXT", "MAN", "GAP", "ON", "FOR", "NOW", "ONE", "TOP", "REAL", "DO", "GO", "US",
|
||||
"DIE", "DER", "DAS", "UND", "VON", "MIT", "AUS", "IN", "AN", "ZU", "WAR", "AUF", "TAG", "PLUS", "FAST", "BEST",
|
||||
"NEW", "AIR", "SEE", "MAX", "PRO", "NET", "HOME", "WORK", "CAR", "AUTO", "CARE", "LIFE", "WELL", "PURE", "TRUE",
|
||||
"APP", "BOX", "HUB", "KEY", "PAY", "BUY", "WIN", "RUN", "SET", "GET", "RED", "BIG", "SUN", "SEA", "STAR", "BAY"
|
||||
};
|
||||
|
||||
public AssetMatcherService(IFinlyticLogger<AssetMatcherService> finlyticLogger)
|
||||
{
|
||||
_finlyticLogger = finlyticLogger;
|
||||
_indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json");
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await ReloadIndexAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async Task ReloadIndexAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!File.Exists(_indexPath))
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file not found at: {Path}", _indexPath);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await using var stream = File.OpenRead(_indexPath);
|
||||
var assets = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream, cancellationToken: cancellationToken);
|
||||
|
||||
if (assets == null || assets.Count == 0)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file is empty.");
|
||||
return;
|
||||
}
|
||||
|
||||
_indexLock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
_isinLookup.Clear();
|
||||
_nameMatchers.Clear();
|
||||
|
||||
foreach (var asset in assets)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(asset.Isin)) continue;
|
||||
_isinLookup[asset.Isin.Trim()] = asset;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(asset.Name))
|
||||
{
|
||||
var cleanName = CleanCompanyName(asset.Name);
|
||||
if (cleanName.Length >= 2)
|
||||
{
|
||||
bool isShort = cleanName.Length < 4 || CommonWordStopwords.Contains(cleanName);
|
||||
var pattern = $@"\b{Regex.Escape(cleanName)}\b";
|
||||
var regex = new Regex(pattern, isShort ? RegexOptions.Compiled : RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
|
||||
_nameMatchers.Add(new CompiledNameMatcher(asset, cleanName, regex, isShort));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_indexLock.ExitWriteLock();
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Successfully loaded {Count} assets and {MatcherCount} name matchers into memory.", _isinLookup.Count, _nameMatchers.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.MatcherChannel, ex, "[AssetMatcherService] Failed to load index from {Path}", _indexPath);
|
||||
}
|
||||
}
|
||||
|
||||
public List<MatchedCandidateAsset> MatchAssets(string title, string content, string? url = null)
|
||||
{
|
||||
var results = new Dictionary<string, MatchedCandidateAsset>(StringComparer.OrdinalIgnoreCase);
|
||||
var fullText = (title + " " + content);
|
||||
|
||||
_indexLock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
// 1. URL Mining for ISINs (100% confidence)
|
||||
if (!string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
var isinMatchesInUrl = IsinRegex.Matches(url);
|
||||
foreach (Match m in isinMatchesInUrl)
|
||||
{
|
||||
if (_isinLookup.TryGetValue(m.Value, out var asset))
|
||||
{
|
||||
results[asset.Isin] = new MatchedCandidateAsset(
|
||||
Isin: asset.Isin,
|
||||
Name: asset.Name,
|
||||
MentionCount: 1,
|
||||
InTitle: false,
|
||||
InUrl: true,
|
||||
InitialConfidence: 1.0
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. ISIN Regex in Title & Content
|
||||
var isinMatches = IsinRegex.Matches(fullText);
|
||||
foreach (Match m in isinMatches)
|
||||
{
|
||||
if (_isinLookup.TryGetValue(m.Value, out var asset))
|
||||
{
|
||||
bool inTitle = title.Contains(m.Value, StringComparison.OrdinalIgnoreCase);
|
||||
int count = isinMatches.Count(x => x.Value.Equals(m.Value, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
results[asset.Isin] = new MatchedCandidateAsset(
|
||||
Isin: asset.Isin,
|
||||
Name: asset.Name,
|
||||
MentionCount: count,
|
||||
InTitle: inTitle,
|
||||
InUrl: results.TryGetValue(asset.Isin, out var existing) && existing.InUrl,
|
||||
InitialConfidence: inTitle ? 1.0 : 0.95
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Name Matchers (Title & Content)
|
||||
bool hasFinancialContext = FinancialContextRegex.IsMatch(fullText);
|
||||
|
||||
foreach (var matcher in _nameMatchers)
|
||||
{
|
||||
if (results.ContainsKey(matcher.Asset.Isin)) continue;
|
||||
|
||||
// Check Title First
|
||||
bool titleMatch = matcher.WordRegex.IsMatch(title);
|
||||
if (titleMatch)
|
||||
{
|
||||
// If short/common, ensure financial context exists
|
||||
if (!matcher.IsShortOrCommon || hasFinancialContext)
|
||||
{
|
||||
results[matcher.Asset.Isin] = new MatchedCandidateAsset(
|
||||
Isin: matcher.Asset.Isin,
|
||||
Name: matcher.Asset.Name,
|
||||
MentionCount: 1,
|
||||
InTitle: true,
|
||||
InUrl: false,
|
||||
InitialConfidence: 0.9
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Check Content
|
||||
var contentMatches = matcher.WordRegex.Matches(content);
|
||||
if (contentMatches.Count > 0)
|
||||
{
|
||||
if (matcher.IsShortOrCommon)
|
||||
{
|
||||
// Short words require >= 2 mentions AND financial context
|
||||
if (contentMatches.Count >= 2 && hasFinancialContext)
|
||||
{
|
||||
results[matcher.Asset.Isin] = new MatchedCandidateAsset(
|
||||
Isin: matcher.Asset.Isin,
|
||||
Name: matcher.Asset.Name,
|
||||
MentionCount: contentMatches.Count,
|
||||
InTitle: false,
|
||||
InUrl: false,
|
||||
InitialConfidence: 0.75
|
||||
);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
results[matcher.Asset.Isin] = new MatchedCandidateAsset(
|
||||
Isin: matcher.Asset.Isin,
|
||||
Name: matcher.Asset.Name,
|
||||
MentionCount: contentMatches.Count,
|
||||
InTitle: false,
|
||||
InUrl: false,
|
||||
InitialConfidence: contentMatches.Count >= 2 ? 0.85 : 0.7
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_indexLock.ExitReadLock();
|
||||
}
|
||||
|
||||
return results.Values.OrderByDescending(r => r.InitialConfidence).ThenByDescending(r => r.MentionCount).ToList();
|
||||
}
|
||||
|
||||
private static string CleanCompanyName(string name)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name)) return string.Empty;
|
||||
var stripped = LegalSuffixRegex.Replace(name, "").Trim();
|
||||
return stripped;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user