using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Models.Assets; using FinlyticCore.Services; using FinlyticCore.Util; using FinlyticNews.Util; using Microsoft.Extensions.Hosting; namespace FinlyticNews.Services; /// /// Candidate asset match identified in an article. /// public record MatchedCandidateAsset( string Isin, string Name, int MentionCount, bool InTitle, bool InUrl, double InitialConfidence ); /// /// Service interface for fast in-memory asset detection from news text, titles, and URLs without AI. /// public interface IAssetMatcherService { /// /// Matches financial assets in the given text, title, and URL against the in-memory asset index. /// List MatchAssets(string title, string content, string? url = null); /// /// Forces a reload of the index.json from disk into memory. /// Task ReloadIndexAsync(CancellationToken cancellationToken = default); } /// /// In-memory asset matching engine supporting ISIN regex, normalized name matching, legal suffix stripping, and stopword defenses. /// public class AssetMatcherService : IAssetMatcherService, IHostedService { private record CompiledNameMatcher( AssetIndex Asset, string CleanName, Regex WordRegex, bool IsShortOrCommon ); private readonly IFinlyticLogger _finlyticLogger; private readonly string _indexPath; private readonly Dictionary _isinLookup = new(StringComparer.OrdinalIgnoreCase); private readonly List _nameMatchers = []; private readonly ReaderWriterLockSlim _indexLock = new(); private static readonly Regex IsinRegex = new(@"\b[A-Z]{2}[A-Z0-9]{9}[0-9]\b", RegexOptions.Compiled); private static readonly Regex FinancialContextRegex = new( @"(aktie|aktien|kurs|kurse|börse|finanz|umsatz|gewinn|quartal|prognose|dividende|kgv|analyst|kursziel|konzern|shares|stock|stocks|revenue|earnings|ebitda|ceo|cfo|nasdaq|dow|dax|s&p|\$|€)", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex LegalSuffixRegex = new( @"\s+(AG|SE|GmbH|KGaA|Inc\.?|Incorporated|Corp\.?|Corporation|Ltd\.?|Limited|PLC|Plc|SA|NV|Holdings?|Group|Co\.?|Class\s+[A-Z]|ADR|SpA|Oyj|AB|A\/S|N\.V\.|S\.A\.|S\.E\.)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly HashSet CommonWordStopwords = new(StringComparer.OrdinalIgnoreCase) { "CAN", "IT", "ALL", "BE", "NEXT", "MAN", "GAP", "ON", "FOR", "NOW", "ONE", "TOP", "REAL", "DO", "GO", "US", "DIE", "DER", "DAS", "UND", "VON", "MIT", "AUS", "IN", "AN", "ZU", "WAR", "AUF", "TAG", "PLUS", "FAST", "BEST", "NEW", "AIR", "SEE", "MAX", "PRO", "NET", "HOME", "WORK", "CAR", "AUTO", "CARE", "LIFE", "WELL", "PURE", "TRUE", "APP", "BOX", "HUB", "KEY", "PAY", "BUY", "WIN", "RUN", "SET", "GET", "RED", "BIG", "SUN", "SEA", "STAR", "BAY" }; public AssetMatcherService(IFinlyticLogger finlyticLogger) { _finlyticLogger = finlyticLogger; _indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json"); } public async Task StartAsync(CancellationToken cancellationToken) { await ReloadIndexAsync(cancellationToken); } public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; public async Task ReloadIndexAsync(CancellationToken cancellationToken = default) { if (!File.Exists(_indexPath)) { await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file not found at: {Path}", _indexPath); return; } try { await using var stream = File.OpenRead(_indexPath); var assets = await JsonSerializer.DeserializeAsync>(stream, cancellationToken: cancellationToken); if (assets == null || assets.Count == 0) { await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file is empty."); return; } _indexLock.EnterWriteLock(); try { _isinLookup.Clear(); _nameMatchers.Clear(); foreach (var asset in assets) { if (string.IsNullOrWhiteSpace(asset.Isin)) continue; _isinLookup[asset.Isin.Trim()] = asset; if (!string.IsNullOrWhiteSpace(asset.Name)) { var cleanName = CleanCompanyName(asset.Name); if (cleanName.Length >= 2) { bool isShort = cleanName.Length < 4 || CommonWordStopwords.Contains(cleanName); var pattern = $@"\b{Regex.Escape(cleanName)}\b"; var regex = new Regex(pattern, isShort ? RegexOptions.Compiled : RegexOptions.Compiled | RegexOptions.IgnoreCase); _nameMatchers.Add(new CompiledNameMatcher(asset, cleanName, regex, isShort)); } } } } finally { _indexLock.ExitWriteLock(); } await _finlyticLogger.LogInfoAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Successfully loaded {Count} assets and {MatcherCount} name matchers into memory.", _isinLookup.Count, _nameMatchers.Count); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.MatcherChannel, ex, "[AssetMatcherService] Failed to load index from {Path}", _indexPath); } } public List MatchAssets(string title, string content, string? url = null) { var results = new Dictionary(StringComparer.OrdinalIgnoreCase); var fullText = (title + " " + content); _indexLock.EnterReadLock(); try { // 1. URL Mining for ISINs (100% confidence) if (!string.IsNullOrWhiteSpace(url)) { var isinMatchesInUrl = IsinRegex.Matches(url); foreach (Match m in isinMatchesInUrl) { if (_isinLookup.TryGetValue(m.Value, out var asset)) { results[asset.Isin] = new MatchedCandidateAsset( Isin: asset.Isin, Name: asset.Name, MentionCount: 1, InTitle: false, InUrl: true, InitialConfidence: 1.0 ); } } } // 2. ISIN Regex in Title & Content var isinMatches = IsinRegex.Matches(fullText); foreach (Match m in isinMatches) { if (_isinLookup.TryGetValue(m.Value, out var asset)) { bool inTitle = title.Contains(m.Value, StringComparison.OrdinalIgnoreCase); int count = isinMatches.Count(x => x.Value.Equals(m.Value, StringComparison.OrdinalIgnoreCase)); results[asset.Isin] = new MatchedCandidateAsset( Isin: asset.Isin, Name: asset.Name, MentionCount: count, InTitle: inTitle, InUrl: results.TryGetValue(asset.Isin, out var existing) && existing.InUrl, InitialConfidence: inTitle ? 1.0 : 0.95 ); } } // 3. Name Matchers (Title & Content) bool hasFinancialContext = FinancialContextRegex.IsMatch(fullText); foreach (var matcher in _nameMatchers) { if (results.ContainsKey(matcher.Asset.Isin)) continue; // Check Title First bool titleMatch = matcher.WordRegex.IsMatch(title); if (titleMatch) { // If short/common, ensure financial context exists if (!matcher.IsShortOrCommon || hasFinancialContext) { results[matcher.Asset.Isin] = new MatchedCandidateAsset( Isin: matcher.Asset.Isin, Name: matcher.Asset.Name, MentionCount: 1, InTitle: true, InUrl: false, InitialConfidence: 0.9 ); continue; } } // Check Content var contentMatches = matcher.WordRegex.Matches(content); if (contentMatches.Count > 0) { if (matcher.IsShortOrCommon) { // Short words require >= 2 mentions AND financial context if (contentMatches.Count >= 2 && hasFinancialContext) { results[matcher.Asset.Isin] = new MatchedCandidateAsset( Isin: matcher.Asset.Isin, Name: matcher.Asset.Name, MentionCount: contentMatches.Count, InTitle: false, InUrl: false, InitialConfidence: 0.75 ); } } else { results[matcher.Asset.Isin] = new MatchedCandidateAsset( Isin: matcher.Asset.Isin, Name: matcher.Asset.Name, MentionCount: contentMatches.Count, InTitle: false, InUrl: false, InitialConfidence: contentMatches.Count >= 2 ? 0.85 : 0.7 ); } } } } finally { _indexLock.ExitReadLock(); } return results.Values.OrderByDescending(r => r.InitialConfidence).ThenByDescending(r => r.MentionCount).ToList(); } private static string CleanCompanyName(string name) { if (string.IsNullOrWhiteSpace(name)) return string.Empty; var stripped = LegalSuffixRegex.Replace(name, "").Trim(); return stripped; } }