274 lines
11 KiB
C#
274 lines
11 KiB
C#
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FinlyticCore.Models.Assets;
|
|
using FinlyticCore.Services;
|
|
using FinlyticCore.Util;
|
|
using FinlyticNews.Util;
|
|
using Microsoft.Extensions.Hosting;
|
|
|
|
namespace FinlyticNews.Services;
|
|
|
|
/// <summary>
|
|
/// Candidate asset match identified in an article.
|
|
/// </summary>
|
|
public record MatchedCandidateAsset(
|
|
string Isin,
|
|
string Name,
|
|
int MentionCount,
|
|
bool InTitle,
|
|
bool InUrl,
|
|
double InitialConfidence
|
|
);
|
|
|
|
/// <summary>
|
|
/// Service interface for fast in-memory asset detection from news text, titles, and URLs without AI.
|
|
/// </summary>
|
|
public interface IAssetMatcherService
|
|
{
|
|
/// <summary>
|
|
/// Matches financial assets in the given text, title, and URL against the in-memory asset index.
|
|
/// </summary>
|
|
List<MatchedCandidateAsset> MatchAssets(string title, string content, string? url = null);
|
|
|
|
/// <summary>
|
|
/// Forces a reload of the index.json from disk into memory.
|
|
/// </summary>
|
|
Task ReloadIndexAsync(CancellationToken cancellationToken = default);
|
|
}
|
|
|
|
/// <summary>
|
|
/// In-memory asset matching engine supporting ISIN regex, normalized name matching, legal suffix stripping, and stopword defenses.
|
|
/// </summary>
|
|
public class AssetMatcherService : IAssetMatcherService, IHostedService
|
|
{
|
|
private record CompiledNameMatcher(
|
|
AssetIndex Asset,
|
|
string CleanName,
|
|
Regex WordRegex,
|
|
bool IsShortOrCommon
|
|
);
|
|
|
|
private readonly IFinlyticLogger<AssetMatcherService> _finlyticLogger;
|
|
private readonly string _indexPath;
|
|
|
|
private readonly Dictionary<string, AssetIndex> _isinLookup = new(StringComparer.OrdinalIgnoreCase);
|
|
private readonly List<CompiledNameMatcher> _nameMatchers = [];
|
|
private readonly ReaderWriterLockSlim _indexLock = new();
|
|
|
|
private static readonly Regex IsinRegex = new(@"\b[A-Z]{2}[A-Z0-9]{9}[0-9]\b", RegexOptions.Compiled);
|
|
private static readonly Regex FinancialContextRegex = new(
|
|
@"(aktie|aktien|kurs|kurse|börse|finanz|umsatz|gewinn|quartal|prognose|dividende|kgv|analyst|kursziel|konzern|shares|stock|stocks|revenue|earnings|ebitda|ceo|cfo|nasdaq|dow|dax|s&p|\$|€)",
|
|
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
|
|
|
private static readonly Regex LegalSuffixRegex = new(
|
|
@"\s+(AG|SE|GmbH|KGaA|Inc\.?|Incorporated|Corp\.?|Corporation|Ltd\.?|Limited|PLC|Plc|SA|NV|Holdings?|Group|Co\.?|Class\s+[A-Z]|ADR|SpA|Oyj|AB|A\/S|N\.V\.|S\.A\.|S\.E\.)\b",
|
|
RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
|
|
|
private static readonly HashSet<string> CommonWordStopwords = new(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
"CAN", "IT", "ALL", "BE", "NEXT", "MAN", "GAP", "ON", "FOR", "NOW", "ONE", "TOP", "REAL", "DO", "GO", "US",
|
|
"DIE", "DER", "DAS", "UND", "VON", "MIT", "AUS", "IN", "AN", "ZU", "WAR", "AUF", "TAG", "PLUS", "FAST", "BEST",
|
|
"NEW", "AIR", "SEE", "MAX", "PRO", "NET", "HOME", "WORK", "CAR", "AUTO", "CARE", "LIFE", "WELL", "PURE", "TRUE",
|
|
"APP", "BOX", "HUB", "KEY", "PAY", "BUY", "WIN", "RUN", "SET", "GET", "RED", "BIG", "SUN", "SEA", "STAR", "BAY"
|
|
};
|
|
|
|
public AssetMatcherService(IFinlyticLogger<AssetMatcherService> finlyticLogger)
|
|
{
|
|
_finlyticLogger = finlyticLogger;
|
|
_indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json");
|
|
}
|
|
|
|
public async Task StartAsync(CancellationToken cancellationToken)
|
|
{
|
|
await ReloadIndexAsync(cancellationToken);
|
|
}
|
|
|
|
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
public async Task ReloadIndexAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
if (!File.Exists(_indexPath))
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file not found at: {Path}", _indexPath);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
await using var stream = File.OpenRead(_indexPath);
|
|
var assets = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream, cancellationToken: cancellationToken);
|
|
|
|
if (assets == null || assets.Count == 0)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file is empty.");
|
|
return;
|
|
}
|
|
|
|
_indexLock.EnterWriteLock();
|
|
try
|
|
{
|
|
_isinLookup.Clear();
|
|
_nameMatchers.Clear();
|
|
|
|
foreach (var asset in assets)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(asset.Isin)) continue;
|
|
_isinLookup[asset.Isin.Trim()] = asset;
|
|
|
|
if (!string.IsNullOrWhiteSpace(asset.Name))
|
|
{
|
|
var cleanName = CleanCompanyName(asset.Name);
|
|
if (cleanName.Length >= 2)
|
|
{
|
|
bool isShort = cleanName.Length < 4 || CommonWordStopwords.Contains(cleanName);
|
|
var pattern = $@"\b{Regex.Escape(cleanName)}\b";
|
|
var regex = new Regex(pattern, isShort ? RegexOptions.Compiled : RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
|
|
|
_nameMatchers.Add(new CompiledNameMatcher(asset, cleanName, regex, isShort));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
_indexLock.ExitWriteLock();
|
|
}
|
|
|
|
await _finlyticLogger.LogInfoAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Successfully loaded {Count} assets and {MatcherCount} name matchers into memory.", _isinLookup.Count, _nameMatchers.Count);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogErrorAsync(SettingKeys.MatcherChannel, ex, "[AssetMatcherService] Failed to load index from {Path}", _indexPath);
|
|
}
|
|
}
|
|
|
|
public List<MatchedCandidateAsset> MatchAssets(string title, string content, string? url = null)
|
|
{
|
|
var results = new Dictionary<string, MatchedCandidateAsset>(StringComparer.OrdinalIgnoreCase);
|
|
var fullText = (title + " " + content);
|
|
|
|
_indexLock.EnterReadLock();
|
|
try
|
|
{
|
|
// 1. URL Mining for ISINs (100% confidence)
|
|
if (!string.IsNullOrWhiteSpace(url))
|
|
{
|
|
var isinMatchesInUrl = IsinRegex.Matches(url);
|
|
foreach (Match m in isinMatchesInUrl)
|
|
{
|
|
if (_isinLookup.TryGetValue(m.Value, out var asset))
|
|
{
|
|
results[asset.Isin] = new MatchedCandidateAsset(
|
|
Isin: asset.Isin,
|
|
Name: asset.Name,
|
|
MentionCount: 1,
|
|
InTitle: false,
|
|
InUrl: true,
|
|
InitialConfidence: 1.0
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// 2. ISIN Regex in Title & Content
|
|
var isinMatches = IsinRegex.Matches(fullText);
|
|
foreach (Match m in isinMatches)
|
|
{
|
|
if (_isinLookup.TryGetValue(m.Value, out var asset))
|
|
{
|
|
bool inTitle = title.Contains(m.Value, StringComparison.OrdinalIgnoreCase);
|
|
int count = isinMatches.Count(x => x.Value.Equals(m.Value, StringComparison.OrdinalIgnoreCase));
|
|
|
|
results[asset.Isin] = new MatchedCandidateAsset(
|
|
Isin: asset.Isin,
|
|
Name: asset.Name,
|
|
MentionCount: count,
|
|
InTitle: inTitle,
|
|
InUrl: results.TryGetValue(asset.Isin, out var existing) && existing.InUrl,
|
|
InitialConfidence: inTitle ? 1.0 : 0.95
|
|
);
|
|
}
|
|
}
|
|
|
|
// 3. Name Matchers (Title & Content)
|
|
bool hasFinancialContext = FinancialContextRegex.IsMatch(fullText);
|
|
|
|
foreach (var matcher in _nameMatchers)
|
|
{
|
|
if (results.ContainsKey(matcher.Asset.Isin)) continue;
|
|
|
|
// Check Title First
|
|
bool titleMatch = matcher.WordRegex.IsMatch(title);
|
|
if (titleMatch)
|
|
{
|
|
// If short/common, ensure financial context exists
|
|
if (!matcher.IsShortOrCommon || hasFinancialContext)
|
|
{
|
|
results[matcher.Asset.Isin] = new MatchedCandidateAsset(
|
|
Isin: matcher.Asset.Isin,
|
|
Name: matcher.Asset.Name,
|
|
MentionCount: 1,
|
|
InTitle: true,
|
|
InUrl: false,
|
|
InitialConfidence: 0.9
|
|
);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Check Content
|
|
var contentMatches = matcher.WordRegex.Matches(content);
|
|
if (contentMatches.Count > 0)
|
|
{
|
|
if (matcher.IsShortOrCommon)
|
|
{
|
|
// Short words require >= 2 mentions AND financial context
|
|
if (contentMatches.Count >= 2 && hasFinancialContext)
|
|
{
|
|
results[matcher.Asset.Isin] = new MatchedCandidateAsset(
|
|
Isin: matcher.Asset.Isin,
|
|
Name: matcher.Asset.Name,
|
|
MentionCount: contentMatches.Count,
|
|
InTitle: false,
|
|
InUrl: false,
|
|
InitialConfidence: 0.75
|
|
);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
results[matcher.Asset.Isin] = new MatchedCandidateAsset(
|
|
Isin: matcher.Asset.Isin,
|
|
Name: matcher.Asset.Name,
|
|
MentionCount: contentMatches.Count,
|
|
InTitle: false,
|
|
InUrl: false,
|
|
InitialConfidence: contentMatches.Count >= 2 ? 0.85 : 0.7
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
_indexLock.ExitReadLock();
|
|
}
|
|
|
|
return results.Values.OrderByDescending(r => r.InitialConfidence).ThenByDescending(r => r.MentionCount).ToList();
|
|
}
|
|
|
|
private static string CleanCompanyName(string name)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(name)) return string.Empty;
|
|
var stripped = LegalSuffixRegex.Replace(name, "").Trim();
|
|
return stripped;
|
|
}
|
|
}
|