using System.Collections.Concurrent; using System.Text.Encodings.Web; using System.Text.Json; using FinlyticCore.Dtos.News; using FinlyticCore.Dtos.Sentiment; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; namespace FinlyticSentiment.Services; /// /// Defines the persistence contract for maintaining two-stage ISIN and Sector JSON sentiment summaries in the file system. /// public interface ISentimentStorageService { /// /// Appends a new FinBERT analysis event to the ISIN summary file and recalculates the current aggregate metrics. /// /// The asset ISIN code. /// The name of the asset company. /// The sector associated with the asset. /// The analyzed article DTO. /// The FinBERT sentiment result. /// A task representing the file update operation. Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert); /// /// Appends a new FinBERT analysis event to the Sector summary file and recalculates the sector aggregate metrics. /// /// The target sector name. /// The asset ISIN code triggering the sector update. /// The unique news article identifier. /// The FinBERT sentiment result. /// A task representing the file update operation. Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert); /// /// Persists an article's sentiment analysis directly in data/summaries/articles/{articleId}.json. /// Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert); /// /// Retrieves the sentiment analysis entry for a specific news article. /// Task GetArticleSentimentAsync(string articleId); /// /// Retrieves the aggregate sentiment summary for a specific asset ISIN. /// Task GetIsinSummaryAsync(string isin); } public class SentimentStorageService : ISentimentStorageService { private static readonly ConcurrentDictionary FileLocks = new(); private readonly ILogger _logger; private readonly string _basePath; private readonly JsonSerializerOptions _jsonOptions; public SentimentStorageService(IConfiguration configuration, ILogger logger) { _logger = logger; _basePath = configuration["Storage:SummariesPath"] ?? "data/summaries"; Directory.CreateDirectory(Path.Combine(_basePath, "isin")); Directory.CreateDirectory(Path.Combine(_basePath, "sectors")); Directory.CreateDirectory(Path.Combine(_basePath, "articles")); _jsonOptions = new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; } /// public async Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert) { if (article == null || article.Id == Guid.Empty) return; string cleanId = article.Id.ToString(); string filePath = Path.Combine(_basePath, "articles", $"{cleanId}.json"); var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); string analysisId = $"sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}"; var entry = new IsinAnalysisEntry { AnalysisId = analysisId, Timestamp = nowIso, Article = new IsinAnalysisArticleRef { ArticleId = cleanId, Title = article.Title ?? "", Source = article.Author ?? "FinlyticNews", PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ") }, FinbertResult = finbert, SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? article.Title ?? "" }; string json = JsonSerializer.Serialize(entry, _jsonOptions); await File.WriteAllTextAsync(filePath, json); _logger.LogInformation("[{Channel}] Successfully saved article sentiment file: {Path}", "SentimentChannel", filePath); } catch (Exception ex) { _logger.LogError(ex, "[{Channel}] Failed to write article sentiment file: {Path}", "SentimentChannel", filePath); } finally { fileLock.Release(); } } /// public async Task GetArticleSentimentAsync(string articleId) { if (string.IsNullOrWhiteSpace(articleId)) return null; var cleanId = articleId.Trim(); var articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json"); // 1. Primärer Lookup if (File.Exists(articleFilePath)) { var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { var json = await File.ReadAllTextAsync(articleFilePath); return JsonSerializer.Deserialize(json, _jsonOptions); } catch (Exception ex) { _logger.LogWarning(ex, "[{Channel}] Failed to read article sentiment file: {Path}", "SentimentChannel", articleFilePath); } finally { fileLock.Release(); } } // 2. Fallback in ISIN-Dateien var dirPath = Path.Combine(_basePath, "isin"); if (!Directory.Exists(dirPath)) return null; foreach (var file in Directory.GetFiles(dirPath, "*.json")) { try { var json = await File.ReadAllTextAsync(file); var doc = JsonSerializer.Deserialize(json, _jsonOptions); var match = doc?.Analyses?.FirstOrDefault(a => string.Equals(a.Article?.ArticleId?.Trim(), cleanId, StringComparison.OrdinalIgnoreCase)); if (match != null) return match; } catch { } } return null; } /// public async Task GetIsinSummaryAsync(string isin) { if (string.IsNullOrWhiteSpace(isin)) return null; var cleanIsin = isin.Trim(); var filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json"); if (!File.Exists(filePath)) return null; var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { var json = await File.ReadAllTextAsync(filePath); return JsonSerializer.Deserialize(json, _jsonOptions); } catch (Exception ex) { _logger.LogError(ex, "[{Channel}] Error reading ISIN summary file: {Path}", "SentimentChannel", filePath); return null; } finally { fileLock.Release(); } } /// public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert) { if (string.IsNullOrWhiteSpace(isin) || article == null) return; string cleanIsin = isin.Trim(); string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json"); var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { IsinSentimentSummaryDto isinDoc; if (File.Exists(filePath)) { try { string json = await File.ReadAllTextAsync(filePath); isinDoc = JsonSerializer.Deserialize(json, _jsonOptions) ?? new IsinSentimentSummaryDto { Isin = cleanIsin }; } catch { isinDoc = new IsinSentimentSummaryDto { Isin = cleanIsin }; } } else { isinDoc = new IsinSentimentSummaryDto { Isin = cleanIsin }; } string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); string analysisId = $"sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}"; var newEntry = new IsinAnalysisEntry { AnalysisId = analysisId, Timestamp = nowIso, Article = new IsinAnalysisArticleRef { ArticleId = article.Id.ToString(), Title = article.Title ?? "", Source = article.Author ?? "FinlyticNews", PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ") }, FinbertResult = finbert, SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? article.Title ?? "" }; // Duplikat-Bereinigung: Falls Artikel bereits existiert, alten Eintrag entfernen! var updatedAnalyses = isinDoc.Analyses? .Where(a => !string.Equals(a.Article?.ArticleId, article.Id.ToString(), StringComparison.OrdinalIgnoreCase)) .ToList() ?? new List(); // Neuen Eintrag oben einfügen updatedAnalyses.Insert(0, newEntry); // Capping: Maximal die letzten 100 Analysen aufheben (verhindert gigantische JSON-Dateien) if (updatedAnalyses.Count > 100) { updatedAnalyses = updatedAnalyses.Take(100).ToList(); } double avgCompound = updatedAnalyses.Average(a => a.FinbertResult.CompoundScore); double avgConf = updatedAnalyses.Average(a => a.FinbertResult.Confidence); string label = CalculateLabel(avgCompound); string textSummary = label switch { "POSITIVE" => $"Die Stimmungsanalyse zeigt einen weiterhin positiven Trend ({avgCompound:F2}). Hauptursache sind positive Berichte und starke Markt-Signale.", "NEGATIVE" => $"Die Stimmungsanalyse deutet auf einen verhaltenen bis negativen Trend hin ({avgCompound:F2}). Auf kritische Markt-Berichte sollte geachtet werden.", _ => $"Das Gesamtsentiment ist neutral ({avgCompound:F2}). Ausgewogene Signale aus der aktuellen Berichterstattung." }; var updatedDoc = isinDoc with { Isin = cleanIsin, CompanyName = !string.IsNullOrWhiteSpace(companyName) ? companyName : isinDoc.CompanyName, Sector = !string.IsNullOrWhiteSpace(sector) ? sector : isinDoc.Sector, LastUpdated = nowIso, CurrentSummary = new IsinCurrentSummary { CompoundScore = Math.Round(avgCompound, 2), SentimentLabel = label, AvgConfidence = Math.Round(avgConf, 2), TotalArticlesAnalyzed = updatedAnalyses.Count, Text = textSummary }, Analyses = updatedAnalyses }; await File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(updatedDoc, _jsonOptions)); _logger.LogInformation("[{Channel}] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", "SentimentChannel", filePath, updatedAnalyses.Count, avgCompound); } finally { fileLock.Release(); } } /// public async Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert) { if (string.IsNullOrWhiteSpace(sector)) return; string sanitizedSector = string.Concat(sector.Split(Path.GetInvalidFileNameChars())).Trim(); string filePath = Path.Combine(_basePath, "sectors", $"{sanitizedSector}.json"); var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { SectorSentimentSummaryDto sectorDoc; if (File.Exists(filePath)) { try { string json = await File.ReadAllTextAsync(filePath); sectorDoc = JsonSerializer.Deserialize(json, _jsonOptions) ?? new SectorSentimentSummaryDto { Sector = sector }; } catch { sectorDoc = new SectorSentimentSummaryDto { Sector = sector }; } } else { sectorDoc = new SectorSentimentSummaryDto { Sector = sector }; } string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); string analysisId = $"sec_sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}"; var newEntry = new SectorAnalysisEntry { AnalysisId = analysisId, Timestamp = nowIso, RelatedIsin = isin, ArticleId = articleId, FinbertResult = finbert }; // Duplikate bereinigen (selber Artikel für denselben Sektor) var updatedAnalyses = sectorDoc.Analyses? .Where(a => !string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase)) .ToList() ?? new List(); updatedAnalyses.Insert(0, newEntry); if (updatedAnalyses.Count > 100) { updatedAnalyses = updatedAnalyses.Take(100).ToList(); } var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrWhiteSpace(i)).Distinct().ToList(); double avgCompound = updatedAnalyses.Average(a => a.FinbertResult.CompoundScore); string label = CalculateLabel(avgCompound); string overviewText = label switch { "POSITIVE" => $"Der Sektor {sector} tendiert insgesamt positiv. Starke Einzelergebnisse stützen den Trend.", "NEGATIVE" => $"Der Sektor {sector} verzeichnet dämpfende Sentiment-Signale.", _ => $"Der Sektor {sector} zeigt ein ausgewogenes neutrales Gesamtbild." }; var updatedDoc = sectorDoc with { Sector = sector, LastUpdated = nowIso, CurrentSummary = new SectorCurrentSummary { CompoundScore = Math.Round(avgCompound, 2), SentimentLabel = label, ActiveIsins = activeIsins, Text = overviewText }, Analyses = updatedAnalyses }; await File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(updatedDoc, _jsonOptions)); _logger.LogInformation("[{Channel}] Updated Sector summary file: {Path} (Active ISINs: {Count})", "SentimentChannel", filePath, activeIsins.Count); } finally { fileLock.Release(); } } private static string CalculateLabel(double score) => score switch { >= 0.15 => "POSITIVE", <= -0.15 => "NEGATIVE", _ => "NEUTRAL" }; }