using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Encodings.Web; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.News; using FinlyticCore.Dtos.Sentiment; using FinlyticCore.Services; using FinlyticSentiment.Util; using Microsoft.Extensions.Configuration; namespace FinlyticSentiment.Services; /// /// Defines the persistence contract for maintaining two-stage ISIN and Sector JSON sentiment summaries in the file system. /// public interface ISentimentStorageService { Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert); Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert); Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert); Task GetArticleSentimentAsync(string articleId); Task GetIsinSummaryAsync(string isin); } public class SentimentStorageService : ISentimentStorageService { private static readonly ConcurrentDictionary FileLocks = new(); private readonly IFinlyticLogger _finlyticLogger; private readonly string _basePath; private readonly JsonSerializerOptions _jsonOptions; public SentimentStorageService(IConfiguration configuration, IFinlyticLogger finlyticLogger) { _finlyticLogger = finlyticLogger; _basePath = configuration["Storage:SummariesPath"] ?? "data/summaries"; Directory.CreateDirectory(Path.Combine(_basePath, "isin")); Directory.CreateDirectory(Path.Combine(_basePath, "sectors")); Directory.CreateDirectory(Path.Combine(_basePath, "articles")); _jsonOptions = new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; } /// public async Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert) { if (article == null || article.Id == Guid.Empty) return; string cleanId = article.Id.ToString(); string filePath = Path.Combine(_basePath, "articles", $"{cleanId}.json"); var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); var entry = new IsinAnalysisEntry { AnalysisId = $"sent_{Guid.NewGuid():N}", Timestamp = nowIso, Article = new IsinAnalysisArticleRef { ArticleId = cleanId, Title = article.Title, PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"), Source = article.Author ?? "FinlyticNews" }, FinbertResult = finbert, SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty }; var json = JsonSerializer.Serialize(entry, _jsonOptions); await File.WriteAllTextAsync(filePath, json); await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Successfully saved article sentiment file: {Path}", filePath); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to write article sentiment file: {Path}", filePath); } finally { fileLock.Release(); } } /// public async Task GetArticleSentimentAsync(string articleId) { if (string.IsNullOrWhiteSpace(articleId)) return null; string cleanId = articleId.Trim(); string articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json"); if (File.Exists(articleFilePath)) { var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { var json = await File.ReadAllTextAsync(articleFilePath); return JsonSerializer.Deserialize(json, _jsonOptions); } catch (Exception ex) { await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to read article sentiment file: {Path}", articleFilePath); } finally { fileLock.Release(); } } return null; } /// public async Task GetIsinSummaryAsync(string isin) { if (string.IsNullOrWhiteSpace(isin)) return null; string cleanIsin = isin.Trim().ToUpperInvariant(); string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json"); if (!File.Exists(filePath)) return null; var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { var json = await File.ReadAllTextAsync(filePath); return JsonSerializer.Deserialize(json, _jsonOptions); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Error reading ISIN summary file: {Path}", filePath); return null; } finally { fileLock.Release(); } } /// public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert) { if (string.IsNullOrWhiteSpace(isin)) return; string cleanIsin = isin.Trim().ToUpperInvariant(); string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json"); var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { var analyses = new List(); if (File.Exists(filePath)) { var existingJson = await File.ReadAllTextAsync(filePath); var existing = JsonSerializer.Deserialize(existingJson, _jsonOptions); if (existing?.Analyses != null) { analyses.AddRange(existing.Analyses); } } string cleanArticleId = article.Id.ToString(); analyses.RemoveAll(a => string.Equals(a.Article?.ArticleId, cleanArticleId, StringComparison.OrdinalIgnoreCase)); string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); var newEntry = new IsinAnalysisEntry { AnalysisId = $"sent_{Guid.NewGuid():N}", Timestamp = nowIso, Article = new IsinAnalysisArticleRef { ArticleId = cleanArticleId, Title = article.Title, PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"), Source = article.Author ?? "FinlyticNews" }, FinbertResult = finbert, SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty }; analyses.Add(newEntry); var cutoff = DateTime.UtcNow.AddDays(-14); var validAnalyses = analyses.Where(a => { if (DateTime.TryParse(a.Article?.PublishedAt ?? a.Timestamp, out var pubDate)) { return pubDate >= cutoff; } return true; }).ToList(); var updatedAnalyses = validAnalyses.OrderByDescending(a => a.Article?.PublishedAt ?? a.Timestamp).Take(50).ToList(); double totalCompound = 0.0; double totalConf = 0.0; foreach (var item in updatedAnalyses) { if (item.FinbertResult == null) continue; totalCompound += item.FinbertResult.CompoundScore; totalConf += item.FinbertResult.Confidence; } int total = updatedAnalyses.Count; double avgCompound = total > 0 ? totalCompound / total : 0.0; double avgConf = total > 0 ? totalConf / total : 0.0; string overallLabel = "NEUTRAL"; if (avgCompound >= 0.15) overallLabel = "POSITIVE"; else if (avgCompound <= -0.15) overallLabel = "NEGATIVE"; var summary = new IsinSentimentSummaryDto { Isin = cleanIsin, CompanyName = companyName, Sector = sector, LastUpdated = nowIso, CurrentSummary = new IsinCurrentSummary { CompoundScore = Math.Round(avgCompound, 4), SentimentLabel = overallLabel, AvgConfidence = Math.Round(avgConf, 4), TotalArticlesAnalyzed = total, Text = $"Synthesized sentiment across {total} articles is {overallLabel}." }, Analyses = updatedAnalyses }; var outJson = JsonSerializer.Serialize(summary, _jsonOptions); await File.WriteAllTextAsync(filePath, outJson); await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", filePath, updatedAnalyses.Count, avgCompound); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update ISIN summary for {Isin}", cleanIsin); } finally { fileLock.Release(); } } /// public async Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert) { if (string.IsNullOrWhiteSpace(sector)) return; string cleanSector = sector.Trim().ToLowerInvariant(); string filePath = Path.Combine(_basePath, "sectors", $"{cleanSector}.json"); var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1)); await fileLock.WaitAsync(); try { var analyses = new List(); if (File.Exists(filePath)) { var existingJson = await File.ReadAllTextAsync(filePath); var existing = JsonSerializer.Deserialize(existingJson, _jsonOptions); if (existing?.Analyses != null) { analyses.AddRange(existing.Analyses); } } string cleanIsin = isin.Trim().ToUpperInvariant(); string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ"); analyses.RemoveAll(a => string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase) && string.Equals(a.RelatedIsin, cleanIsin, StringComparison.OrdinalIgnoreCase)); analyses.Add(new SectorAnalysisEntry { AnalysisId = $"sec_{Guid.NewGuid():N}", Timestamp = nowIso, RelatedIsin = cleanIsin, ArticleId = articleId, FinbertResult = finbert }); var cutoff = DateTime.UtcNow.AddDays(-14); var updatedAnalyses = analyses.Where(a => { if (DateTime.TryParse(a.Timestamp, out var ts)) { return ts >= cutoff; } return true; }).OrderByDescending(a => a.Timestamp).Take(100).ToList(); var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrEmpty(i)).Distinct().ToList(); double totalSectorCompound = updatedAnalyses.Sum(s => s.FinbertResult.CompoundScore); double avgSectorCompound = updatedAnalyses.Count > 0 ? totalSectorCompound / updatedAnalyses.Count : 0.0; string sectorLabel = "NEUTRAL"; if (avgSectorCompound >= 0.15) sectorLabel = "POSITIVE"; else if (avgSectorCompound <= -0.15) sectorLabel = "NEGATIVE"; var summary = new SectorSentimentSummaryDto { Sector = sector, LastUpdated = nowIso, CurrentSummary = new SectorCurrentSummary { CompoundScore = Math.Round(avgSectorCompound, 4), SentimentLabel = sectorLabel, ActiveIsins = activeIsins, Text = $"Sector {sector} aggregate sentiment: {sectorLabel}." }, Analyses = updatedAnalyses }; var outJson = JsonSerializer.Serialize(summary, _jsonOptions); await File.WriteAllTextAsync(filePath, outJson); await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated Sector summary file: {Path} (Active ISINs: {Count})", filePath, activeIsins.Count); } catch (Exception ex) { await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update Sector summary for {Sector}", sector); } finally { fileLock.Release(); } } }