using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos.News; using FinlyticCore.Dtos.Sentiment; using FinlyticCore.Services; using FinlyticSentiment.Database; using FinlyticSentiment.Entities; using FinlyticSentiment.Util; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; namespace FinlyticSentiment.Services; /// /// Service contract for persisting and retrieving asset, article, and sector sentiments. /// public interface ISentimentDbService { Task SaveArticleSentimentAsync( Guid articleId, string isin, string companyName, string? sector, DateTime publishedAt, FinBertResultDto finbert, CancellationToken ct = default); Task GetCompanySentimentAsync(string isin, CancellationToken ct = default); Task GetIsinSummaryDtoAsync(string isin, CancellationToken ct = default); Task> GetAllCompanySentimentsAsync(int limit, int offset, string? sector = null, CancellationToken ct = default); Task GetSectorSentimentAsync(string sector, CancellationToken ct = default); Task> GetArticleSentimentsAsync(Guid articleId, CancellationToken ct = default); Task GetArticleSentimentEntryAsync(Guid articleId, CancellationToken ct = default); Task> GetSentimentTimelineAsync(string isin, int days, CancellationToken ct = default); } /// /// Database persistence service implementing exponential half-life time-decay scoring, /// optimistic concurrency protection, and sub-millisecond pre-aggregated lookups. /// public class SentimentDbService : ISentimentDbService { private readonly SentimentDbContext _context; private readonly IServiceScopeFactory _scopeFactory; private readonly IFinlyticLogger _finlyticLogger; public SentimentDbService( SentimentDbContext context, IServiceScopeFactory scopeFactory, IFinlyticLogger finlyticLogger) { _context = context; _scopeFactory = scopeFactory; _finlyticLogger = finlyticLogger; } /// public async Task SaveArticleSentimentAsync( Guid articleId, string isin, string companyName, string? sector, DateTime publishedAt, FinBertResultDto finbert, CancellationToken ct = default) { if (articleId == Guid.Empty || string.IsNullOrWhiteSpace(isin) || finbert == null) return null; var cleanIsin = isin.Trim().ToUpperInvariant(); var cleanName = !string.IsNullOrWhiteSpace(companyName) ? companyName.Trim() : cleanIsin; var cleanSector = !string.IsNullOrWhiteSpace(sector) ? sector.Trim() : null; var publishedUtc = publishedAt.Kind == DateTimeKind.Unspecified ? DateTime.SpecifyKind(publishedAt, DateTimeKind.Utc) : publishedAt.ToUniversalTime(); var articleSentiment = new ArticleSentimentEntity { Id = Guid.NewGuid(), ArticleId = articleId, Isin = cleanIsin, Name = cleanName, Sector = cleanSector, Label = finbert.Label, CompoundScore = finbert.CompoundScore, Confidence = finbert.Confidence, Impact = finbert.Impact, PositiveProbability = finbert.Probabilities.Positive, NegativeProbability = finbert.Probabilities.Negative, NeutralProbability = finbert.Probabilities.Neutral, KeyHighlight = finbert.KeyHighlight, PublishedAtUtc = publishedUtc, AnalyzedAtUtc = DateTime.UtcNow }; // 1. Insert Article Sentiment with Idempotency Protection try { var existing = await _context.ArticleSentiments .AsNoTracking() .FirstOrDefaultAsync(a => a.ArticleId == articleId && a.Isin == cleanIsin, ct); if (existing != null) { await _finlyticLogger.LogDebugAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Article {ArticleId} already has sentiment for ISIN {Isin}.", articleId, cleanIsin); return existing; } _context.ArticleSentiments.Add(articleSentiment); await _context.SaveChangesAsync(ct); } catch (DbUpdateException ex) { _context.ChangeTracker.Clear(); await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Unique constraint or concurrency hit on insert for ISIN {Isin}. Message: {Msg}", cleanIsin, ex.Message); } // 2. Concurrency-Safe Recalculation of Company Summary with Exponential Time-Decay await UpdateCompanySummaryWithRetryAsync(cleanIsin, cleanName, cleanSector, ct); // 3. Update Sector Summary if sector is present if (!string.IsNullOrWhiteSpace(cleanSector)) { await UpdateSectorSummaryAsync(cleanSector, ct); } return articleSentiment; } private async Task UpdateCompanySummaryWithRetryAsync( string isin, string companyName, string? sector, CancellationToken ct) { const int maxRetries = 5; for (int attempt = 1; attempt <= maxRetries; attempt++) { try { double halfLifeDays = 7.0; int windowDays = 30; try { using var scope = _scopeFactory.CreateScope(); var settings = scope.ServiceProvider.GetRequiredService(); halfLifeDays = await settings.GetSettingAsync(SettingKeys.TimeDecayHalfLifeDays, ct); windowDays = await settings.GetSettingAsync(SettingKeys.SentimentWindowDays, ct); } catch { } var cutoffDate = DateTime.UtcNow.AddDays(-windowDays); // Query all recent sentiment analyses for this asset var recentAnalyses = await _context.ArticleSentiments .AsNoTracking() .Where(a => a.Isin == isin && a.PublishedAtUtc >= cutoffDate) .OrderByDescending(a => a.PublishedAtUtc) .ToListAsync(ct); if (recentAnalyses.Count == 0) { // Fallback to latest available analysis if none within window var latest = await _context.ArticleSentiments .AsNoTracking() .Where(a => a.Isin == isin) .OrderByDescending(a => a.PublishedAtUtc) .Take(1) .ToListAsync(ct); recentAnalyses = latest; } if (recentAnalyses.Count == 0) return; // Mathematical Half-Life Time-Decay Calculation: // lambda = ln(2) / HalfLifeDays // Weight_i = Confidence_i * exp(-lambda * deltaDays_i) // WeightedScore = sum(Weight_i * CompoundScore_i) / sum(Weight_i) double lambda = Math.Log(2.0) / Math.Max(0.1, halfLifeDays); var now = DateTime.UtcNow; double totalWeightedScore = 0.0; double totalWeights = 0.0; double sumRawScore = 0.0; double sumConfidence = 0.0; int positiveCount = 0; int negativeCount = 0; int neutralCount = 0; foreach (var a in recentAnalyses) { double deltaDays = Math.Max(0.0, (now - a.PublishedAtUtc).TotalDays); double timeDecay = Math.Exp(-lambda * deltaDays); double weight = Math.Max(0.01, a.Confidence) * timeDecay; totalWeightedScore += weight * a.CompoundScore; totalWeights += weight; sumRawScore += a.CompoundScore; sumConfidence += a.Confidence; if (a.CompoundScore >= 0.15 || a.Label == "POSITIVE") positiveCount++; else if (a.CompoundScore <= -0.15 || a.Label == "NEGATIVE") negativeCount++; else neutralCount++; } double finalWeightedScore = totalWeights > 0 ? Math.Round(totalWeightedScore / totalWeights, 4) : 0.0; double finalAvgScore = Math.Round(sumRawScore / recentAnalyses.Count, 4); double finalAvgConfidence = Math.Round(sumConfidence / recentAnalyses.Count, 4); string currentLabel = finalWeightedScore switch { > 0.50 => "VERY_BULLISH", > 0.15 => "BULLISH", < -0.50 => "VERY_BEARISH", < -0.15 => "BEARISH", _ => "NEUTRAL" }; // Trend detection: Compare recent (last 7 days) vs older weighted scores string trend = "STABLE"; var last7DaysCutoff = now.AddDays(-7); var veryRecent = recentAnalyses.Where(a => a.PublishedAtUtc >= last7DaysCutoff).ToList(); var older = recentAnalyses.Where(a => a.PublishedAtUtc < last7DaysCutoff).ToList(); if (veryRecent.Count > 0 && older.Count > 0) { double recentScore = veryRecent.Average(a => a.CompoundScore); double olderScore = older.Average(a => a.CompoundScore); double diff = recentScore - olderScore; if (diff > 0.20) trend = "IMPROVING"; else if (diff < -0.20) trend = "DETERIORATING"; } var latestHighlight = recentAnalyses.FirstOrDefault(a => !string.IsNullOrWhiteSpace(a.KeyHighlight))?.KeyHighlight; var existingSummary = await _context.CompanySentiments.FirstOrDefaultAsync(c => c.Isin == isin, ct); if (existingSummary == null) { var newSummary = new CompanySentimentSummaryEntity { Isin = isin, Name = companyName, Sector = sector, CurrentLabel = currentLabel, AverageScore = finalAvgScore, WeightedScore = finalWeightedScore, AverageConfidence = finalAvgConfidence, TotalAnalysesCount = recentAnalyses.Count, PositiveCount = positiveCount, NegativeCount = negativeCount, NeutralCount = neutralCount, LatestKeyHighlight = latestHighlight, Trend = trend, LastUpdatedUtc = DateTime.UtcNow, Version = 1 }; _context.CompanySentiments.Add(newSummary); } else { existingSummary.Name = !string.IsNullOrWhiteSpace(companyName) ? companyName : existingSummary.Name; existingSummary.Sector = sector ?? existingSummary.Sector; existingSummary.CurrentLabel = currentLabel; existingSummary.AverageScore = finalAvgScore; existingSummary.WeightedScore = finalWeightedScore; existingSummary.AverageConfidence = finalAvgConfidence; existingSummary.TotalAnalysesCount = recentAnalyses.Count; existingSummary.PositiveCount = positiveCount; existingSummary.NegativeCount = negativeCount; existingSummary.NeutralCount = neutralCount; existingSummary.LatestKeyHighlight = latestHighlight ?? existingSummary.LatestKeyHighlight; existingSummary.Trend = trend; existingSummary.LastUpdatedUtc = DateTime.UtcNow; existingSummary.Version++; } await _context.SaveChangesAsync(ct); await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Updated company summary for {Isin} ({Name}): {Label} (Weighted: {Score}, Trend: {Trend})", isin, companyName, currentLabel, finalWeightedScore, trend); return; } catch (DbUpdateConcurrencyException) { _context.ChangeTracker.Clear(); if (attempt == maxRetries) throw; await Task.Delay(Random.Shared.Next(50, 150) * attempt, ct); } catch (Exception ex) { _context.ChangeTracker.Clear(); await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentDbService] Error updating company summary for {Isin} (Attempt {Attempt})", isin, attempt); if (attempt == maxRetries) throw; await Task.Delay(100, ct); } } } private async Task UpdateSectorSummaryAsync(string sector, CancellationToken ct) { try { var companySummaries = await _context.CompanySentiments .AsNoTracking() .Where(c => c.Sector == sector) .ToListAsync(ct); if (companySummaries.Count == 0) return; double avgScore = Math.Round(companySummaries.Average(c => c.WeightedScore), 4); int totalArticles = companySummaries.Sum(c => c.TotalAnalysesCount); int totalCompanies = companySummaries.Count; string currentLabel = avgScore switch { > 0.15 => "POSITIVE", < -0.15 => "NEGATIVE", _ => "NEUTRAL" }; var existing = await _context.SectorSentiments.FirstOrDefaultAsync(s => s.Sector == sector, ct); if (existing == null) { _context.SectorSentiments.Add(new SectorSentimentSummaryEntity { Sector = sector, CurrentLabel = currentLabel, AverageScore = avgScore, TotalArticlesCount = totalArticles, TotalCompaniesCount = totalCompanies, LastUpdatedUtc = DateTime.UtcNow }); } else { existing.CurrentLabel = currentLabel; existing.AverageScore = avgScore; existing.TotalArticlesCount = totalArticles; existing.TotalCompaniesCount = totalCompanies; existing.LastUpdatedUtc = DateTime.UtcNow; } await _context.SaveChangesAsync(ct); } catch (Exception ex) { _context.ChangeTracker.Clear(); await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentDbService] Failed to update sector summary for {Sector}", sector); } } /// public async Task GetCompanySentimentAsync(string isin, CancellationToken ct = default) { if (string.IsNullOrWhiteSpace(isin)) return null; var cleanIsin = isin.Trim().ToUpperInvariant(); return await _context.CompanySentiments .AsNoTracking() .FirstOrDefaultAsync(c => c.Isin == cleanIsin, ct); } /// public async Task GetIsinSummaryDtoAsync(string isin, CancellationToken ct = default) { if (string.IsNullOrWhiteSpace(isin)) return null; var cleanIsin = isin.Trim().ToUpperInvariant(); var summary = await _context.CompanySentiments .AsNoTracking() .FirstOrDefaultAsync(c => c.Isin == cleanIsin, ct); if (summary == null) return null; var recentAnalyses = await _context.ArticleSentiments .AsNoTracking() .Where(a => a.Isin == cleanIsin) .OrderByDescending(a => a.PublishedAtUtc) .Take(15) .ToListAsync(ct); return new IsinSentimentSummaryDto { Isin = summary.Isin, CompanyName = summary.Name, Sector = summary.Sector ?? "General", LastUpdated = summary.LastUpdatedUtc.ToString("o"), CurrentSummary = new IsinCurrentSummary { CompoundScore = summary.WeightedScore, SentimentLabel = summary.CurrentLabel, AvgConfidence = summary.AverageConfidence, TotalArticlesAnalyzed = summary.TotalAnalysesCount, PositiveArticles = summary.PositiveCount, NegativeArticles = summary.NegativeCount, NeutralArticles = summary.NeutralCount, Trend = summary.Trend, KeyHighlight = summary.LatestKeyHighlight }, Analyses = recentAnalyses.Select(a => new IsinAnalysisEntry { AnalysisId = a.Id.ToString(), Timestamp = a.AnalyzedAtUtc.ToString("o"), Article = new IsinAnalysisArticleRef { ArticleId = a.ArticleId.ToString(), Title = a.Name, Source = "FinlyticNews", PublishedAt = a.PublishedAtUtc.ToString("o") }, FinbertResult = new FinBertResultDto { Label = a.Label, CompoundScore = a.CompoundScore, Confidence = a.Confidence, Impact = a.Impact, KeyHighlight = a.KeyHighlight, Probabilities = new FinBertProbabilities { Positive = a.PositiveProbability, Negative = a.NegativeProbability, Neutral = a.NeutralProbability } }, SummarySnippet = a.KeyHighlight ?? string.Empty }).ToList() }; } /// public async Task> GetAllCompanySentimentsAsync( int limit, int offset, string? sector = null, CancellationToken ct = default) { var query = _context.CompanySentiments.AsNoTracking().AsQueryable(); if (!string.IsNullOrWhiteSpace(sector)) { var cleanSector = sector.Trim(); query = query.Where(c => c.Sector == cleanSector); } return await query .OrderByDescending(c => c.LastUpdatedUtc) .Skip(Math.Max(0, offset)) .Take(limit > 0 ? Math.Min(limit, 100) : 50) .ToListAsync(ct); } /// public async Task GetSectorSentimentAsync(string sector, CancellationToken ct = default) { if (string.IsNullOrWhiteSpace(sector)) return null; var cleanSector = sector.Trim(); var entity = await _context.SectorSentiments .AsNoTracking() .FirstOrDefaultAsync(s => s.Sector == cleanSector, ct); if (entity == null) return null; var activeIsins = await _context.CompanySentiments .AsNoTracking() .Where(c => c.Sector == cleanSector) .Select(c => c.Isin) .Take(20) .ToListAsync(ct); return new SectorSentimentSummaryDto { Sector = entity.Sector, LastUpdated = entity.LastUpdatedUtc.ToString("o"), CurrentSummary = new SectorCurrentSummary { CompoundScore = entity.AverageScore, SentimentLabel = entity.CurrentLabel, ActiveIsins = activeIsins, TotalArticlesAnalyzed = entity.TotalArticlesCount } }; } /// public async Task> GetArticleSentimentsAsync(Guid articleId, CancellationToken ct = default) { if (articleId == Guid.Empty) return []; return await _context.ArticleSentiments .AsNoTracking() .Where(a => a.ArticleId == articleId) .ToListAsync(ct); } /// public async Task GetArticleSentimentEntryAsync(Guid articleId, CancellationToken ct = default) { if (articleId == Guid.Empty) return null; var a = await _context.ArticleSentiments .AsNoTracking() .FirstOrDefaultAsync(x => x.ArticleId == articleId, ct); if (a == null) return null; return new IsinAnalysisEntry { AnalysisId = a.Id.ToString(), Timestamp = a.AnalyzedAtUtc.ToString("o"), Article = new IsinAnalysisArticleRef { ArticleId = a.ArticleId.ToString(), Title = a.Name, Source = "FinlyticNews", PublishedAt = a.PublishedAtUtc.ToString("o") }, FinbertResult = new FinBertResultDto { Label = a.Label, CompoundScore = a.CompoundScore, Confidence = a.Confidence, Impact = a.Impact, KeyHighlight = a.KeyHighlight, Probabilities = new FinBertProbabilities { Positive = a.PositiveProbability, Negative = a.NegativeProbability, Neutral = a.NeutralProbability } }, SummarySnippet = a.KeyHighlight ?? string.Empty }; } /// public async Task> GetSentimentTimelineAsync(string isin, int days, CancellationToken ct = default) { if (string.IsNullOrWhiteSpace(isin)) return []; var cleanIsin = isin.Trim().ToUpperInvariant(); var cutoff = DateTime.UtcNow.AddDays(-Math.Max(1, days)); return await _context.ArticleSentiments .AsNoTracking() .Where(a => a.Isin == cleanIsin && a.PublishedAtUtc >= cutoff) .OrderByDescending(a => a.PublishedAtUtc) .ToListAsync(ct); } }