feat(sentiment): add persistent sentiment entities, summaries, SentimentDbService and MQTT RPC refactoring

This commit is contained in:
2026-08-24 21:35:48 +02:00
parent 600ccf299e
commit 12e7b57b16
24 changed files with 1510 additions and 1300 deletions
@@ -0,0 +1,556 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using FinlyticCore.Services;
using FinlyticSentiment.Database;
using FinlyticSentiment.Entities;
using FinlyticSentiment.Util;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
namespace FinlyticSentiment.Services;
/// <summary>
/// Service contract for persisting and retrieving asset, article, and sector sentiments.
/// </summary>
public interface ISentimentDbService
{
Task<ArticleSentimentEntity?> SaveArticleSentimentAsync(
Guid articleId,
string isin,
string companyName,
string? sector,
DateTime publishedAt,
FinBertResultDto finbert,
CancellationToken ct = default);
Task<CompanySentimentSummaryEntity?> GetCompanySentimentAsync(string isin, CancellationToken ct = default);
Task<IsinSentimentSummaryDto?> GetIsinSummaryDtoAsync(string isin, CancellationToken ct = default);
Task<List<CompanySentimentSummaryEntity>> GetAllCompanySentimentsAsync(int limit, int offset, string? sector = null, CancellationToken ct = default);
Task<SectorSentimentSummaryDto?> GetSectorSentimentAsync(string sector, CancellationToken ct = default);
Task<List<ArticleSentimentEntity>> GetArticleSentimentsAsync(Guid articleId, CancellationToken ct = default);
Task<IsinAnalysisEntry?> GetArticleSentimentEntryAsync(Guid articleId, CancellationToken ct = default);
Task<List<ArticleSentimentEntity>> GetSentimentTimelineAsync(string isin, int days, CancellationToken ct = default);
}
/// <summary>
/// Database persistence service implementing exponential half-life time-decay scoring,
/// optimistic concurrency protection, and sub-millisecond pre-aggregated lookups.
/// </summary>
public class SentimentDbService : ISentimentDbService
{
private readonly SentimentDbContext _context;
private readonly IServiceScopeFactory _scopeFactory;
private readonly IFinlyticLogger<SentimentDbService> _finlyticLogger;
public SentimentDbService(
SentimentDbContext context,
IServiceScopeFactory scopeFactory,
IFinlyticLogger<SentimentDbService> finlyticLogger)
{
_context = context;
_scopeFactory = scopeFactory;
_finlyticLogger = finlyticLogger;
}
/// <inheritdoc />
public async Task<ArticleSentimentEntity?> SaveArticleSentimentAsync(
Guid articleId,
string isin,
string companyName,
string? sector,
DateTime publishedAt,
FinBertResultDto finbert,
CancellationToken ct = default)
{
if (articleId == Guid.Empty || string.IsNullOrWhiteSpace(isin) || finbert == null)
return null;
var cleanIsin = isin.Trim().ToUpperInvariant();
var cleanName = !string.IsNullOrWhiteSpace(companyName) ? companyName.Trim() : cleanIsin;
var cleanSector = !string.IsNullOrWhiteSpace(sector) ? sector.Trim() : null;
var publishedUtc = publishedAt.Kind == DateTimeKind.Unspecified
? DateTime.SpecifyKind(publishedAt, DateTimeKind.Utc)
: publishedAt.ToUniversalTime();
var articleSentiment = new ArticleSentimentEntity
{
Id = Guid.NewGuid(),
ArticleId = articleId,
Isin = cleanIsin,
Name = cleanName,
Sector = cleanSector,
Label = finbert.Label,
CompoundScore = finbert.CompoundScore,
Confidence = finbert.Confidence,
Impact = finbert.Impact,
PositiveProbability = finbert.Probabilities.Positive,
NegativeProbability = finbert.Probabilities.Negative,
NeutralProbability = finbert.Probabilities.Neutral,
KeyHighlight = finbert.KeyHighlight,
PublishedAtUtc = publishedUtc,
AnalyzedAtUtc = DateTime.UtcNow
};
// 1. Insert Article Sentiment with Idempotency Protection
try
{
var existing = await _context.ArticleSentiments
.AsNoTracking()
.FirstOrDefaultAsync(a => a.ArticleId == articleId && a.Isin == cleanIsin, ct);
if (existing != null)
{
await _finlyticLogger.LogDebugAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Article {ArticleId} already has sentiment for ISIN {Isin}.", articleId, cleanIsin);
return existing;
}
_context.ArticleSentiments.Add(articleSentiment);
await _context.SaveChangesAsync(ct);
}
catch (DbUpdateException ex)
{
_context.ChangeTracker.Clear();
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Unique constraint or concurrency hit on insert for ISIN {Isin}. Message: {Msg}", cleanIsin, ex.Message);
}
// 2. Concurrency-Safe Recalculation of Company Summary with Exponential Time-Decay
await UpdateCompanySummaryWithRetryAsync(cleanIsin, cleanName, cleanSector, ct);
// 3. Update Sector Summary if sector is present
if (!string.IsNullOrWhiteSpace(cleanSector))
{
await UpdateSectorSummaryAsync(cleanSector, ct);
}
return articleSentiment;
}
private async Task UpdateCompanySummaryWithRetryAsync(
string isin,
string companyName,
string? sector,
CancellationToken ct)
{
const int maxRetries = 5;
for (int attempt = 1; attempt <= maxRetries; attempt++)
{
try
{
double halfLifeDays = 7.0;
int windowDays = 30;
try
{
using var scope = _scopeFactory.CreateScope();
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
halfLifeDays = await settings.GetSettingAsync(SettingKeys.TimeDecayHalfLifeDays, ct);
windowDays = await settings.GetSettingAsync(SettingKeys.SentimentWindowDays, ct);
}
catch { }
var cutoffDate = DateTime.UtcNow.AddDays(-windowDays);
// Query all recent sentiment analyses for this asset
var recentAnalyses = await _context.ArticleSentiments
.AsNoTracking()
.Where(a => a.Isin == isin && a.PublishedAtUtc >= cutoffDate)
.OrderByDescending(a => a.PublishedAtUtc)
.ToListAsync(ct);
if (recentAnalyses.Count == 0)
{
// Fallback to latest available analysis if none within window
var latest = await _context.ArticleSentiments
.AsNoTracking()
.Where(a => a.Isin == isin)
.OrderByDescending(a => a.PublishedAtUtc)
.Take(1)
.ToListAsync(ct);
recentAnalyses = latest;
}
if (recentAnalyses.Count == 0) return;
// Mathematical Half-Life Time-Decay Calculation:
// lambda = ln(2) / HalfLifeDays
// Weight_i = Confidence_i * exp(-lambda * deltaDays_i)
// WeightedScore = sum(Weight_i * CompoundScore_i) / sum(Weight_i)
double lambda = Math.Log(2.0) / Math.Max(0.1, halfLifeDays);
var now = DateTime.UtcNow;
double totalWeightedScore = 0.0;
double totalWeights = 0.0;
double sumRawScore = 0.0;
double sumConfidence = 0.0;
int positiveCount = 0;
int negativeCount = 0;
int neutralCount = 0;
foreach (var a in recentAnalyses)
{
double deltaDays = Math.Max(0.0, (now - a.PublishedAtUtc).TotalDays);
double timeDecay = Math.Exp(-lambda * deltaDays);
double weight = Math.Max(0.01, a.Confidence) * timeDecay;
totalWeightedScore += weight * a.CompoundScore;
totalWeights += weight;
sumRawScore += a.CompoundScore;
sumConfidence += a.Confidence;
if (a.CompoundScore >= 0.15 || a.Label == "POSITIVE") positiveCount++;
else if (a.CompoundScore <= -0.15 || a.Label == "NEGATIVE") negativeCount++;
else neutralCount++;
}
double finalWeightedScore = totalWeights > 0 ? Math.Round(totalWeightedScore / totalWeights, 4) : 0.0;
double finalAvgScore = Math.Round(sumRawScore / recentAnalyses.Count, 4);
double finalAvgConfidence = Math.Round(sumConfidence / recentAnalyses.Count, 4);
string currentLabel = finalWeightedScore switch
{
> 0.50 => "VERY_BULLISH",
> 0.15 => "BULLISH",
< -0.50 => "VERY_BEARISH",
< -0.15 => "BEARISH",
_ => "NEUTRAL"
};
// Trend detection: Compare recent (last 7 days) vs older weighted scores
string trend = "STABLE";
var last7DaysCutoff = now.AddDays(-7);
var veryRecent = recentAnalyses.Where(a => a.PublishedAtUtc >= last7DaysCutoff).ToList();
var older = recentAnalyses.Where(a => a.PublishedAtUtc < last7DaysCutoff).ToList();
if (veryRecent.Count > 0 && older.Count > 0)
{
double recentScore = veryRecent.Average(a => a.CompoundScore);
double olderScore = older.Average(a => a.CompoundScore);
double diff = recentScore - olderScore;
if (diff > 0.20) trend = "IMPROVING";
else if (diff < -0.20) trend = "DETERIORATING";
}
var latestHighlight = recentAnalyses.FirstOrDefault(a => !string.IsNullOrWhiteSpace(a.KeyHighlight))?.KeyHighlight;
var existingSummary = await _context.CompanySentiments.FirstOrDefaultAsync(c => c.Isin == isin, ct);
if (existingSummary == null)
{
var newSummary = new CompanySentimentSummaryEntity
{
Isin = isin,
Name = companyName,
Sector = sector,
CurrentLabel = currentLabel,
AverageScore = finalAvgScore,
WeightedScore = finalWeightedScore,
AverageConfidence = finalAvgConfidence,
TotalAnalysesCount = recentAnalyses.Count,
PositiveCount = positiveCount,
NegativeCount = negativeCount,
NeutralCount = neutralCount,
LatestKeyHighlight = latestHighlight,
Trend = trend,
LastUpdatedUtc = DateTime.UtcNow,
Version = 1
};
_context.CompanySentiments.Add(newSummary);
}
else
{
existingSummary.Name = !string.IsNullOrWhiteSpace(companyName) ? companyName : existingSummary.Name;
existingSummary.Sector = sector ?? existingSummary.Sector;
existingSummary.CurrentLabel = currentLabel;
existingSummary.AverageScore = finalAvgScore;
existingSummary.WeightedScore = finalWeightedScore;
existingSummary.AverageConfidence = finalAvgConfidence;
existingSummary.TotalAnalysesCount = recentAnalyses.Count;
existingSummary.PositiveCount = positiveCount;
existingSummary.NegativeCount = negativeCount;
existingSummary.NeutralCount = neutralCount;
existingSummary.LatestKeyHighlight = latestHighlight ?? existingSummary.LatestKeyHighlight;
existingSummary.Trend = trend;
existingSummary.LastUpdatedUtc = DateTime.UtcNow;
existingSummary.Version++;
}
await _context.SaveChangesAsync(ct);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Updated company summary for {Isin} ({Name}): {Label} (Weighted: {Score}, Trend: {Trend})", isin, companyName, currentLabel, finalWeightedScore, trend);
return;
}
catch (DbUpdateConcurrencyException)
{
_context.ChangeTracker.Clear();
if (attempt == maxRetries) throw;
await Task.Delay(Random.Shared.Next(50, 150) * attempt, ct);
}
catch (Exception ex)
{
_context.ChangeTracker.Clear();
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentDbService] Error updating company summary for {Isin} (Attempt {Attempt})", isin, attempt);
if (attempt == maxRetries) throw;
await Task.Delay(100, ct);
}
}
}
private async Task UpdateSectorSummaryAsync(string sector, CancellationToken ct)
{
try
{
var companySummaries = await _context.CompanySentiments
.AsNoTracking()
.Where(c => c.Sector == sector)
.ToListAsync(ct);
if (companySummaries.Count == 0) return;
double avgScore = Math.Round(companySummaries.Average(c => c.WeightedScore), 4);
int totalArticles = companySummaries.Sum(c => c.TotalAnalysesCount);
int totalCompanies = companySummaries.Count;
string currentLabel = avgScore switch
{
> 0.15 => "POSITIVE",
< -0.15 => "NEGATIVE",
_ => "NEUTRAL"
};
var existing = await _context.SectorSentiments.FirstOrDefaultAsync(s => s.Sector == sector, ct);
if (existing == null)
{
_context.SectorSentiments.Add(new SectorSentimentSummaryEntity
{
Sector = sector,
CurrentLabel = currentLabel,
AverageScore = avgScore,
TotalArticlesCount = totalArticles,
TotalCompaniesCount = totalCompanies,
LastUpdatedUtc = DateTime.UtcNow
});
}
else
{
existing.CurrentLabel = currentLabel;
existing.AverageScore = avgScore;
existing.TotalArticlesCount = totalArticles;
existing.TotalCompaniesCount = totalCompanies;
existing.LastUpdatedUtc = DateTime.UtcNow;
}
await _context.SaveChangesAsync(ct);
}
catch (Exception ex)
{
_context.ChangeTracker.Clear();
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentDbService] Failed to update sector summary for {Sector}", sector);
}
}
/// <inheritdoc />
public async Task<CompanySentimentSummaryEntity?> GetCompanySentimentAsync(string isin, CancellationToken ct = default)
{
if (string.IsNullOrWhiteSpace(isin)) return null;
var cleanIsin = isin.Trim().ToUpperInvariant();
return await _context.CompanySentiments
.AsNoTracking()
.FirstOrDefaultAsync(c => c.Isin == cleanIsin, ct);
}
/// <inheritdoc />
public async Task<IsinSentimentSummaryDto?> GetIsinSummaryDtoAsync(string isin, CancellationToken ct = default)
{
if (string.IsNullOrWhiteSpace(isin)) return null;
var cleanIsin = isin.Trim().ToUpperInvariant();
var summary = await _context.CompanySentiments
.AsNoTracking()
.FirstOrDefaultAsync(c => c.Isin == cleanIsin, ct);
if (summary == null) return null;
var recentAnalyses = await _context.ArticleSentiments
.AsNoTracking()
.Where(a => a.Isin == cleanIsin)
.OrderByDescending(a => a.PublishedAtUtc)
.Take(15)
.ToListAsync(ct);
return new IsinSentimentSummaryDto
{
Isin = summary.Isin,
CompanyName = summary.Name,
Sector = summary.Sector ?? "General",
LastUpdated = summary.LastUpdatedUtc.ToString("o"),
CurrentSummary = new IsinCurrentSummary
{
CompoundScore = summary.WeightedScore,
SentimentLabel = summary.CurrentLabel,
AvgConfidence = summary.AverageConfidence,
TotalArticlesAnalyzed = summary.TotalAnalysesCount,
PositiveArticles = summary.PositiveCount,
NegativeArticles = summary.NegativeCount,
NeutralArticles = summary.NeutralCount,
Trend = summary.Trend,
KeyHighlight = summary.LatestKeyHighlight
},
Analyses = recentAnalyses.Select(a => new IsinAnalysisEntry
{
AnalysisId = a.Id.ToString(),
Timestamp = a.AnalyzedAtUtc.ToString("o"),
Article = new IsinAnalysisArticleRef
{
ArticleId = a.ArticleId.ToString(),
Title = a.Name,
Source = "FinlyticNews",
PublishedAt = a.PublishedAtUtc.ToString("o")
},
FinbertResult = new FinBertResultDto
{
Label = a.Label,
CompoundScore = a.CompoundScore,
Confidence = a.Confidence,
Impact = a.Impact,
KeyHighlight = a.KeyHighlight,
Probabilities = new FinBertProbabilities
{
Positive = a.PositiveProbability,
Negative = a.NegativeProbability,
Neutral = a.NeutralProbability
}
},
SummarySnippet = a.KeyHighlight ?? string.Empty
}).ToList()
};
}
/// <inheritdoc />
public async Task<List<CompanySentimentSummaryEntity>> GetAllCompanySentimentsAsync(
int limit,
int offset,
string? sector = null,
CancellationToken ct = default)
{
var query = _context.CompanySentiments.AsNoTracking().AsQueryable();
if (!string.IsNullOrWhiteSpace(sector))
{
var cleanSector = sector.Trim();
query = query.Where(c => c.Sector == cleanSector);
}
return await query
.OrderByDescending(c => c.LastUpdatedUtc)
.Skip(Math.Max(0, offset))
.Take(limit > 0 ? Math.Min(limit, 100) : 50)
.ToListAsync(ct);
}
/// <inheritdoc />
public async Task<SectorSentimentSummaryDto?> GetSectorSentimentAsync(string sector, CancellationToken ct = default)
{
if (string.IsNullOrWhiteSpace(sector)) return null;
var cleanSector = sector.Trim();
var entity = await _context.SectorSentiments
.AsNoTracking()
.FirstOrDefaultAsync(s => s.Sector == cleanSector, ct);
if (entity == null) return null;
var activeIsins = await _context.CompanySentiments
.AsNoTracking()
.Where(c => c.Sector == cleanSector)
.Select(c => c.Isin)
.Take(20)
.ToListAsync(ct);
return new SectorSentimentSummaryDto
{
Sector = entity.Sector,
LastUpdated = entity.LastUpdatedUtc.ToString("o"),
CurrentSummary = new SectorCurrentSummary
{
CompoundScore = entity.AverageScore,
SentimentLabel = entity.CurrentLabel,
ActiveIsins = activeIsins,
TotalArticlesAnalyzed = entity.TotalArticlesCount
}
};
}
/// <inheritdoc />
public async Task<List<ArticleSentimentEntity>> GetArticleSentimentsAsync(Guid articleId, CancellationToken ct = default)
{
if (articleId == Guid.Empty) return [];
return await _context.ArticleSentiments
.AsNoTracking()
.Where(a => a.ArticleId == articleId)
.ToListAsync(ct);
}
/// <inheritdoc />
public async Task<IsinAnalysisEntry?> GetArticleSentimentEntryAsync(Guid articleId, CancellationToken ct = default)
{
if (articleId == Guid.Empty) return null;
var a = await _context.ArticleSentiments
.AsNoTracking()
.FirstOrDefaultAsync(x => x.ArticleId == articleId, ct);
if (a == null) return null;
return new IsinAnalysisEntry
{
AnalysisId = a.Id.ToString(),
Timestamp = a.AnalyzedAtUtc.ToString("o"),
Article = new IsinAnalysisArticleRef
{
ArticleId = a.ArticleId.ToString(),
Title = a.Name,
Source = "FinlyticNews",
PublishedAt = a.PublishedAtUtc.ToString("o")
},
FinbertResult = new FinBertResultDto
{
Label = a.Label,
CompoundScore = a.CompoundScore,
Confidence = a.Confidence,
Impact = a.Impact,
KeyHighlight = a.KeyHighlight,
Probabilities = new FinBertProbabilities
{
Positive = a.PositiveProbability,
Negative = a.NegativeProbability,
Neutral = a.NeutralProbability
}
},
SummarySnippet = a.KeyHighlight ?? string.Empty
};
}
/// <inheritdoc />
public async Task<List<ArticleSentimentEntity>> GetSentimentTimelineAsync(string isin, int days, CancellationToken ct = default)
{
if (string.IsNullOrWhiteSpace(isin)) return [];
var cleanIsin = isin.Trim().ToUpperInvariant();
var cutoff = DateTime.UtcNow.AddDays(-Math.Max(1, days));
return await _context.ArticleSentiments
.AsNoTracking()
.Where(a => a.Isin == cleanIsin && a.PublishedAtUtc >= cutoff)
.OrderByDescending(a => a.PublishedAtUtc)
.ToListAsync(ct);
}
}