feat(sentiment): add persistent sentiment entities, summaries, SentimentDbService and MQTT RPC refactoring
This commit is contained in:
@@ -52,26 +52,13 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
|
||||
double minConfidence = 0.60;
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
{
|
||||
var settingsService = scope.ServiceProvider.GetService<ISettingsService>();
|
||||
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
||||
bool isEnglish = string.Equals(article.Language, "en", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (settingsService != null)
|
||||
{
|
||||
targetUrl = isEnglish
|
||||
? await settingsService.GetSettingAsync(SettingKeys.EnglishWebhookUrl)
|
||||
: await settingsService.GetSettingAsync(SettingKeys.GermanWebhookUrl);
|
||||
minConfidence = await settingsService.GetSettingAsync(SettingKeys.MinimumConfidenceThreshold);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(targetUrl))
|
||||
{
|
||||
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
|
||||
var settings = await settingsDb.GetSettingsAsync();
|
||||
targetUrl = isEnglish
|
||||
? settings.EnglishWebhookUrl
|
||||
: settings.GermanWebhookUrl;
|
||||
minConfidence = settings.MinConfidenceScore;
|
||||
}
|
||||
targetUrl = isEnglish
|
||||
? await settingsService.GetSettingAsync(SettingKeys.EnglishWebhookUrl)
|
||||
: await settingsService.GetSettingAsync(SettingKeys.GermanWebhookUrl);
|
||||
minConfidence = await settingsService.GetSettingAsync(SettingKeys.MinimumConfidenceThreshold);
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] Analyzing article (ID: {Id}, Lang: {Lang}) via webhook: {Url} (MinConf: {Conf})", article.Id, article.Language ?? "de", targetUrl, minConfidence);
|
||||
@@ -128,11 +115,11 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
|
||||
double confidence = GetDoubleProp(root, "confidence")
|
||||
?? GetDoubleProp(root, "confidence_score")
|
||||
?? 0.5;
|
||||
string snippet = GetStringProp(root, "summary_snippet")
|
||||
string? impact = GetStringProp(root, "impact") ?? "MEDIUM";
|
||||
string? keyHighlight = GetStringProp(root, "key_highlight")
|
||||
?? GetStringProp(root, "summary_snippet")
|
||||
?? GetStringProp(root, "summary")
|
||||
?? GetStringProp(root, "text")
|
||||
?? article.Summary
|
||||
?? article.Title;
|
||||
?? article.Summary;
|
||||
|
||||
double pos = 0.0, neg = 0.0, neu = 1.0;
|
||||
if (root.TryGetProperty("probabilities", out var probsElem) && probsElem.ValueKind == JsonValueKind.Object)
|
||||
@@ -165,44 +152,57 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
|
||||
}
|
||||
}
|
||||
|
||||
return new FinBertResultDto
|
||||
var result = new FinBertResultDto
|
||||
{
|
||||
Label = label,
|
||||
CompoundScore = Math.Round(compoundScore, 4),
|
||||
Confidence = Math.Round(confidence, 4),
|
||||
Impact = impact.ToUpperInvariant(),
|
||||
KeyHighlight = keyHighlight,
|
||||
Probabilities = new FinBertProbabilities
|
||||
{
|
||||
Positive = Math.Round(pos, 4),
|
||||
Negative = Math.Round(neg, 4),
|
||||
Neutral = Math.Round(neu, 4)
|
||||
},
|
||||
SummarySnippet = snippet
|
||||
}
|
||||
};
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] Successfully analyzed article {Id}: {Label} (Compound: {Score}, Conf: {Conf}, Impact: {Impact})", article.Id, result.Label, result.CompoundScore, result.Confidence, result.Impact);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] n8n Webhook returned non-success status: {StatusCode}.", response.StatusCode);
|
||||
else
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] Webhook call failed with status: {Status} (Article ID: {Id})", response.StatusCode, article.Id);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[FinBertAnalyzerService] Failed to call n8n sentiment webhook.");
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[FinBertAnalyzerService] Exception during webhook request for article: {Id}", article.Id);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? GetStringProp(JsonElement elem, string propName)
|
||||
private static string? GetStringProp(JsonElement element, string propName)
|
||||
{
|
||||
return elem.TryGetProperty(propName, out var prop) && prop.ValueKind == JsonValueKind.String ? prop.GetString() : null;
|
||||
if (element.TryGetProperty(propName, out var prop) && prop.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return prop.GetString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static double? GetDoubleProp(JsonElement elem, string propName)
|
||||
private static double? GetDoubleProp(JsonElement element, string propName)
|
||||
{
|
||||
if (elem.TryGetProperty(propName, out var prop))
|
||||
if (element.TryGetProperty(propName, out var prop))
|
||||
{
|
||||
if (prop.ValueKind == JsonValueKind.Number && prop.TryGetDouble(out var d)) return d;
|
||||
if (prop.ValueKind == JsonValueKind.String && double.TryParse(prop.GetString(), out var parsed)) return parsed;
|
||||
if (prop.ValueKind == JsonValueKind.Number && prop.TryGetDouble(out var d))
|
||||
return d;
|
||||
if (prop.ValueKind == JsonValueKind.String && double.TryParse(prop.GetString(), System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out var parsed))
|
||||
return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -13,13 +13,12 @@ namespace FinlyticSentiment.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Background hosted worker executing periodic sentiment analysis sweeps on pending news articles
|
||||
/// and processing real-time article broadcasts.
|
||||
/// and processing real-time article broadcasts from FinlyticNews.
|
||||
/// </summary>
|
||||
public class SentimentBackgroundService : BackgroundService
|
||||
{
|
||||
private readonly SentimentMqttClient _mqttClient;
|
||||
private readonly IFinBertAnalyzerService _analyzer;
|
||||
private readonly ISentimentStorageService _storage;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IFinlyticLogger<SentimentBackgroundService> _finlyticLogger;
|
||||
|
||||
@@ -28,13 +27,11 @@ public class SentimentBackgroundService : BackgroundService
|
||||
public SentimentBackgroundService(
|
||||
SentimentMqttClient mqttClient,
|
||||
IFinBertAnalyzerService analyzer,
|
||||
ISentimentStorageService storage,
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IFinlyticLogger<SentimentBackgroundService> finlyticLogger)
|
||||
{
|
||||
_mqttClient = mqttClient;
|
||||
_analyzer = analyzer;
|
||||
_storage = storage;
|
||||
_scopeFactory = scopeFactory;
|
||||
_finlyticLogger = finlyticLogger;
|
||||
}
|
||||
@@ -56,11 +53,13 @@ public class SentimentBackgroundService : BackgroundService
|
||||
int maxBatchSize = 10;
|
||||
int sweepIntervalMinutes = 5;
|
||||
|
||||
using (var scope = _scopeFactory.CreateScope())
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
||||
maxBatchSize = await settings.GetSettingAsync(SettingKeys.MaxBatchSize, stoppingToken);
|
||||
}
|
||||
catch { }
|
||||
|
||||
var interval = TimeSpan.FromMinutes(Math.Max(1, sweepIntervalMinutes));
|
||||
|
||||
@@ -79,10 +78,9 @@ public class SentimentBackgroundService : BackgroundService
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Waiting {Minutes} minute(s) until next sentiment sweep...", interval.TotalMinutes);
|
||||
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
await timer.WaitForNextTickAsync(stoppingToken);
|
||||
await Task.Delay(interval, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -90,7 +88,7 @@ public class SentimentBackgroundService : BackgroundService
|
||||
}
|
||||
}
|
||||
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinlyticSentiment Background Service is shutting down gracefully.");
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinlyticSentiment Background Service shutting down.");
|
||||
}
|
||||
|
||||
private async Task PerformSentimentSweepAsync(int maxBatchSize, CancellationToken cancellationToken)
|
||||
@@ -137,45 +135,40 @@ public class SentimentBackgroundService : BackgroundService
|
||||
|
||||
if (cancellationToken.IsCancellationRequested) return;
|
||||
|
||||
await _storage.SaveArticleSentimentAsync(article, finbert);
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var dbService = scope.ServiceProvider.GetRequiredService<ISentimentDbService>();
|
||||
|
||||
if (article.MatchedAssets != null && article.MatchedAssets.Count > 0)
|
||||
{
|
||||
foreach (var asset in article.MatchedAssets)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested) break;
|
||||
if (string.IsNullOrWhiteSpace(asset.Isin)) continue;
|
||||
|
||||
await _storage.UpdateIsinSummaryAsync(
|
||||
asset.Isin,
|
||||
asset.Name,
|
||||
"General",
|
||||
article,
|
||||
finbert);
|
||||
await dbService.SaveArticleSentimentAsync(
|
||||
articleId: article.Id,
|
||||
isin: asset.Isin,
|
||||
companyName: asset.Name,
|
||||
sector: null,
|
||||
publishedAt: article.PublishedAt,
|
||||
finbert: finbert,
|
||||
ct: cancellationToken
|
||||
);
|
||||
|
||||
await _storage.UpdateSectorSummaryAsync(
|
||||
"General",
|
||||
asset.Isin,
|
||||
article.Id.ToString(),
|
||||
finbert);
|
||||
// Broadcast real-time updated summary for this asset
|
||||
var summaryDto = await dbService.GetIsinSummaryDtoAsync(asset.Isin, cancellationToken);
|
||||
if (summaryDto != null)
|
||||
{
|
||||
await _mqttClient.BroadcastSentimentResultAsync(asset.Isin, summaryDto);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cancellationToken.IsCancellationRequested) return;
|
||||
|
||||
bool updated = await _mqttClient.UpdateArticleStatusAsync(article.Id, "Analyzed");
|
||||
if (updated)
|
||||
{
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Article sentiment processed and status set to 'Analyzed' in FinlyticNews: {Title} (ID: {Id}) -> {Label}", article.Title, article.Id, finbert.Label);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Failed to confirm status update to 'Analyzed' in FinlyticNews for article: {Id}", article.Id);
|
||||
}
|
||||
await _mqttClient.UpdateArticleStatusAsync(article.Id, "Analyzed");
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Completed sentiment persistence and status transition for article {Id}.", article.Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentBackgroundService] Error processing sentiment for article: {Id} ({Title})", article.Id, article.Title);
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentBackgroundService] Failed to process sentiment for article: {Id}", article.Id);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -0,0 +1,556 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Dtos.News;
|
||||
using FinlyticCore.Dtos.Sentiment;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticSentiment.Database;
|
||||
using FinlyticSentiment.Entities;
|
||||
using FinlyticSentiment.Util;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace FinlyticSentiment.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Service contract for persisting and retrieving asset, article, and sector sentiments.
|
||||
/// </summary>
|
||||
public interface ISentimentDbService
|
||||
{
|
||||
Task<ArticleSentimentEntity?> SaveArticleSentimentAsync(
|
||||
Guid articleId,
|
||||
string isin,
|
||||
string companyName,
|
||||
string? sector,
|
||||
DateTime publishedAt,
|
||||
FinBertResultDto finbert,
|
||||
CancellationToken ct = default);
|
||||
|
||||
Task<CompanySentimentSummaryEntity?> GetCompanySentimentAsync(string isin, CancellationToken ct = default);
|
||||
|
||||
Task<IsinSentimentSummaryDto?> GetIsinSummaryDtoAsync(string isin, CancellationToken ct = default);
|
||||
|
||||
Task<List<CompanySentimentSummaryEntity>> GetAllCompanySentimentsAsync(int limit, int offset, string? sector = null, CancellationToken ct = default);
|
||||
|
||||
Task<SectorSentimentSummaryDto?> GetSectorSentimentAsync(string sector, CancellationToken ct = default);
|
||||
|
||||
Task<List<ArticleSentimentEntity>> GetArticleSentimentsAsync(Guid articleId, CancellationToken ct = default);
|
||||
|
||||
Task<IsinAnalysisEntry?> GetArticleSentimentEntryAsync(Guid articleId, CancellationToken ct = default);
|
||||
|
||||
Task<List<ArticleSentimentEntity>> GetSentimentTimelineAsync(string isin, int days, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Database persistence service implementing exponential half-life time-decay scoring,
|
||||
/// optimistic concurrency protection, and sub-millisecond pre-aggregated lookups.
|
||||
/// </summary>
|
||||
public class SentimentDbService : ISentimentDbService
|
||||
{
|
||||
private readonly SentimentDbContext _context;
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly IFinlyticLogger<SentimentDbService> _finlyticLogger;
|
||||
|
||||
public SentimentDbService(
|
||||
SentimentDbContext context,
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IFinlyticLogger<SentimentDbService> finlyticLogger)
|
||||
{
|
||||
_context = context;
|
||||
_scopeFactory = scopeFactory;
|
||||
_finlyticLogger = finlyticLogger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ArticleSentimentEntity?> SaveArticleSentimentAsync(
|
||||
Guid articleId,
|
||||
string isin,
|
||||
string companyName,
|
||||
string? sector,
|
||||
DateTime publishedAt,
|
||||
FinBertResultDto finbert,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (articleId == Guid.Empty || string.IsNullOrWhiteSpace(isin) || finbert == null)
|
||||
return null;
|
||||
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
var cleanName = !string.IsNullOrWhiteSpace(companyName) ? companyName.Trim() : cleanIsin;
|
||||
var cleanSector = !string.IsNullOrWhiteSpace(sector) ? sector.Trim() : null;
|
||||
|
||||
var publishedUtc = publishedAt.Kind == DateTimeKind.Unspecified
|
||||
? DateTime.SpecifyKind(publishedAt, DateTimeKind.Utc)
|
||||
: publishedAt.ToUniversalTime();
|
||||
|
||||
var articleSentiment = new ArticleSentimentEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
ArticleId = articleId,
|
||||
Isin = cleanIsin,
|
||||
Name = cleanName,
|
||||
Sector = cleanSector,
|
||||
Label = finbert.Label,
|
||||
CompoundScore = finbert.CompoundScore,
|
||||
Confidence = finbert.Confidence,
|
||||
Impact = finbert.Impact,
|
||||
PositiveProbability = finbert.Probabilities.Positive,
|
||||
NegativeProbability = finbert.Probabilities.Negative,
|
||||
NeutralProbability = finbert.Probabilities.Neutral,
|
||||
KeyHighlight = finbert.KeyHighlight,
|
||||
PublishedAtUtc = publishedUtc,
|
||||
AnalyzedAtUtc = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// 1. Insert Article Sentiment with Idempotency Protection
|
||||
try
|
||||
{
|
||||
var existing = await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(a => a.ArticleId == articleId && a.Isin == cleanIsin, ct);
|
||||
|
||||
if (existing != null)
|
||||
{
|
||||
await _finlyticLogger.LogDebugAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Article {ArticleId} already has sentiment for ISIN {Isin}.", articleId, cleanIsin);
|
||||
return existing;
|
||||
}
|
||||
|
||||
_context.ArticleSentiments.Add(articleSentiment);
|
||||
await _context.SaveChangesAsync(ct);
|
||||
}
|
||||
catch (DbUpdateException ex)
|
||||
{
|
||||
_context.ChangeTracker.Clear();
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Unique constraint or concurrency hit on insert for ISIN {Isin}. Message: {Msg}", cleanIsin, ex.Message);
|
||||
}
|
||||
|
||||
// 2. Concurrency-Safe Recalculation of Company Summary with Exponential Time-Decay
|
||||
await UpdateCompanySummaryWithRetryAsync(cleanIsin, cleanName, cleanSector, ct);
|
||||
|
||||
// 3. Update Sector Summary if sector is present
|
||||
if (!string.IsNullOrWhiteSpace(cleanSector))
|
||||
{
|
||||
await UpdateSectorSummaryAsync(cleanSector, ct);
|
||||
}
|
||||
|
||||
return articleSentiment;
|
||||
}
|
||||
|
||||
private async Task UpdateCompanySummaryWithRetryAsync(
|
||||
string isin,
|
||||
string companyName,
|
||||
string? sector,
|
||||
CancellationToken ct)
|
||||
{
|
||||
const int maxRetries = 5;
|
||||
for (int attempt = 1; attempt <= maxRetries; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
double halfLifeDays = 7.0;
|
||||
int windowDays = 30;
|
||||
|
||||
try
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
||||
halfLifeDays = await settings.GetSettingAsync(SettingKeys.TimeDecayHalfLifeDays, ct);
|
||||
windowDays = await settings.GetSettingAsync(SettingKeys.SentimentWindowDays, ct);
|
||||
}
|
||||
catch { }
|
||||
|
||||
var cutoffDate = DateTime.UtcNow.AddDays(-windowDays);
|
||||
|
||||
// Query all recent sentiment analyses for this asset
|
||||
var recentAnalyses = await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.Where(a => a.Isin == isin && a.PublishedAtUtc >= cutoffDate)
|
||||
.OrderByDescending(a => a.PublishedAtUtc)
|
||||
.ToListAsync(ct);
|
||||
|
||||
if (recentAnalyses.Count == 0)
|
||||
{
|
||||
// Fallback to latest available analysis if none within window
|
||||
var latest = await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.Where(a => a.Isin == isin)
|
||||
.OrderByDescending(a => a.PublishedAtUtc)
|
||||
.Take(1)
|
||||
.ToListAsync(ct);
|
||||
recentAnalyses = latest;
|
||||
}
|
||||
|
||||
if (recentAnalyses.Count == 0) return;
|
||||
|
||||
// Mathematical Half-Life Time-Decay Calculation:
|
||||
// lambda = ln(2) / HalfLifeDays
|
||||
// Weight_i = Confidence_i * exp(-lambda * deltaDays_i)
|
||||
// WeightedScore = sum(Weight_i * CompoundScore_i) / sum(Weight_i)
|
||||
double lambda = Math.Log(2.0) / Math.Max(0.1, halfLifeDays);
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
double totalWeightedScore = 0.0;
|
||||
double totalWeights = 0.0;
|
||||
double sumRawScore = 0.0;
|
||||
double sumConfidence = 0.0;
|
||||
int positiveCount = 0;
|
||||
int negativeCount = 0;
|
||||
int neutralCount = 0;
|
||||
|
||||
foreach (var a in recentAnalyses)
|
||||
{
|
||||
double deltaDays = Math.Max(0.0, (now - a.PublishedAtUtc).TotalDays);
|
||||
double timeDecay = Math.Exp(-lambda * deltaDays);
|
||||
double weight = Math.Max(0.01, a.Confidence) * timeDecay;
|
||||
|
||||
totalWeightedScore += weight * a.CompoundScore;
|
||||
totalWeights += weight;
|
||||
sumRawScore += a.CompoundScore;
|
||||
sumConfidence += a.Confidence;
|
||||
|
||||
if (a.CompoundScore >= 0.15 || a.Label == "POSITIVE") positiveCount++;
|
||||
else if (a.CompoundScore <= -0.15 || a.Label == "NEGATIVE") negativeCount++;
|
||||
else neutralCount++;
|
||||
}
|
||||
|
||||
double finalWeightedScore = totalWeights > 0 ? Math.Round(totalWeightedScore / totalWeights, 4) : 0.0;
|
||||
double finalAvgScore = Math.Round(sumRawScore / recentAnalyses.Count, 4);
|
||||
double finalAvgConfidence = Math.Round(sumConfidence / recentAnalyses.Count, 4);
|
||||
|
||||
string currentLabel = finalWeightedScore switch
|
||||
{
|
||||
> 0.50 => "VERY_BULLISH",
|
||||
> 0.15 => "BULLISH",
|
||||
< -0.50 => "VERY_BEARISH",
|
||||
< -0.15 => "BEARISH",
|
||||
_ => "NEUTRAL"
|
||||
};
|
||||
|
||||
// Trend detection: Compare recent (last 7 days) vs older weighted scores
|
||||
string trend = "STABLE";
|
||||
var last7DaysCutoff = now.AddDays(-7);
|
||||
var veryRecent = recentAnalyses.Where(a => a.PublishedAtUtc >= last7DaysCutoff).ToList();
|
||||
var older = recentAnalyses.Where(a => a.PublishedAtUtc < last7DaysCutoff).ToList();
|
||||
|
||||
if (veryRecent.Count > 0 && older.Count > 0)
|
||||
{
|
||||
double recentScore = veryRecent.Average(a => a.CompoundScore);
|
||||
double olderScore = older.Average(a => a.CompoundScore);
|
||||
double diff = recentScore - olderScore;
|
||||
if (diff > 0.20) trend = "IMPROVING";
|
||||
else if (diff < -0.20) trend = "DETERIORATING";
|
||||
}
|
||||
|
||||
var latestHighlight = recentAnalyses.FirstOrDefault(a => !string.IsNullOrWhiteSpace(a.KeyHighlight))?.KeyHighlight;
|
||||
|
||||
var existingSummary = await _context.CompanySentiments.FirstOrDefaultAsync(c => c.Isin == isin, ct);
|
||||
if (existingSummary == null)
|
||||
{
|
||||
var newSummary = new CompanySentimentSummaryEntity
|
||||
{
|
||||
Isin = isin,
|
||||
Name = companyName,
|
||||
Sector = sector,
|
||||
CurrentLabel = currentLabel,
|
||||
AverageScore = finalAvgScore,
|
||||
WeightedScore = finalWeightedScore,
|
||||
AverageConfidence = finalAvgConfidence,
|
||||
TotalAnalysesCount = recentAnalyses.Count,
|
||||
PositiveCount = positiveCount,
|
||||
NegativeCount = negativeCount,
|
||||
NeutralCount = neutralCount,
|
||||
LatestKeyHighlight = latestHighlight,
|
||||
Trend = trend,
|
||||
LastUpdatedUtc = DateTime.UtcNow,
|
||||
Version = 1
|
||||
};
|
||||
_context.CompanySentiments.Add(newSummary);
|
||||
}
|
||||
else
|
||||
{
|
||||
existingSummary.Name = !string.IsNullOrWhiteSpace(companyName) ? companyName : existingSummary.Name;
|
||||
existingSummary.Sector = sector ?? existingSummary.Sector;
|
||||
existingSummary.CurrentLabel = currentLabel;
|
||||
existingSummary.AverageScore = finalAvgScore;
|
||||
existingSummary.WeightedScore = finalWeightedScore;
|
||||
existingSummary.AverageConfidence = finalAvgConfidence;
|
||||
existingSummary.TotalAnalysesCount = recentAnalyses.Count;
|
||||
existingSummary.PositiveCount = positiveCount;
|
||||
existingSummary.NegativeCount = negativeCount;
|
||||
existingSummary.NeutralCount = neutralCount;
|
||||
existingSummary.LatestKeyHighlight = latestHighlight ?? existingSummary.LatestKeyHighlight;
|
||||
existingSummary.Trend = trend;
|
||||
existingSummary.LastUpdatedUtc = DateTime.UtcNow;
|
||||
existingSummary.Version++;
|
||||
}
|
||||
|
||||
await _context.SaveChangesAsync(ct);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentDbService] Updated company summary for {Isin} ({Name}): {Label} (Weighted: {Score}, Trend: {Trend})", isin, companyName, currentLabel, finalWeightedScore, trend);
|
||||
return;
|
||||
}
|
||||
catch (DbUpdateConcurrencyException)
|
||||
{
|
||||
_context.ChangeTracker.Clear();
|
||||
if (attempt == maxRetries) throw;
|
||||
await Task.Delay(Random.Shared.Next(50, 150) * attempt, ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_context.ChangeTracker.Clear();
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentDbService] Error updating company summary for {Isin} (Attempt {Attempt})", isin, attempt);
|
||||
if (attempt == maxRetries) throw;
|
||||
await Task.Delay(100, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateSectorSummaryAsync(string sector, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var companySummaries = await _context.CompanySentiments
|
||||
.AsNoTracking()
|
||||
.Where(c => c.Sector == sector)
|
||||
.ToListAsync(ct);
|
||||
|
||||
if (companySummaries.Count == 0) return;
|
||||
|
||||
double avgScore = Math.Round(companySummaries.Average(c => c.WeightedScore), 4);
|
||||
int totalArticles = companySummaries.Sum(c => c.TotalAnalysesCount);
|
||||
int totalCompanies = companySummaries.Count;
|
||||
|
||||
string currentLabel = avgScore switch
|
||||
{
|
||||
> 0.15 => "POSITIVE",
|
||||
< -0.15 => "NEGATIVE",
|
||||
_ => "NEUTRAL"
|
||||
};
|
||||
|
||||
var existing = await _context.SectorSentiments.FirstOrDefaultAsync(s => s.Sector == sector, ct);
|
||||
if (existing == null)
|
||||
{
|
||||
_context.SectorSentiments.Add(new SectorSentimentSummaryEntity
|
||||
{
|
||||
Sector = sector,
|
||||
CurrentLabel = currentLabel,
|
||||
AverageScore = avgScore,
|
||||
TotalArticlesCount = totalArticles,
|
||||
TotalCompaniesCount = totalCompanies,
|
||||
LastUpdatedUtc = DateTime.UtcNow
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
existing.CurrentLabel = currentLabel;
|
||||
existing.AverageScore = avgScore;
|
||||
existing.TotalArticlesCount = totalArticles;
|
||||
existing.TotalCompaniesCount = totalCompanies;
|
||||
existing.LastUpdatedUtc = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
await _context.SaveChangesAsync(ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_context.ChangeTracker.Clear();
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentDbService] Failed to update sector summary for {Sector}", sector);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<CompanySentimentSummaryEntity?> GetCompanySentimentAsync(string isin, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isin)) return null;
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
return await _context.CompanySentiments
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(c => c.Isin == cleanIsin, ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IsinSentimentSummaryDto?> GetIsinSummaryDtoAsync(string isin, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isin)) return null;
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
|
||||
var summary = await _context.CompanySentiments
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(c => c.Isin == cleanIsin, ct);
|
||||
|
||||
if (summary == null) return null;
|
||||
|
||||
var recentAnalyses = await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.Where(a => a.Isin == cleanIsin)
|
||||
.OrderByDescending(a => a.PublishedAtUtc)
|
||||
.Take(15)
|
||||
.ToListAsync(ct);
|
||||
|
||||
return new IsinSentimentSummaryDto
|
||||
{
|
||||
Isin = summary.Isin,
|
||||
CompanyName = summary.Name,
|
||||
Sector = summary.Sector ?? "General",
|
||||
LastUpdated = summary.LastUpdatedUtc.ToString("o"),
|
||||
CurrentSummary = new IsinCurrentSummary
|
||||
{
|
||||
CompoundScore = summary.WeightedScore,
|
||||
SentimentLabel = summary.CurrentLabel,
|
||||
AvgConfidence = summary.AverageConfidence,
|
||||
TotalArticlesAnalyzed = summary.TotalAnalysesCount,
|
||||
PositiveArticles = summary.PositiveCount,
|
||||
NegativeArticles = summary.NegativeCount,
|
||||
NeutralArticles = summary.NeutralCount,
|
||||
Trend = summary.Trend,
|
||||
KeyHighlight = summary.LatestKeyHighlight
|
||||
},
|
||||
Analyses = recentAnalyses.Select(a => new IsinAnalysisEntry
|
||||
{
|
||||
AnalysisId = a.Id.ToString(),
|
||||
Timestamp = a.AnalyzedAtUtc.ToString("o"),
|
||||
Article = new IsinAnalysisArticleRef
|
||||
{
|
||||
ArticleId = a.ArticleId.ToString(),
|
||||
Title = a.Name,
|
||||
Source = "FinlyticNews",
|
||||
PublishedAt = a.PublishedAtUtc.ToString("o")
|
||||
},
|
||||
FinbertResult = new FinBertResultDto
|
||||
{
|
||||
Label = a.Label,
|
||||
CompoundScore = a.CompoundScore,
|
||||
Confidence = a.Confidence,
|
||||
Impact = a.Impact,
|
||||
KeyHighlight = a.KeyHighlight,
|
||||
Probabilities = new FinBertProbabilities
|
||||
{
|
||||
Positive = a.PositiveProbability,
|
||||
Negative = a.NegativeProbability,
|
||||
Neutral = a.NeutralProbability
|
||||
}
|
||||
},
|
||||
SummarySnippet = a.KeyHighlight ?? string.Empty
|
||||
}).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<CompanySentimentSummaryEntity>> GetAllCompanySentimentsAsync(
|
||||
int limit,
|
||||
int offset,
|
||||
string? sector = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var query = _context.CompanySentiments.AsNoTracking().AsQueryable();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(sector))
|
||||
{
|
||||
var cleanSector = sector.Trim();
|
||||
query = query.Where(c => c.Sector == cleanSector);
|
||||
}
|
||||
|
||||
return await query
|
||||
.OrderByDescending(c => c.LastUpdatedUtc)
|
||||
.Skip(Math.Max(0, offset))
|
||||
.Take(limit > 0 ? Math.Min(limit, 100) : 50)
|
||||
.ToListAsync(ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<SectorSentimentSummaryDto?> GetSectorSentimentAsync(string sector, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sector)) return null;
|
||||
var cleanSector = sector.Trim();
|
||||
|
||||
var entity = await _context.SectorSentiments
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(s => s.Sector == cleanSector, ct);
|
||||
|
||||
if (entity == null) return null;
|
||||
|
||||
var activeIsins = await _context.CompanySentiments
|
||||
.AsNoTracking()
|
||||
.Where(c => c.Sector == cleanSector)
|
||||
.Select(c => c.Isin)
|
||||
.Take(20)
|
||||
.ToListAsync(ct);
|
||||
|
||||
return new SectorSentimentSummaryDto
|
||||
{
|
||||
Sector = entity.Sector,
|
||||
LastUpdated = entity.LastUpdatedUtc.ToString("o"),
|
||||
CurrentSummary = new SectorCurrentSummary
|
||||
{
|
||||
CompoundScore = entity.AverageScore,
|
||||
SentimentLabel = entity.CurrentLabel,
|
||||
ActiveIsins = activeIsins,
|
||||
TotalArticlesAnalyzed = entity.TotalArticlesCount
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<ArticleSentimentEntity>> GetArticleSentimentsAsync(Guid articleId, CancellationToken ct = default)
|
||||
{
|
||||
if (articleId == Guid.Empty) return [];
|
||||
return await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.Where(a => a.ArticleId == articleId)
|
||||
.ToListAsync(ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IsinAnalysisEntry?> GetArticleSentimentEntryAsync(Guid articleId, CancellationToken ct = default)
|
||||
{
|
||||
if (articleId == Guid.Empty) return null;
|
||||
|
||||
var a = await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(x => x.ArticleId == articleId, ct);
|
||||
|
||||
if (a == null) return null;
|
||||
|
||||
return new IsinAnalysisEntry
|
||||
{
|
||||
AnalysisId = a.Id.ToString(),
|
||||
Timestamp = a.AnalyzedAtUtc.ToString("o"),
|
||||
Article = new IsinAnalysisArticleRef
|
||||
{
|
||||
ArticleId = a.ArticleId.ToString(),
|
||||
Title = a.Name,
|
||||
Source = "FinlyticNews",
|
||||
PublishedAt = a.PublishedAtUtc.ToString("o")
|
||||
},
|
||||
FinbertResult = new FinBertResultDto
|
||||
{
|
||||
Label = a.Label,
|
||||
CompoundScore = a.CompoundScore,
|
||||
Confidence = a.Confidence,
|
||||
Impact = a.Impact,
|
||||
KeyHighlight = a.KeyHighlight,
|
||||
Probabilities = new FinBertProbabilities
|
||||
{
|
||||
Positive = a.PositiveProbability,
|
||||
Negative = a.NegativeProbability,
|
||||
Neutral = a.NeutralProbability
|
||||
}
|
||||
},
|
||||
SummarySnippet = a.KeyHighlight ?? string.Empty
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<List<ArticleSentimentEntity>> GetSentimentTimelineAsync(string isin, int days, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isin)) return [];
|
||||
var cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
var cutoff = DateTime.UtcNow.AddDays(-Math.Max(1, days));
|
||||
|
||||
return await _context.ArticleSentiments
|
||||
.AsNoTracking()
|
||||
.Where(a => a.Isin == cleanIsin && a.PublishedAtUtc >= cutoff)
|
||||
.OrderByDescending(a => a.PublishedAtUtc)
|
||||
.ToListAsync(ct);
|
||||
}
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticCore.Dtos.News;
|
||||
using FinlyticCore.Dtos.Sentiment;
|
||||
using FinlyticCore.Services;
|
||||
using FinlyticSentiment.Util;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace FinlyticSentiment.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Defines the persistence contract for maintaining two-stage ISIN and Sector JSON sentiment summaries in the file system.
|
||||
/// </summary>
|
||||
public interface ISentimentStorageService
|
||||
{
|
||||
Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert);
|
||||
Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert);
|
||||
Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert);
|
||||
Task<IsinAnalysisEntry?> GetArticleSentimentAsync(string articleId);
|
||||
Task<IsinSentimentSummaryDto?> GetIsinSummaryAsync(string isin);
|
||||
}
|
||||
|
||||
public class SentimentStorageService : ISentimentStorageService
|
||||
{
|
||||
private static readonly ConcurrentDictionary<string, SemaphoreSlim> FileLocks = new();
|
||||
|
||||
private readonly IFinlyticLogger<SentimentStorageService> _finlyticLogger;
|
||||
private readonly string _basePath;
|
||||
private readonly JsonSerializerOptions _jsonOptions;
|
||||
|
||||
public SentimentStorageService(IConfiguration configuration, IFinlyticLogger<SentimentStorageService> finlyticLogger)
|
||||
{
|
||||
_finlyticLogger = finlyticLogger;
|
||||
_basePath = configuration["Storage:SummariesPath"] ?? "data/summaries";
|
||||
|
||||
Directory.CreateDirectory(Path.Combine(_basePath, "isin"));
|
||||
Directory.CreateDirectory(Path.Combine(_basePath, "sectors"));
|
||||
Directory.CreateDirectory(Path.Combine(_basePath, "articles"));
|
||||
|
||||
_jsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
|
||||
};
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert)
|
||||
{
|
||||
if (article == null || article.Id == Guid.Empty) return;
|
||||
|
||||
string cleanId = article.Id.ToString();
|
||||
string filePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
|
||||
|
||||
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
|
||||
var entry = new IsinAnalysisEntry
|
||||
{
|
||||
AnalysisId = $"sent_{Guid.NewGuid():N}",
|
||||
Timestamp = nowIso,
|
||||
Article = new IsinAnalysisArticleRef
|
||||
{
|
||||
ArticleId = cleanId,
|
||||
Title = article.Title,
|
||||
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
|
||||
Source = article.Author ?? "FinlyticNews"
|
||||
},
|
||||
FinbertResult = finbert,
|
||||
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
|
||||
};
|
||||
|
||||
var json = JsonSerializer.Serialize(entry, _jsonOptions);
|
||||
await File.WriteAllTextAsync(filePath, json);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Successfully saved article sentiment file: {Path}", filePath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to write article sentiment file: {Path}", filePath);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IsinAnalysisEntry?> GetArticleSentimentAsync(string articleId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(articleId)) return null;
|
||||
|
||||
string cleanId = articleId.Trim();
|
||||
string articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
|
||||
|
||||
if (File.Exists(articleFilePath))
|
||||
{
|
||||
var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync();
|
||||
try
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(articleFilePath);
|
||||
return JsonSerializer.Deserialize<IsinAnalysisEntry>(json, _jsonOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to read article sentiment file: {Path}", articleFilePath);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IsinSentimentSummaryDto?> GetIsinSummaryAsync(string isin)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isin)) return null;
|
||||
|
||||
string cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
|
||||
|
||||
if (!File.Exists(filePath)) return null;
|
||||
|
||||
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(filePath);
|
||||
return JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, _jsonOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Error reading ISIN summary file: {Path}", filePath);
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(isin)) return;
|
||||
|
||||
string cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
|
||||
|
||||
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
var analyses = new List<IsinAnalysisEntry>();
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
var existingJson = await File.ReadAllTextAsync(filePath);
|
||||
var existing = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(existingJson, _jsonOptions);
|
||||
if (existing?.Analyses != null)
|
||||
{
|
||||
analyses.AddRange(existing.Analyses);
|
||||
}
|
||||
}
|
||||
|
||||
string cleanArticleId = article.Id.ToString();
|
||||
analyses.RemoveAll(a => string.Equals(a.Article?.ArticleId, cleanArticleId, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
|
||||
var newEntry = new IsinAnalysisEntry
|
||||
{
|
||||
AnalysisId = $"sent_{Guid.NewGuid():N}",
|
||||
Timestamp = nowIso,
|
||||
Article = new IsinAnalysisArticleRef
|
||||
{
|
||||
ArticleId = cleanArticleId,
|
||||
Title = article.Title,
|
||||
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
|
||||
Source = article.Author ?? "FinlyticNews"
|
||||
},
|
||||
FinbertResult = finbert,
|
||||
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
|
||||
};
|
||||
|
||||
analyses.Add(newEntry);
|
||||
|
||||
var cutoff = DateTime.UtcNow.AddDays(-14);
|
||||
var validAnalyses = analyses.Where(a =>
|
||||
{
|
||||
if (DateTime.TryParse(a.Article?.PublishedAt ?? a.Timestamp, out var pubDate))
|
||||
{
|
||||
return pubDate >= cutoff;
|
||||
}
|
||||
return true;
|
||||
}).ToList();
|
||||
|
||||
var updatedAnalyses = validAnalyses.OrderByDescending(a => a.Article?.PublishedAt ?? a.Timestamp).Take(50).ToList();
|
||||
|
||||
double totalCompound = 0.0;
|
||||
double totalConf = 0.0;
|
||||
|
||||
foreach (var item in updatedAnalyses)
|
||||
{
|
||||
if (item.FinbertResult == null) continue;
|
||||
totalCompound += item.FinbertResult.CompoundScore;
|
||||
totalConf += item.FinbertResult.Confidence;
|
||||
}
|
||||
|
||||
int total = updatedAnalyses.Count;
|
||||
double avgCompound = total > 0 ? totalCompound / total : 0.0;
|
||||
double avgConf = total > 0 ? totalConf / total : 0.0;
|
||||
|
||||
string overallLabel = "NEUTRAL";
|
||||
if (avgCompound >= 0.15) overallLabel = "POSITIVE";
|
||||
else if (avgCompound <= -0.15) overallLabel = "NEGATIVE";
|
||||
|
||||
var summary = new IsinSentimentSummaryDto
|
||||
{
|
||||
Isin = cleanIsin,
|
||||
CompanyName = companyName,
|
||||
Sector = sector,
|
||||
LastUpdated = nowIso,
|
||||
CurrentSummary = new IsinCurrentSummary
|
||||
{
|
||||
CompoundScore = Math.Round(avgCompound, 4),
|
||||
SentimentLabel = overallLabel,
|
||||
AvgConfidence = Math.Round(avgConf, 4),
|
||||
TotalArticlesAnalyzed = total,
|
||||
Text = $"Synthesized sentiment across {total} articles is {overallLabel}."
|
||||
},
|
||||
Analyses = updatedAnalyses
|
||||
};
|
||||
|
||||
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
|
||||
await File.WriteAllTextAsync(filePath, outJson);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", filePath, updatedAnalyses.Count, avgCompound);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update ISIN summary for {Isin}", cleanIsin);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sector)) return;
|
||||
|
||||
string cleanSector = sector.Trim().ToLowerInvariant();
|
||||
string filePath = Path.Combine(_basePath, "sectors", $"{cleanSector}.json");
|
||||
|
||||
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
||||
await fileLock.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
var analyses = new List<SectorAnalysisEntry>();
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
var existingJson = await File.ReadAllTextAsync(filePath);
|
||||
var existing = JsonSerializer.Deserialize<SectorSentimentSummaryDto>(existingJson, _jsonOptions);
|
||||
if (existing?.Analyses != null)
|
||||
{
|
||||
analyses.AddRange(existing.Analyses);
|
||||
}
|
||||
}
|
||||
|
||||
string cleanIsin = isin.Trim().ToUpperInvariant();
|
||||
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
|
||||
|
||||
analyses.RemoveAll(a => string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase) && string.Equals(a.RelatedIsin, cleanIsin, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
analyses.Add(new SectorAnalysisEntry
|
||||
{
|
||||
AnalysisId = $"sec_{Guid.NewGuid():N}",
|
||||
Timestamp = nowIso,
|
||||
RelatedIsin = cleanIsin,
|
||||
ArticleId = articleId,
|
||||
FinbertResult = finbert
|
||||
});
|
||||
|
||||
var cutoff = DateTime.UtcNow.AddDays(-14);
|
||||
var updatedAnalyses = analyses.Where(a =>
|
||||
{
|
||||
if (DateTime.TryParse(a.Timestamp, out var ts))
|
||||
{
|
||||
return ts >= cutoff;
|
||||
}
|
||||
return true;
|
||||
}).OrderByDescending(a => a.Timestamp).Take(100).ToList();
|
||||
|
||||
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrEmpty(i)).Distinct().ToList();
|
||||
double totalSectorCompound = updatedAnalyses.Sum(s => s.FinbertResult.CompoundScore);
|
||||
double avgSectorCompound = updatedAnalyses.Count > 0 ? totalSectorCompound / updatedAnalyses.Count : 0.0;
|
||||
|
||||
string sectorLabel = "NEUTRAL";
|
||||
if (avgSectorCompound >= 0.15) sectorLabel = "POSITIVE";
|
||||
else if (avgSectorCompound <= -0.15) sectorLabel = "NEGATIVE";
|
||||
|
||||
var summary = new SectorSentimentSummaryDto
|
||||
{
|
||||
Sector = sector,
|
||||
LastUpdated = nowIso,
|
||||
CurrentSummary = new SectorCurrentSummary
|
||||
{
|
||||
CompoundScore = Math.Round(avgSectorCompound, 4),
|
||||
SentimentLabel = sectorLabel,
|
||||
ActiveIsins = activeIsins,
|
||||
Text = $"Sector {sector} aggregate sentiment: {sectorLabel}."
|
||||
},
|
||||
Analyses = updatedAnalyses
|
||||
};
|
||||
|
||||
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
|
||||
await File.WriteAllTextAsync(filePath, outJson);
|
||||
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated Sector summary file: {Path} (Active ISINs: {Count})", filePath, activeIsins.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update Sector summary for {Sector}", sector);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fileLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using FinlyticSentiment.Database;
|
||||
using FinlyticSentiment.Entities;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FinlyticSentiment.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for reading and persisting FinlyticSentiment runtime configuration settings in PostgreSQL.
|
||||
/// </summary>
|
||||
public interface ISettingsDbService
|
||||
{
|
||||
/// <summary>
|
||||
/// Retrieves current sentiment settings from PostgreSQL database, seeding defaults if empty.
|
||||
/// </summary>
|
||||
Task<SentimentSettingsEntity> GetSettingsAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Persists updated settings entity to PostgreSQL.
|
||||
/// </summary>
|
||||
Task<SentimentSettingsEntity> SaveSettingsAsync(SentimentSettingsEntity settings);
|
||||
|
||||
/// <summary>
|
||||
/// Updates settings from a key-value dictionary received via Admin Panel MQTT events.
|
||||
/// </summary>
|
||||
Task UpdateSettingsFromDictionaryAsync(Dictionary<string, string> dictionary);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EF Core PostgreSQL implementation of <see cref="ISettingsDbService"/>.
|
||||
/// </summary>
|
||||
public class SettingsDbService : ISettingsDbService
|
||||
{
|
||||
private readonly SentimentDbContext _context;
|
||||
private readonly ILogger<SettingsDbService> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SettingsDbService"/> class.
|
||||
/// </summary>
|
||||
public SettingsDbService(SentimentDbContext context, ILogger<SettingsDbService> logger)
|
||||
{
|
||||
_context = context;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves current sentiment settings from PostgreSQL database.
|
||||
/// </summary>
|
||||
public async Task<SentimentSettingsEntity> GetSettingsAsync()
|
||||
{
|
||||
var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync();
|
||||
if (settings == null)
|
||||
{
|
||||
settings = new SentimentSettingsEntity { Id = Guid.NewGuid() };
|
||||
_context.Settings.Add(settings);
|
||||
await _context.SaveChangesAsync();
|
||||
_context.ChangeTracker.Clear();
|
||||
}
|
||||
return settings;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Persists updated settings entity to PostgreSQL.
|
||||
/// </summary>
|
||||
public async Task<SentimentSettingsEntity> SaveSettingsAsync(SentimentSettingsEntity settings)
|
||||
{
|
||||
var existing = await _context.Settings.FirstOrDefaultAsync();
|
||||
if (existing == null)
|
||||
{
|
||||
if (settings.Id == Guid.Empty) settings.Id = Guid.NewGuid();
|
||||
_context.Settings.Add(settings);
|
||||
}
|
||||
else
|
||||
{
|
||||
existing.MinConfidenceScore = settings.MinConfidenceScore;
|
||||
existing.MaxBatchSize = settings.MaxBatchSize;
|
||||
existing.SweepIntervalMinutes = settings.SweepIntervalMinutes;
|
||||
existing.GermanWebhookUrl = settings.GermanWebhookUrl;
|
||||
existing.EnglishWebhookUrl = settings.EnglishWebhookUrl;
|
||||
existing.UpdatedAt = settings.UpdatedAt;
|
||||
_context.Settings.Update(existing);
|
||||
}
|
||||
await _context.SaveChangesAsync();
|
||||
return settings;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates settings from a key-value dictionary.
|
||||
/// </summary>
|
||||
public async Task UpdateSettingsFromDictionaryAsync(Dictionary<string, string> dictionary)
|
||||
{
|
||||
var settings = await GetSettingsAsync();
|
||||
|
||||
foreach (var (key, value) in dictionary)
|
||||
{
|
||||
if (string.Equals(key, "MinConfidenceScore", StringComparison.OrdinalIgnoreCase) && double.TryParse(value, out var mcs))
|
||||
settings.MinConfidenceScore = mcs;
|
||||
else if (string.Equals(key, "MaxBatchSize", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var mbs))
|
||||
settings.MaxBatchSize = mbs;
|
||||
else if (string.Equals(key, "SweepIntervalMinutes", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var sim))
|
||||
settings.SweepIntervalMinutes = sim;
|
||||
else if (string.Equals(key, "GermanWebhookUrl", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(value))
|
||||
settings.GermanWebhookUrl = value.Trim();
|
||||
else if (string.Equals(key, "EnglishWebhookUrl", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(value))
|
||||
settings.EnglishWebhookUrl = value.Trim();
|
||||
}
|
||||
|
||||
settings.UpdatedAt = DateTime.UtcNow;
|
||||
await SaveSettingsAsync(settings);
|
||||
_logger.LogInformation("[{Channel}] Successfully updated {Count} sentiment settings in PostgreSQL database.", "SentimentChannel", dictionary.Count);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user