Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sentiment): dynamic settings, IFinlyticLogger, live log streaming, and EF migration

Browse Source
This commit is contained in:
Kleidukos
2026-08-15 21:30:12 +02:00
parent 1522c3480f
commit 62e030e2cf
10 changed files with 519 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticSentiment/Services/FinBertAnalyzerService.cs
+12 -21
View File
@@ -1,9 +1,13 @@
using System;
using System.Net.Http;
using System.Net.Http.Json;
using System.Text.Json;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.DependencyInjection;
namespace FinlyticSentiment.Services;
@@ -15,8 +19,6 @@ public interface IFinBertAnalyzerService
/// <summary>
/// Analyzes a news article using language-targeted FinBERT webhooks and returns structured metrics.
/// </summary>
/// <param name="article">The news article DTO to evaluate.</param>
/// <returns>A task returning the FinBERT sentiment analysis result.</returns>
Task<FinBertResultDto?> AnalyzeArticleAsync(NewsArticleDto article);
}
@@ -27,22 +29,16 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
{
private readonly HttpClient _httpClient;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<FinBertAnalyzerService> _logger;
private readonly IFinlyticLogger<FinBertAnalyzerService> _finlyticLogger;
/// <summary>
/// Initializes a new instance of the <see cref="FinBertAnalyzerService"/> class.
/// </summary>
/// <param name="httpClient">The HTTP client instance.</param>
/// <param name="scopeFactory">The service scope factory for DB access.</param>
/// <param name="logger">The logging channel.</param>
public FinBertAnalyzerService(
HttpClient httpClient,
IServiceScopeFactory scopeFactory,
ILogger<FinBertAnalyzerService> logger)
IFinlyticLogger<FinBertAnalyzerService> finlyticLogger)
{
_httpClient = httpClient;
_scopeFactory = scopeFactory;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <summary>
@@ -64,7 +60,7 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
minConfidence = settings.MinConfidenceScore;
}
_logger.LogInformation("[{Channel}] Analyzing article (ID: {Id}, Lang: {Lang}) via webhook: {Url} (MinConf: {Conf})", "SentimentChannel", article.Id, article.Language ?? "de", targetUrl, minConfidence);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] Analyzing article (ID: {Id}, Lang: {Lang}) via webhook: {Url} (MinConf: {Conf})", article.Id, article.Language ?? "de", targetUrl, minConfidence);
var requestBody = new
{
@@ -88,13 +84,11 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
using var doc = JsonDocument.Parse(content);
var root = doc.RootElement;
// Handle array response if n8n returns an array of items (e.g. [{ "json": { ... } }])
if (root.ValueKind == JsonValueKind.Array && root.GetArrayLength() > 0)
{
root = root[0];
}
// Unwrap n8n wrapper objects: "json", "output", "data", "result", "body"
if (root.ValueKind == JsonValueKind.Object)
{
if (root.TryGetProperty("json", out var jsonChild) && jsonChild.ValueKind == JsonValueKind.Object)
@@ -134,7 +128,6 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
neu = GetDoubleProp(probsElem, "neutral") ?? neu;
}
// Normalize German vs English labels
string label = rawLabel.Trim().ToUpperInvariant() switch
{
"POSITIV" or "POSITIVE" => "POSITIVE",
@@ -142,7 +135,6 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
_ => "NEUTRAL"
};
// If compoundScore is 0 but probabilities or label indicate sentiment, compute compoundScore
if (Math.Abs(compoundScore) < 0.001)
{
if (pos > 0 || neg > 0)
@@ -173,15 +165,14 @@ public class FinBertAnalyzerService : IFinBertAnalyzerService
SummarySnippet = snippet
};
}
}
}
_logger.LogWarning("[{Channel}] n8n Webhook returned non-success status: {StatusCode}. Falling back to rule analyzer.", "SentimentChannel", response.StatusCode);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[FinBertAnalyzerService] n8n Webhook returned non-success status: {StatusCode}.", response.StatusCode);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to call n8n sentiment webhook. Executing fallback sentiment analyzer.", "SentimentChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[FinBertAnalyzerService] Failed to call n8n sentiment webhook.");
}
return null;
FinlyticSentiment/Services/SentimentBackgroundService.cs
+25 -56
View File
@@ -1,9 +1,13 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace FinlyticSentiment.Services;
@@ -17,53 +21,45 @@ public class SentimentBackgroundService : BackgroundService
private readonly IFinBertAnalyzerService _analyzer;
private readonly ISentimentStorageService _storage;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<SentimentBackgroundService> _logger;
private readonly IFinlyticLogger<SentimentBackgroundService> _finlyticLogger;
// In-Memory Mutex / Cache zur Vermeidung doppelter Verarbeitung (Race Conditions zwischen Broadcast & Sweep)
private static readonly ConcurrentDictionary<Guid, byte> ProcessingArticles = new();
/// <summary>
/// Initializes a new instance of the <see cref="SentimentBackgroundService"/> class.
/// </summary>
public SentimentBackgroundService(
SentimentMqttClient mqttClient,
IFinBertAnalyzerService analyzer,
ISentimentStorageService storage,
IServiceScopeFactory scopeFactory,
ILogger<SentimentBackgroundService> logger)
IFinlyticLogger<SentimentBackgroundService> finlyticLogger)
{
_mqttClient = mqttClient;
_analyzer = analyzer;
_storage = storage;
_scopeFactory = scopeFactory;
_logger = logger;
_finlyticLogger = finlyticLogger;
}
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("[{Channel}] FinlyticSentiment Background Service started.", "SentimentChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinlyticSentiment Background Service started.");
// Realtime Broadcast Event registrieren (Echtzeit-Artikel)
_mqttClient.OnArticleReceived += async (article) =>
{
await ProcessSingleArticleAsync(article, stoppingToken);
};
// Kurze Initialisierungs-Verzögerung für die MQTT-Verbindung
await Task.Delay(3000, stoppingToken);
while (!stoppingToken.IsCancellationRequested)
{
int maxBatchSize;
int sweepIntervalMinutes;
int maxBatchSize = 10;
int sweepIntervalMinutes = 5;
using (var scope = _scopeFactory.CreateScope())
{
var settingsDb = scope.ServiceProvider.GetRequiredService<ISettingsDbService>();
var settings = await settingsDb.GetSettingsAsync();
maxBatchSize = settings.MaxBatchSize;
sweepIntervalMinutes = settings.SweepIntervalMinutes;
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
maxBatchSize = await settings.GetSettingAsync(SettingKeys.MaxBatchSize, stoppingToken);
}
var interval = TimeSpan.FromMinutes(Math.Max(1, sweepIntervalMinutes));
@@ -74,19 +70,15 @@ public class SentimentBackgroundService : BackgroundService
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Normales Beenden beim Stoppen des Hosts
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Unhandled exception encountered during sentiment sweep cycle.",
"SentimentChannel");
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentBackgroundService] Unhandled exception encountered during sentiment sweep cycle.");
}
_logger.LogInformation("[{Channel}] Waiting {Minutes} minute(s) until next sentiment sweep...",
"SentimentChannel", interval.TotalMinutes);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Waiting {Minutes} minute(s) until next sentiment sweep...", interval.TotalMinutes);
// Verwendet PeriodicTimer oder CancellationToken-resistenten Delay
using var timer = new PeriodicTimer(interval);
try
{
@@ -98,27 +90,21 @@ public class SentimentBackgroundService : BackgroundService
}
}
_logger.LogInformation("[{Channel}] FinlyticSentiment Background Service is shutting down gracefully.",
"SentimentChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinlyticSentiment Background Service is shutting down gracefully.");
}
/// <summary>
/// Performs a single batch sweep of pending news articles.
/// </summary>
private async Task PerformSentimentSweepAsync(int maxBatchSize, CancellationToken cancellationToken)
{
_logger.LogInformation("[{Channel}] Starting sentiment sweep for pending news articles (Limit: {Limit})...",
"SentimentChannel", maxBatchSize);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Starting sentiment sweep for pending news articles (Limit: {Limit})...", maxBatchSize);
List<NewsArticleDto> pendingArticles = await _mqttClient.GetPendingArticlesAsync(limit: maxBatchSize);
if (pendingArticles.Count == 0)
{
_logger.LogInformation("[{Channel}] No pending news articles found in FinlyticNews.", "SentimentChannel");
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] No pending news articles found in FinlyticNews.");
return;
}
_logger.LogInformation("[{Channel}] Retrieved {Count} pending article(s) for FinBERT sentiment evaluation.",
"SentimentChannel", pendingArticles.Count);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Retrieved {Count} pending article(s) for FinBERT sentiment evaluation.", pendingArticles.Count);
foreach (var article in pendingArticles)
{
@@ -127,42 +113,32 @@ public class SentimentBackgroundService : BackgroundService
}
}
/// <summary>
/// Processes FinBERT sentiment analysis for a single article, updates two-stage JSON summaries, and notifies FinlyticNews of status update.
/// </summary>
private async Task ProcessSingleArticleAsync(NewsArticleDto article, CancellationToken cancellationToken = default)
{
if (article == null || article.Id == Guid.Empty) return;
// Deduplizierung: Verhindert, dass derselbe Artikel zeitgleich im Sweep & im Broadcast verarbeitet wird
if (!ProcessingArticles.TryAdd(article.Id, 0))
{
_logger.LogDebug("[{Channel}] Article {Id} is already being processed. Skipping duplicate run.",
"SentimentChannel", article.Id);
await _finlyticLogger.LogDebugAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Article {Id} is already being processed. Skipping duplicate run.", article.Id);
return;
}
try
{
_logger.LogInformation("[{Channel}] Evaluating FinBERT sentiment for article: '{Title}' (ID: {Id})",
"SentimentChannel", article.Title, article.Id);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Evaluating FinBERT sentiment for article: '{Title}' (ID: {Id})", article.Title, article.Id);
var finbert = await _analyzer.AnalyzeArticleAsync(article);
if (finbert == null)
{
_logger.LogWarning(
"[{Channel}] FinBERT analysis returned NULL for article {Id} ('{Title}'). Aborting processing for this run.",
"SentimentChannel", article.Id, article.Title);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] FinBERT analysis returned NULL for article {Id} ('{Title}'). Aborting processing for this run.", article.Id, article.Title);
return;
}
if (cancellationToken.IsCancellationRequested) return;
// 1. Artikel-Level Sentiment speichern
await _storage.SaveArticleSentimentAsync(article, finbert);
// 2. ISIN- & Sektor-Summaries aktualisieren
if (article.MatchedAssets != null && article.MatchedAssets.Count > 0)
{
foreach (var asset in article.MatchedAssets)
@@ -187,29 +163,22 @@ public class SentimentBackgroundService : BackgroundService
if (cancellationToken.IsCancellationRequested) return;
// 3. FinlyticNews über Erfolg informieren
bool updated = await _mqttClient.UpdateArticleStatusAsync(article.Id, "Analyzed");
if (updated)
{
_logger.LogInformation(
"[{Channel}] Article sentiment processed and status set to 'Analyzed' in FinlyticNews: {Title} (ID: {Id}) -> {Label}",
"SentimentChannel", article.Title, article.Id, finbert.Label);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Article sentiment processed and status set to 'Analyzed' in FinlyticNews: {Title} (ID: {Id}) -> {Label}", article.Title, article.Id, finbert.Label);
}
else
{
_logger.LogWarning(
"[{Channel}] Failed to confirm status update to 'Analyzed' in FinlyticNews for article: {Id}",
"SentimentChannel", article.Id);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentBackgroundService] Failed to confirm status update to 'Analyzed' in FinlyticNews for article: {Id}", article.Id);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error processing sentiment for article: {Id} ({Title})",
"SentimentChannel", article.Id, article.Title);
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentBackgroundService] Error processing sentiment for article: {Id} ({Title})", article.Id, article.Title);
}
finally
{
// Lock nach der Verarbeitung immer freigeben
ProcessingArticles.TryRemove(article.Id, out _);
}
}
FinlyticSentiment/Services/SentimentStorageService.cs
+121 -171
View File
@@ -1,10 +1,17 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace FinlyticSentiment.Services;
@@ -13,40 +20,10 @@ namespace FinlyticSentiment.Services;
/// </summary>
public interface ISentimentStorageService
{
/// <summary>
/// Appends a new FinBERT analysis event to the ISIN summary file and recalculates the current aggregate metrics.
/// </summary>
/// <param name="isin">The asset ISIN code.</param>
/// <param name="companyName">The name of the asset company.</param>
/// <param name="sector">The sector associated with the asset.</param>
/// <param name="article">The analyzed article DTO.</param>
/// <param name="finbert">The FinBERT sentiment result.</param>
/// <returns>A task representing the file update operation.</returns>
Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert);
/// <summary>
/// Appends a new FinBERT analysis event to the Sector summary file and recalculates the sector aggregate metrics.
/// </summary>
/// <param name="sector">The target sector name.</param>
/// <param name="isin">The asset ISIN code triggering the sector update.</param>
/// <param name="articleId">The unique news article identifier.</param>
/// <param name="finbert">The FinBERT sentiment result.</param>
/// <returns>A task representing the file update operation.</returns>
Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert);
/// <summary>
/// Persists an article's sentiment analysis directly in data/summaries/articles/{articleId}.json.
/// </summary>
Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert);
/// <summary>
/// Retrieves the sentiment analysis entry for a specific news article.
/// </summary>
Task<IsinAnalysisEntry?> GetArticleSentimentAsync(string articleId);
/// <summary>
/// Retrieves the aggregate sentiment summary for a specific asset ISIN.
/// </summary>
Task<IsinSentimentSummaryDto?> GetIsinSummaryAsync(string isin);
}
@@ -54,13 +31,13 @@ public class SentimentStorageService : ISentimentStorageService
{
private static readonly ConcurrentDictionary<string, SemaphoreSlim> FileLocks = new();
private readonly ILogger<SentimentStorageService> _logger;
private readonly IFinlyticLogger<SentimentStorageService> _finlyticLogger;
private readonly string _basePath;
private readonly JsonSerializerOptions _jsonOptions;
public SentimentStorageService(IConfiguration configuration, ILogger<SentimentStorageService> logger)
public SentimentStorageService(IConfiguration configuration, IFinlyticLogger<SentimentStorageService> finlyticLogger)
{
_logger = logger;
_finlyticLogger = finlyticLogger;
_basePath = configuration["Storage:SummariesPath"] ?? "data/summaries";
Directory.CreateDirectory(Path.Combine(_basePath, "isin"));
@@ -88,30 +65,28 @@ public class SentimentStorageService : ISentimentStorageService
try
{
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
string analysisId = $"sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}";
var entry = new IsinAnalysisEntry
{
AnalysisId = analysisId,
AnalysisId = $"sent_{Guid.NewGuid():N}",
Timestamp = nowIso,
Article = new IsinAnalysisArticleRef
{
ArticleId = cleanId,
Title = article.Title ?? "",
Source = article.Author ?? "FinlyticNews",
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ")
Title = article.Title,
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
Source = article.Author ?? "FinlyticNews"
},
FinbertResult = finbert,
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? article.Title ?? ""
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
};
string json = JsonSerializer.Serialize(entry, _jsonOptions);
var json = JsonSerializer.Serialize(entry, _jsonOptions);
await File.WriteAllTextAsync(filePath, json);
_logger.LogInformation("[{Channel}] Successfully saved article sentiment file: {Path}", "SentimentChannel", filePath);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Successfully saved article sentiment file: {Path}", filePath);
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Failed to write article sentiment file: {Path}", "SentimentChannel", filePath);
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to write article sentiment file: {Path}", filePath);
}
finally
{
@@ -124,10 +99,9 @@ public class SentimentStorageService : ISentimentStorageService
{
if (string.IsNullOrWhiteSpace(articleId)) return null;
var cleanId = articleId.Trim();
var articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
string cleanId = articleId.Trim();
string articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
// 1. Primärer Lookup
if (File.Exists(articleFilePath))
{
var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1));
@@ -139,7 +113,7 @@ public class SentimentStorageService : ISentimentStorageService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[{Channel}] Failed to read article sentiment file: {Path}", "SentimentChannel", articleFilePath);
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to read article sentiment file: {Path}", articleFilePath);
}
finally
{
@@ -147,22 +121,6 @@ public class SentimentStorageService : ISentimentStorageService
}
}
// 2. Fallback in ISIN-Dateien
var dirPath = Path.Combine(_basePath, "isin");
if (!Directory.Exists(dirPath)) return null;
foreach (var file in Directory.GetFiles(dirPath, "*.json"))
{
try
{
var json = await File.ReadAllTextAsync(file);
var doc = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, _jsonOptions);
var match = doc?.Analyses?.FirstOrDefault(a => string.Equals(a.Article?.ArticleId?.Trim(), cleanId, StringComparison.OrdinalIgnoreCase));
if (match != null) return match;
}
catch { }
}
return null;
}
@@ -171,8 +129,8 @@ public class SentimentStorageService : ISentimentStorageService
{
if (string.IsNullOrWhiteSpace(isin)) return null;
var cleanIsin = isin.Trim();
var filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
string cleanIsin = isin.Trim().ToUpperInvariant();
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
if (!File.Exists(filePath)) return null;
@@ -186,7 +144,7 @@ public class SentimentStorageService : ISentimentStorageService
}
catch (Exception ex)
{
_logger.LogError(ex, "[{Channel}] Error reading ISIN summary file: {Path}", "SentimentChannel", filePath);
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Error reading ISIN summary file: {Path}", filePath);
return null;
}
finally
@@ -198,9 +156,9 @@ public class SentimentStorageService : ISentimentStorageService
/// <inheritdoc />
public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert)
{
if (string.IsNullOrWhiteSpace(isin) || article == null) return;
if (string.IsNullOrWhiteSpace(isin)) return;
string cleanIsin = isin.Trim();
string cleanIsin = isin.Trim().ToUpperInvariant();
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
@@ -208,87 +166,92 @@ public class SentimentStorageService : ISentimentStorageService
try
{
IsinSentimentSummaryDto isinDoc;
var analyses = new List<IsinAnalysisEntry>();
if (File.Exists(filePath))
{
try
var existingJson = await File.ReadAllTextAsync(filePath);
var existing = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(existingJson, _jsonOptions);
if (existing?.Analyses != null)
{
string json = await File.ReadAllTextAsync(filePath);
isinDoc = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, _jsonOptions) ?? new IsinSentimentSummaryDto { Isin = cleanIsin };
}
catch
{
isinDoc = new IsinSentimentSummaryDto { Isin = cleanIsin };
analyses.AddRange(existing.Analyses);
}
}
else
{
isinDoc = new IsinSentimentSummaryDto { Isin = cleanIsin };
}
string cleanArticleId = article.Id.ToString();
analyses.RemoveAll(a => string.Equals(a.Article?.ArticleId, cleanArticleId, StringComparison.OrdinalIgnoreCase));
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
string analysisId = $"sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}";
var newEntry = new IsinAnalysisEntry
{
AnalysisId = analysisId,
AnalysisId = $"sent_{Guid.NewGuid():N}",
Timestamp = nowIso,
Article = new IsinAnalysisArticleRef
{
ArticleId = article.Id.ToString(),
Title = article.Title ?? "",
Source = article.Author ?? "FinlyticNews",
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ")
ArticleId = cleanArticleId,
Title = article.Title,
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
Source = article.Author ?? "FinlyticNews"
},
FinbertResult = finbert,
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? article.Title ?? ""
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
};
// Duplikat-Bereinigung: Falls Artikel bereits existiert, alten Eintrag entfernen!
var updatedAnalyses = isinDoc.Analyses?
.Where(a => !string.Equals(a.Article?.ArticleId, article.Id.ToString(), StringComparison.OrdinalIgnoreCase))
.ToList() ?? new List<IsinAnalysisEntry>();
analyses.Add(newEntry);
// Neuen Eintrag oben einfügen
updatedAnalyses.Insert(0, newEntry);
// Capping: Maximal die letzten 100 Analysen aufheben (verhindert gigantische JSON-Dateien)
if (updatedAnalyses.Count > 100)
var cutoff = DateTime.UtcNow.AddDays(-14);
var validAnalyses = analyses.Where(a =>
{
updatedAnalyses = updatedAnalyses.Take(100).ToList();
if (DateTime.TryParse(a.Article?.PublishedAt ?? a.Timestamp, out var pubDate))
{
return pubDate >= cutoff;
}
return true;
}).ToList();
var updatedAnalyses = validAnalyses.OrderByDescending(a => a.Article?.PublishedAt ?? a.Timestamp).Take(50).ToList();
double totalCompound = 0.0;
double totalConf = 0.0;
foreach (var item in updatedAnalyses)
{
if (item.FinbertResult == null) continue;
totalCompound += item.FinbertResult.CompoundScore;
totalConf += item.FinbertResult.Confidence;
}
double avgCompound = updatedAnalyses.Average(a => a.FinbertResult.CompoundScore);
double avgConf = updatedAnalyses.Average(a => a.FinbertResult.Confidence);
string label = CalculateLabel(avgCompound);
int total = updatedAnalyses.Count;
double avgCompound = total > 0 ? totalCompound / total : 0.0;
double avgConf = total > 0 ? totalConf / total : 0.0;
string textSummary = label switch
{
"POSITIVE" => $"Die Stimmungsanalyse zeigt einen weiterhin positiven Trend ({avgCompound:F2}). Hauptursache sind positive Berichte und starke Markt-Signale.",
"NEGATIVE" => $"Die Stimmungsanalyse deutet auf einen verhaltenen bis negativen Trend hin ({avgCompound:F2}). Auf kritische Markt-Berichte sollte geachtet werden.",
_ => $"Das Gesamtsentiment ist neutral ({avgCompound:F2}). Ausgewogene Signale aus der aktuellen Berichterstattung."
};
string overallLabel = "NEUTRAL";
if (avgCompound >= 0.15) overallLabel = "POSITIVE";
else if (avgCompound <= -0.15) overallLabel = "NEGATIVE";
var updatedDoc = isinDoc with
var summary = new IsinSentimentSummaryDto
{
Isin = cleanIsin,
CompanyName = !string.IsNullOrWhiteSpace(companyName) ? companyName : isinDoc.CompanyName,
Sector = !string.IsNullOrWhiteSpace(sector) ? sector : isinDoc.Sector,
CompanyName = companyName,
Sector = sector,
LastUpdated = nowIso,
CurrentSummary = new IsinCurrentSummary
{
CompoundScore = Math.Round(avgCompound, 2),
SentimentLabel = label,
AvgConfidence = Math.Round(avgConf, 2),
TotalArticlesAnalyzed = updatedAnalyses.Count,
Text = textSummary
CompoundScore = Math.Round(avgCompound, 4),
SentimentLabel = overallLabel,
AvgConfidence = Math.Round(avgConf, 4),
TotalArticlesAnalyzed = total,
Text = $"Synthesized sentiment across {total} articles is {overallLabel}."
},
Analyses = updatedAnalyses
};
await File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(updatedDoc, _jsonOptions));
_logger.LogInformation("[{Channel}] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", "SentimentChannel", filePath, updatedAnalyses.Count, avgCompound);
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
await File.WriteAllTextAsync(filePath, outJson);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", filePath, updatedAnalyses.Count, avgCompound);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update ISIN summary for {Isin}", cleanIsin);
}
finally
{
@@ -301,95 +264,82 @@ public class SentimentStorageService : ISentimentStorageService
{
if (string.IsNullOrWhiteSpace(sector)) return;
string sanitizedSector = string.Concat(sector.Split(Path.GetInvalidFileNameChars())).Trim();
string filePath = Path.Combine(_basePath, "sectors", $"{sanitizedSector}.json");
string cleanSector = sector.Trim().ToLowerInvariant();
string filePath = Path.Combine(_basePath, "sectors", $"{cleanSector}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
SectorSentimentSummaryDto sectorDoc;
var analyses = new List<SectorAnalysisEntry>();
if (File.Exists(filePath))
{
try
var existingJson = await File.ReadAllTextAsync(filePath);
var existing = JsonSerializer.Deserialize<SectorSentimentSummaryDto>(existingJson, _jsonOptions);
if (existing?.Analyses != null)
{
string json = await File.ReadAllTextAsync(filePath);
sectorDoc = JsonSerializer.Deserialize<SectorSentimentSummaryDto>(json, _jsonOptions) ?? new SectorSentimentSummaryDto { Sector = sector };
analyses.AddRange(existing.Analyses);
}
catch
{
sectorDoc = new SectorSentimentSummaryDto { Sector = sector };
}
}
else
{
sectorDoc = new SectorSentimentSummaryDto { Sector = sector };
}
string cleanIsin = isin.Trim().ToUpperInvariant();
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
string analysisId = $"sec_sent_{DateTime.UtcNow:yyyyMMdd}_{Random.Shared.Next(100, 999)}";
var newEntry = new SectorAnalysisEntry
analyses.RemoveAll(a => string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase) && string.Equals(a.RelatedIsin, cleanIsin, StringComparison.OrdinalIgnoreCase));
analyses.Add(new SectorAnalysisEntry
{
AnalysisId = analysisId,
AnalysisId = $"sec_{Guid.NewGuid():N}",
Timestamp = nowIso,
RelatedIsin = isin,
RelatedIsin = cleanIsin,
ArticleId = articleId,
FinbertResult = finbert
};
});
// Duplikate bereinigen (selber Artikel für denselben Sektor)
var updatedAnalyses = sectorDoc.Analyses?
.Where(a => !string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase))
.ToList() ?? new List<SectorAnalysisEntry>();
updatedAnalyses.Insert(0, newEntry);
if (updatedAnalyses.Count > 100)
var cutoff = DateTime.UtcNow.AddDays(-14);
var updatedAnalyses = analyses.Where(a =>
{
updatedAnalyses = updatedAnalyses.Take(100).ToList();
}
if (DateTime.TryParse(a.Timestamp, out var ts))
{
return ts >= cutoff;
}
return true;
}).OrderByDescending(a => a.Timestamp).Take(100).ToList();
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrWhiteSpace(i)).Distinct().ToList();
double avgCompound = updatedAnalyses.Average(a => a.FinbertResult.CompoundScore);
string label = CalculateLabel(avgCompound);
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrEmpty(i)).Distinct().ToList();
double totalSectorCompound = updatedAnalyses.Sum(s => s.FinbertResult.CompoundScore);
double avgSectorCompound = updatedAnalyses.Count > 0 ? totalSectorCompound / updatedAnalyses.Count : 0.0;
string overviewText = label switch
{
"POSITIVE" => $"Der Sektor {sector} tendiert insgesamt positiv. Starke Einzelergebnisse stützen den Trend.",
"NEGATIVE" => $"Der Sektor {sector} verzeichnet dämpfende Sentiment-Signale.",
_ => $"Der Sektor {sector} zeigt ein ausgewogenes neutrales Gesamtbild."
};
string sectorLabel = "NEUTRAL";
if (avgSectorCompound >= 0.15) sectorLabel = "POSITIVE";
else if (avgSectorCompound <= -0.15) sectorLabel = "NEGATIVE";
var updatedDoc = sectorDoc with
var summary = new SectorSentimentSummaryDto
{
Sector = sector,
LastUpdated = nowIso,
CurrentSummary = new SectorCurrentSummary
{
CompoundScore = Math.Round(avgCompound, 2),
SentimentLabel = label,
CompoundScore = Math.Round(avgSectorCompound, 4),
SentimentLabel = sectorLabel,
ActiveIsins = activeIsins,
Text = overviewText
Text = $"Sector {sector} aggregate sentiment: {sectorLabel}."
},
Analyses = updatedAnalyses
};
await File.WriteAllTextAsync(filePath, JsonSerializer.Serialize(updatedDoc, _jsonOptions));
_logger.LogInformation("[{Channel}] Updated Sector summary file: {Path} (Active ISINs: {Count})", "SentimentChannel", filePath, activeIsins.Count);
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
await File.WriteAllTextAsync(filePath, outJson);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated Sector summary file: {Path} (Active ISINs: {Count})", filePath, activeIsins.Count);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update Sector summary for {Sector}", sector);
}
finally
{
fileLock.Release();
}
}
private static string CalculateLabel(double score) => score switch
{
>= 0.15 => "POSITIVE",
<= -0.15 => "NEGATIVE",
_ => "NEUTRAL"
};
}