345 lines
14 KiB
C#
345 lines
14 KiB
C#
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text.Encodings.Web;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FinlyticCore.Dtos.News;
|
|
using FinlyticCore.Dtos.Sentiment;
|
|
using FinlyticCore.Services;
|
|
using FinlyticSentiment.Util;
|
|
using Microsoft.Extensions.Configuration;
|
|
|
|
namespace FinlyticSentiment.Services;
|
|
|
|
/// <summary>
|
|
/// Defines the persistence contract for maintaining two-stage ISIN and Sector JSON sentiment summaries in the file system.
|
|
/// </summary>
|
|
public interface ISentimentStorageService
|
|
{
|
|
Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert);
|
|
Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert);
|
|
Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert);
|
|
Task<IsinAnalysisEntry?> GetArticleSentimentAsync(string articleId);
|
|
Task<IsinSentimentSummaryDto?> GetIsinSummaryAsync(string isin);
|
|
}
|
|
|
|
public class SentimentStorageService : ISentimentStorageService
|
|
{
|
|
private static readonly ConcurrentDictionary<string, SemaphoreSlim> FileLocks = new();
|
|
|
|
private readonly IFinlyticLogger<SentimentStorageService> _finlyticLogger;
|
|
private readonly string _basePath;
|
|
private readonly JsonSerializerOptions _jsonOptions;
|
|
|
|
public SentimentStorageService(IConfiguration configuration, IFinlyticLogger<SentimentStorageService> finlyticLogger)
|
|
{
|
|
_finlyticLogger = finlyticLogger;
|
|
_basePath = configuration["Storage:SummariesPath"] ?? "data/summaries";
|
|
|
|
Directory.CreateDirectory(Path.Combine(_basePath, "isin"));
|
|
Directory.CreateDirectory(Path.Combine(_basePath, "sectors"));
|
|
Directory.CreateDirectory(Path.Combine(_basePath, "articles"));
|
|
|
|
_jsonOptions = new JsonSerializerOptions
|
|
{
|
|
WriteIndented = true,
|
|
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
|
|
};
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert)
|
|
{
|
|
if (article == null || article.Id == Guid.Empty) return;
|
|
|
|
string cleanId = article.Id.ToString();
|
|
string filePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
|
|
|
|
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
|
await fileLock.WaitAsync();
|
|
|
|
try
|
|
{
|
|
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
|
|
var entry = new IsinAnalysisEntry
|
|
{
|
|
AnalysisId = $"sent_{Guid.NewGuid():N}",
|
|
Timestamp = nowIso,
|
|
Article = new IsinAnalysisArticleRef
|
|
{
|
|
ArticleId = cleanId,
|
|
Title = article.Title,
|
|
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
|
|
Source = article.Author ?? "FinlyticNews"
|
|
},
|
|
FinbertResult = finbert,
|
|
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
|
|
};
|
|
|
|
var json = JsonSerializer.Serialize(entry, _jsonOptions);
|
|
await File.WriteAllTextAsync(filePath, json);
|
|
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Successfully saved article sentiment file: {Path}", filePath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to write article sentiment file: {Path}", filePath);
|
|
}
|
|
finally
|
|
{
|
|
fileLock.Release();
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IsinAnalysisEntry?> GetArticleSentimentAsync(string articleId)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(articleId)) return null;
|
|
|
|
string cleanId = articleId.Trim();
|
|
string articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
|
|
|
|
if (File.Exists(articleFilePath))
|
|
{
|
|
var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1));
|
|
await fileLock.WaitAsync();
|
|
try
|
|
{
|
|
var json = await File.ReadAllTextAsync(articleFilePath);
|
|
return JsonSerializer.Deserialize<IsinAnalysisEntry>(json, _jsonOptions);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to read article sentiment file: {Path}", articleFilePath);
|
|
}
|
|
finally
|
|
{
|
|
fileLock.Release();
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IsinSentimentSummaryDto?> GetIsinSummaryAsync(string isin)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(isin)) return null;
|
|
|
|
string cleanIsin = isin.Trim().ToUpperInvariant();
|
|
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
|
|
|
|
if (!File.Exists(filePath)) return null;
|
|
|
|
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
|
await fileLock.WaitAsync();
|
|
|
|
try
|
|
{
|
|
var json = await File.ReadAllTextAsync(filePath);
|
|
return JsonSerializer.Deserialize<IsinSentimentSummaryDto>(json, _jsonOptions);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Error reading ISIN summary file: {Path}", filePath);
|
|
return null;
|
|
}
|
|
finally
|
|
{
|
|
fileLock.Release();
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(isin)) return;
|
|
|
|
string cleanIsin = isin.Trim().ToUpperInvariant();
|
|
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
|
|
|
|
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
|
await fileLock.WaitAsync();
|
|
|
|
try
|
|
{
|
|
var analyses = new List<IsinAnalysisEntry>();
|
|
if (File.Exists(filePath))
|
|
{
|
|
var existingJson = await File.ReadAllTextAsync(filePath);
|
|
var existing = JsonSerializer.Deserialize<IsinSentimentSummaryDto>(existingJson, _jsonOptions);
|
|
if (existing?.Analyses != null)
|
|
{
|
|
analyses.AddRange(existing.Analyses);
|
|
}
|
|
}
|
|
|
|
string cleanArticleId = article.Id.ToString();
|
|
analyses.RemoveAll(a => string.Equals(a.Article?.ArticleId, cleanArticleId, StringComparison.OrdinalIgnoreCase));
|
|
|
|
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
|
|
var newEntry = new IsinAnalysisEntry
|
|
{
|
|
AnalysisId = $"sent_{Guid.NewGuid():N}",
|
|
Timestamp = nowIso,
|
|
Article = new IsinAnalysisArticleRef
|
|
{
|
|
ArticleId = cleanArticleId,
|
|
Title = article.Title,
|
|
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
|
|
Source = article.Author ?? "FinlyticNews"
|
|
},
|
|
FinbertResult = finbert,
|
|
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
|
|
};
|
|
|
|
analyses.Add(newEntry);
|
|
|
|
var cutoff = DateTime.UtcNow.AddDays(-14);
|
|
var validAnalyses = analyses.Where(a =>
|
|
{
|
|
if (DateTime.TryParse(a.Article?.PublishedAt ?? a.Timestamp, out var pubDate))
|
|
{
|
|
return pubDate >= cutoff;
|
|
}
|
|
return true;
|
|
}).ToList();
|
|
|
|
var updatedAnalyses = validAnalyses.OrderByDescending(a => a.Article?.PublishedAt ?? a.Timestamp).Take(50).ToList();
|
|
|
|
double totalCompound = 0.0;
|
|
double totalConf = 0.0;
|
|
|
|
foreach (var item in updatedAnalyses)
|
|
{
|
|
if (item.FinbertResult == null) continue;
|
|
totalCompound += item.FinbertResult.CompoundScore;
|
|
totalConf += item.FinbertResult.Confidence;
|
|
}
|
|
|
|
int total = updatedAnalyses.Count;
|
|
double avgCompound = total > 0 ? totalCompound / total : 0.0;
|
|
double avgConf = total > 0 ? totalConf / total : 0.0;
|
|
|
|
string overallLabel = "NEUTRAL";
|
|
if (avgCompound >= 0.15) overallLabel = "POSITIVE";
|
|
else if (avgCompound <= -0.15) overallLabel = "NEGATIVE";
|
|
|
|
var summary = new IsinSentimentSummaryDto
|
|
{
|
|
Isin = cleanIsin,
|
|
CompanyName = companyName,
|
|
Sector = sector,
|
|
LastUpdated = nowIso,
|
|
CurrentSummary = new IsinCurrentSummary
|
|
{
|
|
CompoundScore = Math.Round(avgCompound, 4),
|
|
SentimentLabel = overallLabel,
|
|
AvgConfidence = Math.Round(avgConf, 4),
|
|
TotalArticlesAnalyzed = total,
|
|
Text = $"Synthesized sentiment across {total} articles is {overallLabel}."
|
|
},
|
|
Analyses = updatedAnalyses
|
|
};
|
|
|
|
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
|
|
await File.WriteAllTextAsync(filePath, outJson);
|
|
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", filePath, updatedAnalyses.Count, avgCompound);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update ISIN summary for {Isin}", cleanIsin);
|
|
}
|
|
finally
|
|
{
|
|
fileLock.Release();
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(sector)) return;
|
|
|
|
string cleanSector = sector.Trim().ToLowerInvariant();
|
|
string filePath = Path.Combine(_basePath, "sectors", $"{cleanSector}.json");
|
|
|
|
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
|
|
await fileLock.WaitAsync();
|
|
|
|
try
|
|
{
|
|
var analyses = new List<SectorAnalysisEntry>();
|
|
if (File.Exists(filePath))
|
|
{
|
|
var existingJson = await File.ReadAllTextAsync(filePath);
|
|
var existing = JsonSerializer.Deserialize<SectorSentimentSummaryDto>(existingJson, _jsonOptions);
|
|
if (existing?.Analyses != null)
|
|
{
|
|
analyses.AddRange(existing.Analyses);
|
|
}
|
|
}
|
|
|
|
string cleanIsin = isin.Trim().ToUpperInvariant();
|
|
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
|
|
|
|
analyses.RemoveAll(a => string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase) && string.Equals(a.RelatedIsin, cleanIsin, StringComparison.OrdinalIgnoreCase));
|
|
|
|
analyses.Add(new SectorAnalysisEntry
|
|
{
|
|
AnalysisId = $"sec_{Guid.NewGuid():N}",
|
|
Timestamp = nowIso,
|
|
RelatedIsin = cleanIsin,
|
|
ArticleId = articleId,
|
|
FinbertResult = finbert
|
|
});
|
|
|
|
var cutoff = DateTime.UtcNow.AddDays(-14);
|
|
var updatedAnalyses = analyses.Where(a =>
|
|
{
|
|
if (DateTime.TryParse(a.Timestamp, out var ts))
|
|
{
|
|
return ts >= cutoff;
|
|
}
|
|
return true;
|
|
}).OrderByDescending(a => a.Timestamp).Take(100).ToList();
|
|
|
|
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrEmpty(i)).Distinct().ToList();
|
|
double totalSectorCompound = updatedAnalyses.Sum(s => s.FinbertResult.CompoundScore);
|
|
double avgSectorCompound = updatedAnalyses.Count > 0 ? totalSectorCompound / updatedAnalyses.Count : 0.0;
|
|
|
|
string sectorLabel = "NEUTRAL";
|
|
if (avgSectorCompound >= 0.15) sectorLabel = "POSITIVE";
|
|
else if (avgSectorCompound <= -0.15) sectorLabel = "NEGATIVE";
|
|
|
|
var summary = new SectorSentimentSummaryDto
|
|
{
|
|
Sector = sector,
|
|
LastUpdated = nowIso,
|
|
CurrentSummary = new SectorCurrentSummary
|
|
{
|
|
CompoundScore = Math.Round(avgSectorCompound, 4),
|
|
SentimentLabel = sectorLabel,
|
|
ActiveIsins = activeIsins,
|
|
Text = $"Sector {sector} aggregate sentiment: {sectorLabel}."
|
|
},
|
|
Analyses = updatedAnalyses
|
|
};
|
|
|
|
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
|
|
await File.WriteAllTextAsync(filePath, outJson);
|
|
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated Sector summary file: {Path} (Active ISINs: {Count})", filePath, activeIsins.Count);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update Sector summary for {Sector}", sector);
|
|
}
|
|
finally
|
|
{
|
|
fileLock.Release();
|
|
}
|
|
}
|
|
} |