using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using FinlyticCore.Dtos.News;
using FinlyticCore.Dtos.Sentiment;
using FinlyticCore.Services;
using FinlyticSentiment.Util;
using Microsoft.Extensions.Configuration;
namespace FinlyticSentiment.Services;
///
/// Defines the persistence contract for maintaining two-stage ISIN and Sector JSON sentiment summaries in the file system.
///
public interface ISentimentStorageService
{
Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert);
Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert);
Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert);
Task GetArticleSentimentAsync(string articleId);
Task GetIsinSummaryAsync(string isin);
}
public class SentimentStorageService : ISentimentStorageService
{
private static readonly ConcurrentDictionary FileLocks = new();
private readonly IFinlyticLogger _finlyticLogger;
private readonly string _basePath;
private readonly JsonSerializerOptions _jsonOptions;
public SentimentStorageService(IConfiguration configuration, IFinlyticLogger finlyticLogger)
{
_finlyticLogger = finlyticLogger;
_basePath = configuration["Storage:SummariesPath"] ?? "data/summaries";
Directory.CreateDirectory(Path.Combine(_basePath, "isin"));
Directory.CreateDirectory(Path.Combine(_basePath, "sectors"));
Directory.CreateDirectory(Path.Combine(_basePath, "articles"));
_jsonOptions = new JsonSerializerOptions
{
WriteIndented = true,
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
};
}
///
public async Task SaveArticleSentimentAsync(NewsArticleDto article, FinBertResultDto finbert)
{
if (article == null || article.Id == Guid.Empty) return;
string cleanId = article.Id.ToString();
string filePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
var entry = new IsinAnalysisEntry
{
AnalysisId = $"sent_{Guid.NewGuid():N}",
Timestamp = nowIso,
Article = new IsinAnalysisArticleRef
{
ArticleId = cleanId,
Title = article.Title,
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
Source = article.Author ?? "FinlyticNews"
},
FinbertResult = finbert,
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
};
var json = JsonSerializer.Serialize(entry, _jsonOptions);
await File.WriteAllTextAsync(filePath, json);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Successfully saved article sentiment file: {Path}", filePath);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to write article sentiment file: {Path}", filePath);
}
finally
{
fileLock.Release();
}
}
///
public async Task GetArticleSentimentAsync(string articleId)
{
if (string.IsNullOrWhiteSpace(articleId)) return null;
string cleanId = articleId.Trim();
string articleFilePath = Path.Combine(_basePath, "articles", $"{cleanId}.json");
if (File.Exists(articleFilePath))
{
var fileLock = FileLocks.GetOrAdd(articleFilePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
var json = await File.ReadAllTextAsync(articleFilePath);
return JsonSerializer.Deserialize(json, _jsonOptions);
}
catch (Exception ex)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to read article sentiment file: {Path}", articleFilePath);
}
finally
{
fileLock.Release();
}
}
return null;
}
///
public async Task GetIsinSummaryAsync(string isin)
{
if (string.IsNullOrWhiteSpace(isin)) return null;
string cleanIsin = isin.Trim().ToUpperInvariant();
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
if (!File.Exists(filePath)) return null;
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
var json = await File.ReadAllTextAsync(filePath);
return JsonSerializer.Deserialize(json, _jsonOptions);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Error reading ISIN summary file: {Path}", filePath);
return null;
}
finally
{
fileLock.Release();
}
}
///
public async Task UpdateIsinSummaryAsync(string isin, string companyName, string sector, NewsArticleDto article, FinBertResultDto finbert)
{
if (string.IsNullOrWhiteSpace(isin)) return;
string cleanIsin = isin.Trim().ToUpperInvariant();
string filePath = Path.Combine(_basePath, "isin", $"{cleanIsin}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
var analyses = new List();
if (File.Exists(filePath))
{
var existingJson = await File.ReadAllTextAsync(filePath);
var existing = JsonSerializer.Deserialize(existingJson, _jsonOptions);
if (existing?.Analyses != null)
{
analyses.AddRange(existing.Analyses);
}
}
string cleanArticleId = article.Id.ToString();
analyses.RemoveAll(a => string.Equals(a.Article?.ArticleId, cleanArticleId, StringComparison.OrdinalIgnoreCase));
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
var newEntry = new IsinAnalysisEntry
{
AnalysisId = $"sent_{Guid.NewGuid():N}",
Timestamp = nowIso,
Article = new IsinAnalysisArticleRef
{
ArticleId = cleanArticleId,
Title = article.Title,
PublishedAt = article.PublishedAt.ToString("yyyy-MM-ddTHH:mm:ssZ"),
Source = article.Author ?? "FinlyticNews"
},
FinbertResult = finbert,
SummarySnippet = finbert.SummarySnippet ?? article.Summary ?? string.Empty
};
analyses.Add(newEntry);
var cutoff = DateTime.UtcNow.AddDays(-14);
var validAnalyses = analyses.Where(a =>
{
if (DateTime.TryParse(a.Article?.PublishedAt ?? a.Timestamp, out var pubDate))
{
return pubDate >= cutoff;
}
return true;
}).ToList();
var updatedAnalyses = validAnalyses.OrderByDescending(a => a.Article?.PublishedAt ?? a.Timestamp).Take(50).ToList();
double totalCompound = 0.0;
double totalConf = 0.0;
foreach (var item in updatedAnalyses)
{
if (item.FinbertResult == null) continue;
totalCompound += item.FinbertResult.CompoundScore;
totalConf += item.FinbertResult.Confidence;
}
int total = updatedAnalyses.Count;
double avgCompound = total > 0 ? totalCompound / total : 0.0;
double avgConf = total > 0 ? totalConf / total : 0.0;
string overallLabel = "NEUTRAL";
if (avgCompound >= 0.15) overallLabel = "POSITIVE";
else if (avgCompound <= -0.15) overallLabel = "NEGATIVE";
var summary = new IsinSentimentSummaryDto
{
Isin = cleanIsin,
CompanyName = companyName,
Sector = sector,
LastUpdated = nowIso,
CurrentSummary = new IsinCurrentSummary
{
CompoundScore = Math.Round(avgCompound, 4),
SentimentLabel = overallLabel,
AvgConfidence = Math.Round(avgConf, 4),
TotalArticlesAnalyzed = total,
Text = $"Synthesized sentiment across {total} articles is {overallLabel}."
},
Analyses = updatedAnalyses
};
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
await File.WriteAllTextAsync(filePath, outJson);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated ISIN summary file: {Path} (Total: {Count}, Score: {Score:F2})", filePath, updatedAnalyses.Count, avgCompound);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update ISIN summary for {Isin}", cleanIsin);
}
finally
{
fileLock.Release();
}
}
///
public async Task UpdateSectorSummaryAsync(string sector, string isin, string articleId, FinBertResultDto finbert)
{
if (string.IsNullOrWhiteSpace(sector)) return;
string cleanSector = sector.Trim().ToLowerInvariant();
string filePath = Path.Combine(_basePath, "sectors", $"{cleanSector}.json");
var fileLock = FileLocks.GetOrAdd(filePath, _ => new SemaphoreSlim(1, 1));
await fileLock.WaitAsync();
try
{
var analyses = new List();
if (File.Exists(filePath))
{
var existingJson = await File.ReadAllTextAsync(filePath);
var existing = JsonSerializer.Deserialize(existingJson, _jsonOptions);
if (existing?.Analyses != null)
{
analyses.AddRange(existing.Analyses);
}
}
string cleanIsin = isin.Trim().ToUpperInvariant();
string nowIso = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ");
analyses.RemoveAll(a => string.Equals(a.ArticleId, articleId, StringComparison.OrdinalIgnoreCase) && string.Equals(a.RelatedIsin, cleanIsin, StringComparison.OrdinalIgnoreCase));
analyses.Add(new SectorAnalysisEntry
{
AnalysisId = $"sec_{Guid.NewGuid():N}",
Timestamp = nowIso,
RelatedIsin = cleanIsin,
ArticleId = articleId,
FinbertResult = finbert
});
var cutoff = DateTime.UtcNow.AddDays(-14);
var updatedAnalyses = analyses.Where(a =>
{
if (DateTime.TryParse(a.Timestamp, out var ts))
{
return ts >= cutoff;
}
return true;
}).OrderByDescending(a => a.Timestamp).Take(100).ToList();
var activeIsins = updatedAnalyses.Select(a => a.RelatedIsin).Where(i => !string.IsNullOrEmpty(i)).Distinct().ToList();
double totalSectorCompound = updatedAnalyses.Sum(s => s.FinbertResult.CompoundScore);
double avgSectorCompound = updatedAnalyses.Count > 0 ? totalSectorCompound / updatedAnalyses.Count : 0.0;
string sectorLabel = "NEUTRAL";
if (avgSectorCompound >= 0.15) sectorLabel = "POSITIVE";
else if (avgSectorCompound <= -0.15) sectorLabel = "NEGATIVE";
var summary = new SectorSentimentSummaryDto
{
Sector = sector,
LastUpdated = nowIso,
CurrentSummary = new SectorCurrentSummary
{
CompoundScore = Math.Round(avgSectorCompound, 4),
SentimentLabel = sectorLabel,
ActiveIsins = activeIsins,
Text = $"Sector {sector} aggregate sentiment: {sectorLabel}."
},
Analyses = updatedAnalyses
};
var outJson = JsonSerializer.Serialize(summary, _jsonOptions);
await File.WriteAllTextAsync(filePath, outJson);
await _finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentStorageService] Updated Sector summary file: {Path} (Active ISINs: {Count})", filePath, activeIsins.Count);
}
catch (Exception ex)
{
await _finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentStorageService] Failed to update Sector summary for {Sector}", sector);
}
finally
{
fileLock.Release();
}
}
}