473 lines
21 KiB
C#
473 lines
21 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using FinlyticCore.Dtos;
|
|
using FinlyticCore.Dtos.News;
|
|
using FinlyticCore.Dtos.Sentiment;
|
|
using FinlyticCore.Dtos.Settings;
|
|
using FinlyticCore.Models;
|
|
using FinlyticCore.Services;
|
|
using FinlyticCore.Util;
|
|
using FinlyticSentiment.Services;
|
|
using FinlyticSentiment.Util;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace FinlyticSentiment.Util;
|
|
|
|
/// <summary>
|
|
/// Managed MQTT client for requesting pending news articles and updating sentiment results.
|
|
/// </summary>
|
|
public class SentimentMqttClient : ManagedMqttClient, IHostedService
|
|
{
|
|
private readonly ILogger<SentimentMqttClient> _logger;
|
|
private readonly IConfiguration _configuration;
|
|
private readonly IServiceScopeFactory _scopeFactory;
|
|
|
|
public SentimentMqttClient(
|
|
ILogger<SentimentMqttClient> logger,
|
|
IConfiguration configuration,
|
|
IServiceScopeFactory scopeFactory) : base(logger)
|
|
{
|
|
_logger = logger;
|
|
_configuration = configuration;
|
|
_scopeFactory = scopeFactory;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Starts the MQTT client and connects to the broker.
|
|
/// </summary>
|
|
public async Task StartAsync(CancellationToken cancellationToken)
|
|
{
|
|
var config = new MqttConfiguration
|
|
{
|
|
Host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost",
|
|
Port = Convert.ToInt32(_configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883"),
|
|
ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticSentiment")}_{Guid.NewGuid()}"
|
|
};
|
|
|
|
_logger.LogInformation("Starting Sentiment MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId);
|
|
await ConnectAsync(config);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stops the MQTT client and disconnects from the broker.
|
|
/// </summary>
|
|
public async Task StopAsync(CancellationToken cancellationToken)
|
|
{
|
|
_logger.LogInformation("Stopping Sentiment MQTT client and disconnecting.");
|
|
await DisconnectAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Event triggered when a real-time article is broadcasted on services/news/completed.
|
|
/// </summary>
|
|
public event Func<NewsArticleDto, Task>? OnArticleReceived;
|
|
|
|
/// <inheritdoc />
|
|
protected override async Task OnConnectedAsync()
|
|
{
|
|
_logger.LogInformation("Sentiment MQTT client connected. Subscribing to RPC response and broadcast topics...");
|
|
await SubscribeAsync("services/response/#");
|
|
await SubscribeAsync("services/news/completed");
|
|
await SubscribeAsync("services/request/sentiment_GetArticle/#");
|
|
await SubscribeAsync("services/request/sentiment_GetIsin/#");
|
|
await SubscribeAsync("services/request/sentiment_Analyze/#");
|
|
await SubscribeAsync("services/request/sentiment_settings_GetAll/#");
|
|
await SubscribeAsync("services/request/sentiment_settings_Update/#");
|
|
await SubscribeAsync("services/request/health_Ping/#");
|
|
await SubscribeAsync("services/config/updated/#");
|
|
|
|
FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) =>
|
|
{
|
|
if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticSentiment", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await PublishAsync("finlytic/logs/FinlyticSentiment", logDto);
|
|
}
|
|
};
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
protected override async Task OnMessageReceivedAsync(string topic, string payload)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(topic)) return;
|
|
|
|
if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (topic.EndsWith("FinlyticSentiment", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnConfigUpdatedAsync(payload);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (topic.Equals("services/news/completed", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnNewsCompletedAsync(payload);
|
|
return;
|
|
}
|
|
|
|
var lastSlash = topic.LastIndexOf('/');
|
|
if (lastSlash < 0 || lastSlash >= topic.Length - 1) return;
|
|
|
|
var correlationId = topic.Substring(lastSlash + 1);
|
|
|
|
if (topic.StartsWith("services/request/sentiment_GetArticle", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnSentimentGetArticleAsync(payload, correlationId);
|
|
}
|
|
else if (topic.StartsWith("services/request/sentiment_GetIsin", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnSentimentGetIsinAsync(payload, correlationId);
|
|
}
|
|
else if (topic.StartsWith("services/request/sentiment_Analyze", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnSentimentAnalyzeAsync(payload, correlationId);
|
|
}
|
|
else if (topic.StartsWith("services/request/sentiment_settings_GetAll", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnSettingsGetAllAsync(correlationId);
|
|
}
|
|
else if (topic.StartsWith("services/request/sentiment_settings_Update", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnSettingsUpdateAsync(payload, correlationId);
|
|
}
|
|
else if (topic.StartsWith("services/request/health_Ping", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
await OnHealthPingAsync(topic, correlationId);
|
|
}
|
|
}
|
|
|
|
private async Task OnSettingsGetAllAsync(string correlationId)
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
|
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId);
|
|
try
|
|
{
|
|
var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
|
|
var responseTopic = $"services/response/sentiment_settings_GetAll/{correlationId}";
|
|
|
|
await PublishAsync(responseTopic, settings);
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticSentiment] [Settings_GetAll] Failed to retrieve settings.");
|
|
}
|
|
}
|
|
|
|
private async Task OnSettingsUpdateAsync(string payload, string correlationId)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(payload)) return;
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
var settingsService = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
|
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId);
|
|
try
|
|
{
|
|
Dictionary<string, object?>? updates = null;
|
|
try
|
|
{
|
|
updates = JsonSerializer.Deserialize<Dictionary<string, object?>>(payload);
|
|
}
|
|
catch
|
|
{
|
|
var list = JsonSerializer.Deserialize<List<DynamicSettingDto>>(payload);
|
|
if (list != null)
|
|
{
|
|
updates = new Dictionary<string, object?>();
|
|
foreach (var item in list) updates[item.Key] = item.Value;
|
|
}
|
|
}
|
|
|
|
if (updates != null && updates.Count > 0)
|
|
{
|
|
await settingsService.UpdateSettingsAsync(updates);
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count);
|
|
}
|
|
|
|
var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) });
|
|
var responseTopic = $"services/response/sentiment_settings_Update/{correlationId}";
|
|
await PublishAsync(responseTopic, currentSettings);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticSentiment] [Settings_Update] Failed to update settings.");
|
|
}
|
|
}
|
|
|
|
private async Task OnConfigUpdatedAsync(string payload)
|
|
{
|
|
try
|
|
{
|
|
using var doc = JsonDocument.Parse(payload);
|
|
if (doc.RootElement.TryGetProperty("settings", out var settingsProp))
|
|
{
|
|
var dict = JsonSerializer.Deserialize<Dictionary<string, object?>>(settingsProp.GetRawText());
|
|
if (dict != null && dict.Count > 0)
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
|
|
await settings.UpdateSettingsAsync(dict);
|
|
}
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
|
|
private async Task OnHealthPingAsync(string topic, string correlationId)
|
|
{
|
|
if (topic.Contains("FinlyticSentiment", StringComparison.OrdinalIgnoreCase) ||
|
|
!topic.Contains("/", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
string respTopic = $"services/response/health_Ping/{correlationId}";
|
|
await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticSentiment", "Online", DateTime.UtcNow, "Connected"));
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[FinlyticSentiment] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId);
|
|
}
|
|
}
|
|
|
|
private async Task OnNewsCompletedAsync(string payload)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(payload)) return;
|
|
try
|
|
{
|
|
var article = JsonSerializer.Deserialize(payload, typeof(NewsArticleDto), FinlyticJsonSerializerContext.Default) as NewsArticleDto;
|
|
if (article != null && article.Id != Guid.Empty && OnArticleReceived != null)
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "Received real-time article broadcast on services/news/completed: {Title} (ID: {Id})", article.Title, article.Id);
|
|
await OnArticleReceived.Invoke(article);
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
|
|
private async Task OnSentimentGetArticleAsync(string payload, string correlationId)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(payload)) return;
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_GetArticle request [CorrelationId: {CorrelationId}]", correlationId);
|
|
|
|
try
|
|
{
|
|
var request = JsonSerializer.Deserialize(payload, typeof(ArticleRequest), FinlyticJsonSerializerContext.Default) as ArticleRequest;
|
|
var articleId = request?.ArticleId ?? request?.Id;
|
|
|
|
if (string.IsNullOrWhiteSpace(articleId))
|
|
{
|
|
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing articleId in request payload.");
|
|
return;
|
|
}
|
|
|
|
var storageService = scope.ServiceProvider.GetRequiredService<ISentimentStorageService>();
|
|
var sentimentEntry = await storageService.GetArticleSentimentAsync(articleId);
|
|
|
|
string responseTopic = $"services/response/sentiment_GetArticle/{correlationId}";
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_GetArticle response for article {ArticleId} to {ResponseTopic}", articleId, responseTopic);
|
|
await PublishAsync(responseTopic, sentimentEntry);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_GetArticle RPC request.");
|
|
}
|
|
}
|
|
|
|
private async Task OnSentimentGetIsinAsync(string payload, string correlationId)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(payload)) return;
|
|
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_GetIsin request [CorrelationId: {CorrelationId}]", correlationId);
|
|
|
|
try
|
|
{
|
|
var request = JsonSerializer.Deserialize(payload, typeof(IsinRequest), FinlyticJsonSerializerContext.Default) as IsinRequest;
|
|
var isin = request?.Isin;
|
|
|
|
if (string.IsNullOrWhiteSpace(isin))
|
|
{
|
|
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing ISIN in request payload.");
|
|
return;
|
|
}
|
|
|
|
var storageService = scope.ServiceProvider.GetRequiredService<ISentimentStorageService>();
|
|
var isinSummary = await storageService.GetIsinSummaryAsync(isin);
|
|
|
|
string responseTopic = $"services/response/sentiment_GetIsin/{correlationId}";
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_GetIsin response for ISIN {Isin} to {ResponseTopic}", isin, responseTopic);
|
|
await PublishAsync(responseTopic, isinSummary);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_GetIsin RPC request.");
|
|
}
|
|
}
|
|
|
|
private async Task OnSentimentAnalyzeAsync(string payload, string correlationId)
|
|
{
|
|
using var scope = _scopeFactory.CreateScope();
|
|
var finlyticLogger = scope.ServiceProvider.GetRequiredService<IFinlyticLogger<SentimentMqttClient>>();
|
|
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_Analyze request [CorrelationId: {CorrelationId}]", correlationId);
|
|
string responseTopic = $"services/response/sentiment_Analyze/{correlationId}";
|
|
|
|
if (string.IsNullOrWhiteSpace(payload))
|
|
{
|
|
await PublishAsync(responseTopic, (object?)null);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
var request = JsonSerializer.Deserialize(payload, typeof(AnalyzeSentimentRequest), FinlyticJsonSerializerContext.Default) as AnalyzeSentimentRequest;
|
|
|
|
if (request == null || (string.IsNullOrWhiteSpace(request.ArticleId) && string.IsNullOrWhiteSpace(request.Isin)))
|
|
{
|
|
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing ArticleId or Isin in sentiment_Analyze request payload.");
|
|
await PublishAsync(responseTopic, (object?)null);
|
|
return;
|
|
}
|
|
|
|
var storageService = scope.ServiceProvider.GetRequiredService<ISentimentStorageService>();
|
|
var analyzerService = scope.ServiceProvider.GetRequiredService<IFinBertAnalyzerService>();
|
|
|
|
object? result = null;
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.ArticleId))
|
|
{
|
|
var cleanArticleId = request.ArticleId.Trim();
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing article analysis for ArticleId: {ArticleId} (ForceReload: {ForceReload})", cleanArticleId, request.ForceReload);
|
|
|
|
IsinAnalysisEntry? existingEntry = null;
|
|
|
|
if (!request.ForceReload)
|
|
{
|
|
existingEntry = await storageService.GetArticleSentimentAsync(cleanArticleId);
|
|
}
|
|
|
|
if (existingEntry != null)
|
|
{
|
|
result = existingEntry;
|
|
}
|
|
else
|
|
{
|
|
if (Guid.TryParse(cleanArticleId, out var articleGuid))
|
|
{
|
|
var article = await SendRpcRequestAsync<NewsArticleDto, ArticleRequest>(
|
|
"news_GetById",
|
|
new ArticleRequest(cleanArticleId, cleanArticleId),
|
|
TimeSpan.FromSeconds(8));
|
|
|
|
if (article != null)
|
|
{
|
|
var finbertResult = await analyzerService.AnalyzeArticleAsync(article);
|
|
|
|
if (finbertResult == null)
|
|
{
|
|
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] FinBERT analysis returned NULL for article {ArticleId}. Aborting manual analysis.", cleanArticleId);
|
|
await PublishAsync(responseTopic, (object?)null);
|
|
return;
|
|
}
|
|
|
|
await storageService.SaveArticleSentimentAsync(article, finbertResult);
|
|
|
|
if (article.MatchedAssets != null && article.MatchedAssets.Count > 0)
|
|
{
|
|
foreach (var asset in article.MatchedAssets)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(asset.Isin)) continue;
|
|
|
|
await storageService.UpdateIsinSummaryAsync(
|
|
asset.Isin,
|
|
asset.Name,
|
|
"General",
|
|
article,
|
|
finbertResult);
|
|
|
|
await storageService.UpdateSectorSummaryAsync(
|
|
"General",
|
|
asset.Isin,
|
|
article.Id.ToString(),
|
|
finbertResult);
|
|
}
|
|
}
|
|
|
|
await UpdateArticleStatusAsync(article.Id, "Analyzed");
|
|
|
|
result = await storageService.GetArticleSentimentAsync(cleanArticleId);
|
|
}
|
|
else
|
|
{
|
|
await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Could not retrieve article {ArticleId} from FinlyticNews for re-analysis.", cleanArticleId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (!string.IsNullOrWhiteSpace(request.Isin))
|
|
{
|
|
var cleanIsin = request.Isin.Trim();
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Fetching sentiment summary for ISIN: {Isin}", cleanIsin);
|
|
|
|
result = await storageService.GetIsinSummaryAsync(cleanIsin);
|
|
}
|
|
|
|
await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_Analyze response to {ResponseTopic}", responseTopic);
|
|
await PublishAsync(responseTopic, result);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_Analyze RPC request.");
|
|
await PublishAsync(responseTopic, (object?)null);
|
|
}
|
|
}
|
|
|
|
public async Task<List<NewsArticleDto>> GetPendingArticlesAsync(int limit = 10)
|
|
{
|
|
try
|
|
{
|
|
var payload = new LimitRequest(Math.Min(limit, 10));
|
|
var articles = await SendRpcRequestAsync<List<NewsArticleDto>, LimitRequest>("news_GetPending", payload, TimeSpan.FromSeconds(10));
|
|
return articles ?? [];
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Error executing MQTT RPC for news_GetPending.");
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
public async Task<bool> UpdateArticleStatusAsync(Guid id, string status = "Analyzed")
|
|
{
|
|
try
|
|
{
|
|
var request = new UpdateNewsStatusRequest(id, status);
|
|
var response = await SendRpcRequestAsync<UpdateNewsStatusResponse, UpdateNewsStatusRequest>("news_UpdateStatus", request, TimeSpan.FromSeconds(8));
|
|
return response?.Success ?? false;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Error executing MQTT RPC for news_UpdateStatus (ID: {Id}).", id);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
} |