using System; using System.Collections.Generic; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using FinlyticCore.Dtos; using FinlyticCore.Dtos.News; using FinlyticCore.Dtos.Sentiment; using FinlyticCore.Dtos.Settings; using FinlyticCore.Models; using FinlyticCore.Services; using FinlyticCore.Util; using FinlyticSentiment.Services; using FinlyticSentiment.Util; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace FinlyticSentiment.Util; /// /// Managed MQTT client for requesting pending news articles and updating sentiment results. /// public class SentimentMqttClient : ManagedMqttClient, IHostedService { private readonly ILogger _logger; private readonly IConfiguration _configuration; private readonly IServiceScopeFactory _scopeFactory; public SentimentMqttClient( ILogger logger, IConfiguration configuration, IServiceScopeFactory scopeFactory) : base(logger) { _logger = logger; _configuration = configuration; _scopeFactory = scopeFactory; } /// /// Starts the MQTT client and connects to the broker. /// public async Task StartAsync(CancellationToken cancellationToken) { var config = new MqttConfiguration { Host = _configuration["MQTT:Host"] ?? _configuration["MQTT__Host"] ?? "localhost", Port = Convert.ToInt32(_configuration["MQTT:Port"] ?? _configuration["MQTT__Port"] ?? "1883"), ClientId = $"{(_configuration["MQTT:ClientId"] ?? _configuration["MQTT__ClientId"] ?? "FinlyticSentiment")}_{Guid.NewGuid()}" }; _logger.LogInformation("Starting Sentiment MQTT client. Host: {Host}, ClientId: {ClientId}", config.Host, config.ClientId); await ConnectAsync(config); } /// /// Stops the MQTT client and disconnects from the broker. /// public async Task StopAsync(CancellationToken cancellationToken) { _logger.LogInformation("Stopping Sentiment MQTT client and disconnecting."); await DisconnectAsync(); } /// /// Event triggered when a real-time article is broadcasted on services/news/completed. /// public event Func? OnArticleReceived; /// protected override async Task OnConnectedAsync() { _logger.LogInformation("Sentiment MQTT client connected. Subscribing to RPC response and broadcast topics..."); await SubscribeAsync("services/response/#"); await SubscribeAsync("services/news/completed"); await SubscribeAsync("services/request/sentiment_GetArticle/#"); await SubscribeAsync("services/request/sentiment_GetIsin/#"); await SubscribeAsync("services/request/sentiment_Analyze/#"); await SubscribeAsync("services/request/sentiment_settings_GetAll/#"); await SubscribeAsync("services/request/sentiment_settings_Update/#"); await SubscribeAsync("services/request/health_Ping/#"); await SubscribeAsync("services/config/updated/#"); FinlyticCore.Services.FinlyticLogBroadcaster.OnLogPublished = async (logDto) => { if (IsConnected && string.Equals(logDto.ServiceName, "FinlyticSentiment", StringComparison.OrdinalIgnoreCase)) { await PublishAsync("finlytic/logs/FinlyticSentiment", logDto); } }; } /// protected override async Task OnMessageReceivedAsync(string topic, string payload) { if (string.IsNullOrWhiteSpace(topic)) return; if (topic.StartsWith("services/config/updated", StringComparison.OrdinalIgnoreCase)) { if (topic.EndsWith("FinlyticSentiment", StringComparison.OrdinalIgnoreCase)) { await OnConfigUpdatedAsync(payload); } return; } if (topic.Equals("services/news/completed", StringComparison.OrdinalIgnoreCase)) { await OnNewsCompletedAsync(payload); return; } var lastSlash = topic.LastIndexOf('/'); if (lastSlash < 0 || lastSlash >= topic.Length - 1) return; var correlationId = topic.Substring(lastSlash + 1); if (topic.StartsWith("services/request/sentiment_GetArticle", StringComparison.OrdinalIgnoreCase)) { await OnSentimentGetArticleAsync(payload, correlationId); } else if (topic.StartsWith("services/request/sentiment_GetIsin", StringComparison.OrdinalIgnoreCase)) { await OnSentimentGetIsinAsync(payload, correlationId); } else if (topic.StartsWith("services/request/sentiment_Analyze", StringComparison.OrdinalIgnoreCase)) { await OnSentimentAnalyzeAsync(payload, correlationId); } else if (topic.StartsWith("services/request/sentiment_settings_GetAll", StringComparison.OrdinalIgnoreCase)) { await OnSettingsGetAllAsync(correlationId); } else if (topic.StartsWith("services/request/sentiment_settings_Update", StringComparison.OrdinalIgnoreCase)) { await OnSettingsUpdateAsync(payload, correlationId); } else if (topic.StartsWith("services/request/health_Ping", StringComparison.OrdinalIgnoreCase)) { await OnHealthPingAsync(topic, correlationId); } } private async Task OnSettingsGetAllAsync(string correlationId) { using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); var settingsService = scope.ServiceProvider.GetRequiredService(); await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_GetAll] Retrieving all dynamic settings via reflection [CorrelationId: {CorrelationId}]", correlationId); try { var settings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) }); var responseTopic = $"services/response/sentiment_settings_GetAll/{correlationId}"; await PublishAsync(responseTopic, settings); await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_GetAll] Published {Count} settings to '{ResponseTopic}'", settings.Count, responseTopic); } catch (Exception ex) { await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticSentiment] [Settings_GetAll] Failed to retrieve settings."); } } private async Task OnSettingsUpdateAsync(string payload, string correlationId) { if (string.IsNullOrWhiteSpace(payload)) return; using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); var settingsService = scope.ServiceProvider.GetRequiredService(); await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_Update] Processing settings update RPC [CorrelationId: {CorrelationId}]", correlationId); try { Dictionary? updates = null; try { updates = JsonSerializer.Deserialize>(payload); } catch { var list = JsonSerializer.Deserialize>(payload); if (list != null) { updates = new Dictionary(); foreach (var item in list) updates[item.Key] = item.Value; } } if (updates != null && updates.Count > 0) { await settingsService.UpdateSettingsAsync(updates); await finlyticLogger.LogInfoAsync(SettingKeys.MqttChannel, "[FinlyticSentiment] [Settings_Update] Successfully updated {Count} settings in database and cache.", updates.Count); } var currentSettings = await settingsService.GetAllRegisteredSettingsAsync(new[] { typeof(SettingKeys) }); var responseTopic = $"services/response/sentiment_settings_Update/{correlationId}"; await PublishAsync(responseTopic, currentSettings); } catch (Exception ex) { await finlyticLogger.LogErrorAsync(SettingKeys.MqttChannel, ex, "[FinlyticSentiment] [Settings_Update] Failed to update settings."); } } private async Task OnConfigUpdatedAsync(string payload) { try { using var doc = JsonDocument.Parse(payload); if (doc.RootElement.TryGetProperty("settings", out var settingsProp)) { var dict = JsonSerializer.Deserialize>(settingsProp.GetRawText()); if (dict != null && dict.Count > 0) { using var scope = _scopeFactory.CreateScope(); var settings = scope.ServiceProvider.GetRequiredService(); await settings.UpdateSettingsAsync(dict); } } } catch { } } private async Task OnHealthPingAsync(string topic, string correlationId) { if (topic.Contains("FinlyticSentiment", StringComparison.OrdinalIgnoreCase) || !topic.Contains("/", StringComparison.OrdinalIgnoreCase)) { string respTopic = $"services/response/health_Ping/{correlationId}"; await PublishAsync(respTopic, new ServiceHealthResponse("FinlyticSentiment", "Online", DateTime.UtcNow, "Connected")); using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); await finlyticLogger.LogInfoAsync(SettingKeys.HealthPingChannel, "[FinlyticSentiment] Responded to live health_Ping RPC request [CorrelationId: {CorrelationId}].", correlationId); } } private async Task OnNewsCompletedAsync(string payload) { if (string.IsNullOrWhiteSpace(payload)) return; try { var article = JsonSerializer.Deserialize(payload, typeof(NewsArticleDto), FinlyticJsonSerializerContext.Default) as NewsArticleDto; if (article != null && article.Id != Guid.Empty && OnArticleReceived != null) { using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "Received real-time article broadcast on services/news/completed: {Title} (ID: {Id})", article.Title, article.Id); await OnArticleReceived.Invoke(article); } } catch { } } private async Task OnSentimentGetArticleAsync(string payload, string correlationId) { if (string.IsNullOrWhiteSpace(payload)) return; using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_GetArticle request [CorrelationId: {CorrelationId}]", correlationId); try { var request = JsonSerializer.Deserialize(payload, typeof(ArticleRequest), FinlyticJsonSerializerContext.Default) as ArticleRequest; var articleId = request?.ArticleId ?? request?.Id; if (string.IsNullOrWhiteSpace(articleId)) { await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing articleId in request payload."); return; } var storageService = scope.ServiceProvider.GetRequiredService(); var sentimentEntry = await storageService.GetArticleSentimentAsync(articleId); string responseTopic = $"services/response/sentiment_GetArticle/{correlationId}"; await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_GetArticle response for article {ArticleId} to {ResponseTopic}", articleId, responseTopic); await PublishAsync(responseTopic, sentimentEntry); } catch (Exception ex) { await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_GetArticle RPC request."); } } private async Task OnSentimentGetIsinAsync(string payload, string correlationId) { if (string.IsNullOrWhiteSpace(payload)) return; using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_GetIsin request [CorrelationId: {CorrelationId}]", correlationId); try { var request = JsonSerializer.Deserialize(payload, typeof(IsinRequest), FinlyticJsonSerializerContext.Default) as IsinRequest; var isin = request?.Isin; if (string.IsNullOrWhiteSpace(isin)) { await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing ISIN in request payload."); return; } var storageService = scope.ServiceProvider.GetRequiredService(); var isinSummary = await storageService.GetIsinSummaryAsync(isin); string responseTopic = $"services/response/sentiment_GetIsin/{correlationId}"; await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_GetIsin response for ISIN {Isin} to {ResponseTopic}", isin, responseTopic); await PublishAsync(responseTopic, isinSummary); } catch (Exception ex) { await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_GetIsin RPC request."); } } private async Task OnSentimentAnalyzeAsync(string payload, string correlationId) { using var scope = _scopeFactory.CreateScope(); var finlyticLogger = scope.ServiceProvider.GetRequiredService>(); await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing RPC sentiment_Analyze request [CorrelationId: {CorrelationId}]", correlationId); string responseTopic = $"services/response/sentiment_Analyze/{correlationId}"; if (string.IsNullOrWhiteSpace(payload)) { await PublishAsync(responseTopic, (object?)null); return; } try { var request = JsonSerializer.Deserialize(payload, typeof(AnalyzeSentimentRequest), FinlyticJsonSerializerContext.Default) as AnalyzeSentimentRequest; if (request == null || (string.IsNullOrWhiteSpace(request.ArticleId) && string.IsNullOrWhiteSpace(request.Isin))) { await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Missing ArticleId or Isin in sentiment_Analyze request payload."); await PublishAsync(responseTopic, (object?)null); return; } var storageService = scope.ServiceProvider.GetRequiredService(); var analyzerService = scope.ServiceProvider.GetRequiredService(); object? result = null; if (!string.IsNullOrWhiteSpace(request.ArticleId)) { var cleanArticleId = request.ArticleId.Trim(); await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Processing article analysis for ArticleId: {ArticleId} (ForceReload: {ForceReload})", cleanArticleId, request.ForceReload); IsinAnalysisEntry? existingEntry = null; if (!request.ForceReload) { existingEntry = await storageService.GetArticleSentimentAsync(cleanArticleId); } if (existingEntry != null) { result = existingEntry; } else { if (Guid.TryParse(cleanArticleId, out var articleGuid)) { var article = await SendRpcRequestAsync( "news_GetById", new ArticleRequest(cleanArticleId, cleanArticleId), TimeSpan.FromSeconds(8)); if (article != null) { var finbertResult = await analyzerService.AnalyzeArticleAsync(article); if (finbertResult == null) { await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] FinBERT analysis returned NULL for article {ArticleId}. Aborting manual analysis.", cleanArticleId); await PublishAsync(responseTopic, (object?)null); return; } await storageService.SaveArticleSentimentAsync(article, finbertResult); if (article.MatchedAssets != null && article.MatchedAssets.Count > 0) { foreach (var asset in article.MatchedAssets) { if (string.IsNullOrWhiteSpace(asset.Isin)) continue; await storageService.UpdateIsinSummaryAsync( asset.Isin, asset.Name, "General", article, finbertResult); await storageService.UpdateSectorSummaryAsync( "General", asset.Isin, article.Id.ToString(), finbertResult); } } await UpdateArticleStatusAsync(article.Id, "Analyzed"); result = await storageService.GetArticleSentimentAsync(cleanArticleId); } else { await finlyticLogger.LogWarningAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Could not retrieve article {ArticleId} from FinlyticNews for re-analysis.", cleanArticleId); } } } } else if (!string.IsNullOrWhiteSpace(request.Isin)) { var cleanIsin = request.Isin.Trim(); await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Fetching sentiment summary for ISIN: {Isin}", cleanIsin); result = await storageService.GetIsinSummaryAsync(cleanIsin); } await finlyticLogger.LogInfoAsync(SettingKeys.SentimentChannel, "[SentimentMqttClient] Publishing RPC sentiment_Analyze response to {ResponseTopic}", responseTopic); await PublishAsync(responseTopic, result); } catch (Exception ex) { await finlyticLogger.LogErrorAsync(SettingKeys.SentimentChannel, ex, "[SentimentMqttClient] Failed to process sentiment_Analyze RPC request."); await PublishAsync(responseTopic, (object?)null); } } public async Task> GetPendingArticlesAsync(int limit = 10) { try { var payload = new LimitRequest(Math.Min(limit, 10)); var articles = await SendRpcRequestAsync, LimitRequest>("news_GetPending", payload, TimeSpan.FromSeconds(10)); return articles ?? []; } catch (Exception ex) { _logger.LogError(ex, "Error executing MQTT RPC for news_GetPending."); } return []; } public async Task UpdateArticleStatusAsync(Guid id, string status = "Analyzed") { try { var request = new UpdateNewsStatusRequest(id, status); var response = await SendRpcRequestAsync("news_UpdateStatus", request, TimeSpan.FromSeconds(8)); return response?.Success ?? false; } catch (Exception ex) { _logger.LogError(ex, "Error executing MQTT RPC for news_UpdateStatus (ID: {Id}).", id); } return false; } }