feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

FinlyticNews/Services/ArticleDeduplicationService.cs

@@ -0,0 +1,357 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using System.Numerics;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Services;
+using FinlyticNews.Database;
+using FinlyticNews.Util;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Result of a duplicate check on a candidate news article.
+/// </summary>
+public record DuplicateCheckResult(
+    bool IsDuplicate,
+    Guid? DuplicateOfArticleId,
+    string? Reason,
+    string TitleHash,
+    long SimHash
+);
+
+/// <summary>
+/// Service interface for detecting syndicated or near-identical duplicate news articles without AI.
+/// </summary>
+public interface IArticleDeduplicationService
+{
+    /// <summary>
+    /// Evaluates whether an article is a duplicate based on normalized title similarity and 64-bit text SimHash fingerprinting.
+    /// </summary>
+    DuplicateCheckResult CheckDuplicate(string title, string content, DateTime publishedAt);
+
+    /// <summary>
+    /// Registers a newly processed article into the in-memory deduplication cache.
+    /// </summary>
+    void RegisterArticle(Guid articleId, string titleHash, long simHash, string title, DateTime publishedAt);
+
+    /// <summary>
+    /// Loads recent articles from the database into the deduplication cache on startup.
+    /// </summary>
+    Task InitializeAsync(CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Computes the normalized SHA-256 hash of an article title.
+    /// </summary>
+    string ComputeTitleHash(string title);
+
+    /// <summary>
+    /// Computes the 64-bit SimHash content fingerprint of article text.
+    /// </summary>
+    long ComputeSimHash(string content);
+}
+
+/// <summary>
+/// High-performance non-AI deduplication engine using 64-bit SimHash and N-Gram title similarity.
+/// </summary>
+public class ArticleDeduplicationService : IArticleDeduplicationService
+{
+    private record CachedArticleEntry(
+        Guid ArticleId,
+        string TitleHash,
+        long SimHash,
+        string NormalizedTitle,
+        DateTime PublishedAt
+    );
+
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly IFinlyticLogger<ArticleDeduplicationService> _finlyticLogger;
+    private readonly ISettingsService _settingsService;
+
+    // Rolling in-memory cache of recent articles
+    private readonly ConcurrentDictionary<string, Guid> _titleHashIndex = new(StringComparer.OrdinalIgnoreCase);
+    private readonly List<CachedArticleEntry> _simHashIndex = [];
+    private readonly ReaderWriterLockSlim _simHashLock = new();
+
+    private bool _initialized;
+    private readonly SemaphoreSlim _initLock = new(1, 1);
+
+    private static readonly Regex TitlePortalSuffixRegex = new(
+        @"\s*[-|–—]\s*(DER AKTIONÄR|ARIVA\.DE|IT-Times|onvista|wallstreet:online|Sharedeals\.de|boerse\.de|Handelsblatt|WirtschaftsWoche|finanzen\.net|Finanznachrichten|XTB|Lynx|T3n|ntg24|Moneycab).*$",
+        RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+    private static readonly Regex NonAlphanumericRegex = new(@"[^\w\s]", RegexOptions.Compiled);
+    private static readonly Regex MultipleSpacesRegex = new(@"\s+", RegexOptions.Compiled);
+
+    public ArticleDeduplicationService(
+        IServiceScopeFactory scopeFactory,
+        IFinlyticLogger<ArticleDeduplicationService> finlyticLogger,
+        ISettingsService settingsService)
+    {
+        _scopeFactory = scopeFactory;
+        _finlyticLogger = finlyticLogger;
+        _settingsService = settingsService;
+    }
+
+    public async Task InitializeAsync(CancellationToken cancellationToken = default)
+    {
+        if (_initialized) return;
+
+        await _initLock.WaitAsync(cancellationToken);
+        try
+        {
+            if (_initialized) return;
+
+            using var scope = _scopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();
+
+            var windowDays = await _settingsService.GetSettingAsync(SettingKeys.DeduplicationWindowDays, cancellationToken);
+            var cutoff = DateTime.UtcNow.AddDays(-Math.Max(windowDays, 7));
+
+            var articles = await dbContext.NewsArticles
+                .AsNoTracking()
+                .Where(a => a.PublishedAt >= cutoff && a.Status == "Completed")
+                .Select(a => new
+                {
+                    a.Id,
+                    a.Title,
+                    a.TitleHash,
+                    a.SimHash,
+                    a.PublishedAt
+                })
+                .ToListAsync(cancellationToken);
+
+            _simHashLock.EnterWriteLock();
+            try
+            {
+                foreach (var a in articles)
+                {
+                    var tHash = a.TitleHash ?? ComputeTitleHash(a.Title);
+                    var sHash = a.SimHash ?? 0L;
+                    var normTitle = NormalizeTitle(a.Title);
+
+                    _titleHashIndex[tHash] = a.Id;
+
+                    if (sHash != 0L)
+                    {
+                        _simHashIndex.Add(new CachedArticleEntry(a.Id, tHash, sHash, normTitle, a.PublishedAt));
+                    }
+                }
+            }
+            finally
+            {
+                _simHashLock.ExitWriteLock();
+            }
+
+            _initialized = true;
+            await _finlyticLogger.LogInfoAsync(SettingKeys.DeduplicationChannel, "[ArticleDeduplicationService] Initialized with {Count} recent articles for duplicate detection.", articles.Count);
+        }
+        finally
+        {
+            _initLock.Release();
+        }
+    }
+
+    public DuplicateCheckResult CheckDuplicate(string title, string content, DateTime publishedAt)
+    {
+        var titleHash = ComputeTitleHash(title);
+        var simHash = ComputeSimHash(content);
+        var normalizedTitle = NormalizeTitle(title);
+
+        // 1. Exact Title Hash Match
+        if (_titleHashIndex.TryGetValue(titleHash, out var exactMatchId))
+        {
+            return new DuplicateCheckResult(
+                IsDuplicate: true,
+                DuplicateOfArticleId: exactMatchId,
+                Reason: "ExactTitleHashMatch",
+                TitleHash: titleHash,
+                SimHash: simHash
+            );
+        }
+
+        _simHashLock.EnterReadLock();
+        try
+        {
+            var windowCutoff = publishedAt.AddDays(-7);
+
+            foreach (var entry in _simHashIndex)
+            {
+                if (entry.PublishedAt < windowCutoff) continue;
+
+                // 2. SimHash Content Similarity (Hamming Distance <= 3 bits -> > 90% identical text)
+                if (simHash != 0L && entry.SimHash != 0L)
+                {
+                    int distance = HammingDistance(simHash, entry.SimHash);
+                    if (distance <= 3)
+                    {
+                        return new DuplicateCheckResult(
+                            IsDuplicate: true,
+                            DuplicateOfArticleId: entry.ArticleId,
+                            Reason: $"SimHashSimilarity (HammingDistance: {distance})",
+                            TitleHash: titleHash,
+                            SimHash: simHash
+                        );
+                    }
+                }
+
+                // 3. Fuzzy Title Similarity (3-gram Jaccard Index >= 0.85)
+                double titleSim = Calculate3GramJaccard(normalizedTitle, entry.NormalizedTitle);
+                if (titleSim >= 0.85)
+                {
+                    return new DuplicateCheckResult(
+                        IsDuplicate: true,
+                        DuplicateOfArticleId: entry.ArticleId,
+                        Reason: $"FuzzyTitleSimilarity ({titleSim:P0})",
+                        TitleHash: titleHash,
+                        SimHash: simHash
+                    );
+                }
+            }
+        }
+        finally
+        {
+            _simHashLock.ExitReadLock();
+        }
+
+        return new DuplicateCheckResult(
+            IsDuplicate: false,
+            DuplicateOfArticleId: null,
+            Reason: null,
+            TitleHash: titleHash,
+            SimHash: simHash
+        );
+    }
+
+    public void RegisterArticle(Guid articleId, string titleHash, long simHash, string title, DateTime publishedAt)
+    {
+        _titleHashIndex[titleHash] = articleId;
+
+        _simHashLock.EnterWriteLock();
+        try
+        {
+            _simHashIndex.Add(new CachedArticleEntry(articleId, titleHash, simHash, NormalizeTitle(title), publishedAt));
+        }
+        finally
+        {
+            _simHashLock.ExitWriteLock();
+        }
+    }
+
+    public string ComputeTitleHash(string title)
+    {
+        var normalized = NormalizeTitle(title);
+        var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(normalized));
+        return Convert.ToHexString(bytes).ToLowerInvariant();
+    }
+
+    public long ComputeSimHash(string content)
+    {
+        if (string.IsNullOrWhiteSpace(content)) return 0L;
+
+        // Clean and tokenize content into words
+        var clean = NonAlphanumericRegex.Replace(content.ToLowerInvariant(), " ");
+        var words = clean.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+        if (words.Length < 10) return 0L;
+
+        // Generate 3-word shingles
+        var v = new int[64];
+        for (int i = 0; i < words.Length - 2; i++)
+        {
+            var shingle = $"{words[i]} {words[i + 1]} {words[i + 2]}";
+            var hash = Hash64(shingle);
+
+            for (int bit = 0; bit < 64; bit++)
+            {
+                if (((hash >> bit) & 1L) == 1L)
+                {
+                    v[bit]++;
+                }
+                else
+                {
+                    v[bit]--;
+                }
+            }
+        }
+
+        long simHash = 0L;
+        for (int bit = 0; bit < 64; bit++)
+        {
+            if (v[bit] > 0)
+            {
+                simHash |= (1L << bit);
+            }
+        }
+
+        return simHash;
+    }
+
+    private static string NormalizeTitle(string title)
+    {
+        if (string.IsNullOrWhiteSpace(title)) return string.Empty;
+        var stripped = TitlePortalSuffixRegex.Replace(title, "");
+        var cleaned = NonAlphanumericRegex.Replace(stripped.ToLowerInvariant(), " ");
+        return MultipleSpacesRegex.Replace(cleaned, " ").Trim();
+    }
+
+    private static double Calculate3GramJaccard(string a, string b)
+    {
+        if (string.IsNullOrEmpty(a) || string.IsNullOrEmpty(b)) return 0.0;
+        if (a == b) return 1.0;
+
+        var gramsA = Get3Grams(a);
+        var gramsB = Get3Grams(b);
+
+        if (gramsA.Count == 0 || gramsB.Count == 0) return 0.0;
+
+        int intersection = 0;
+        foreach (var g in gramsA)
+        {
+            if (gramsB.Contains(g)) intersection++;
+        }
+
+        int union = gramsA.Count + gramsB.Count - intersection;
+        return union == 0 ? 0.0 : (double)intersection / union;
+    }
+
+    private static HashSet<string> Get3Grams(string text)
+    {
+        var set = new HashSet<string>(StringComparer.Ordinal);
+        if (text.Length < 3)
+        {
+            set.Add(text);
+            return set;
+        }
+
+        for (int i = 0; i <= text.Length - 3; i++)
+        {
+            set.Add(text.Substring(i, 3));
+        }
+        return set;
+    }
+
+    private static int HammingDistance(long a, long b)
+    {
+        return BitOperations.PopCount((ulong)(a ^ b));
+    }
+
+    private static long Hash64(string text)
+    {
+        // 64-bit FNV-1a Hash
+        ulong hash = 14695981039346656037UL;
+        var bytes = Encoding.UTF8.GetBytes(text);
+        foreach (var b in bytes)
+        {
+            hash ^= b;
+            hash *= 1099511628211UL;
+        }
+        return (long)hash;
+    }
+}

FinlyticNews/Services/AssetMatcherService.cs

@@ -0,0 +1,273 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Models.Assets;
+using FinlyticCore.Services;
+using FinlyticCore.Util;
+using FinlyticNews.Util;
+using Microsoft.Extensions.Hosting;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Candidate asset match identified in an article.
+/// </summary>
+public record MatchedCandidateAsset(
+    string Isin,
+    string Name,
+    int MentionCount,
+    bool InTitle,
+    bool InUrl,
+    double InitialConfidence
+);
+
+/// <summary>
+/// Service interface for fast in-memory asset detection from news text, titles, and URLs without AI.
+/// </summary>
+public interface IAssetMatcherService
+{
+    /// <summary>
+    /// Matches financial assets in the given text, title, and URL against the in-memory asset index.
+    /// </summary>
+    List<MatchedCandidateAsset> MatchAssets(string title, string content, string? url = null);
+
+    /// <summary>
+    /// Forces a reload of the index.json from disk into memory.
+    /// </summary>
+    Task ReloadIndexAsync(CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// In-memory asset matching engine supporting ISIN regex, normalized name matching, legal suffix stripping, and stopword defenses.
+/// </summary>
+public class AssetMatcherService : IAssetMatcherService, IHostedService
+{
+    private record CompiledNameMatcher(
+        AssetIndex Asset,
+        string CleanName,
+        Regex WordRegex,
+        bool IsShortOrCommon
+    );
+
+    private readonly IFinlyticLogger<AssetMatcherService> _finlyticLogger;
+    private readonly string _indexPath;
+
+    private readonly Dictionary<string, AssetIndex> _isinLookup = new(StringComparer.OrdinalIgnoreCase);
+    private readonly List<CompiledNameMatcher> _nameMatchers = [];
+    private readonly ReaderWriterLockSlim _indexLock = new();
+
+    private static readonly Regex IsinRegex = new(@"\b[A-Z]{2}[A-Z0-9]{9}[0-9]\b", RegexOptions.Compiled);
+    private static readonly Regex FinancialContextRegex = new(
+        @"(aktie|aktien|kurs|kurse|börse|finanz|umsatz|gewinn|quartal|prognose|dividende|kgv|analyst|kursziel|konzern|shares|stock|stocks|revenue|earnings|ebitda|ceo|cfo|nasdaq|dow|dax|s&p|\$|€)",
+        RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+    private static readonly Regex LegalSuffixRegex = new(
+        @"\s+(AG|SE|GmbH|KGaA|Inc\.?|Incorporated|Corp\.?|Corporation|Ltd\.?|Limited|PLC|Plc|SA|NV|Holdings?|Group|Co\.?|Class\s+[A-Z]|ADR|SpA|Oyj|AB|A\/S|N\.V\.|S\.A\.|S\.E\.)\b",
+        RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+    private static readonly HashSet<string> CommonWordStopwords = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "CAN", "IT", "ALL", "BE", "NEXT", "MAN", "GAP", "ON", "FOR", "NOW", "ONE", "TOP", "REAL", "DO", "GO", "US",
+        "DIE", "DER", "DAS", "UND", "VON", "MIT", "AUS", "IN", "AN", "ZU", "WAR", "AUF", "TAG", "PLUS", "FAST", "BEST",
+        "NEW", "AIR", "SEE", "MAX", "PRO", "NET", "HOME", "WORK", "CAR", "AUTO", "CARE", "LIFE", "WELL", "PURE", "TRUE",
+        "APP", "BOX", "HUB", "KEY", "PAY", "BUY", "WIN", "RUN", "SET", "GET", "RED", "BIG", "SUN", "SEA", "STAR", "BAY"
+    };
+
+    public AssetMatcherService(IFinlyticLogger<AssetMatcherService> finlyticLogger)
+    {
+        _finlyticLogger = finlyticLogger;
+        _indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json");
+    }
+
+    public async Task StartAsync(CancellationToken cancellationToken)
+    {
+        await ReloadIndexAsync(cancellationToken);
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+
+    public async Task ReloadIndexAsync(CancellationToken cancellationToken = default)
+    {
+        if (!File.Exists(_indexPath))
+        {
+            await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file not found at: {Path}", _indexPath);
+            return;
+        }
+
+        try
+        {
+            await using var stream = File.OpenRead(_indexPath);
+            var assets = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream, cancellationToken: cancellationToken);
+
+            if (assets == null || assets.Count == 0)
+            {
+                await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Index file is empty.");
+                return;
+            }
+
+            _indexLock.EnterWriteLock();
+            try
+            {
+                _isinLookup.Clear();
+                _nameMatchers.Clear();
+
+                foreach (var asset in assets)
+                {
+                    if (string.IsNullOrWhiteSpace(asset.Isin)) continue;
+                    _isinLookup[asset.Isin.Trim()] = asset;
+
+                    if (!string.IsNullOrWhiteSpace(asset.Name))
+                    {
+                        var cleanName = CleanCompanyName(asset.Name);
+                        if (cleanName.Length >= 2)
+                        {
+                            bool isShort = cleanName.Length < 4 || CommonWordStopwords.Contains(cleanName);
+                            var pattern = $@"\b{Regex.Escape(cleanName)}\b";
+                            var regex = new Regex(pattern, isShort ? RegexOptions.Compiled : RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+                            _nameMatchers.Add(new CompiledNameMatcher(asset, cleanName, regex, isShort));
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                _indexLock.ExitWriteLock();
+            }
+
+            await _finlyticLogger.LogInfoAsync(SettingKeys.MatcherChannel, "[AssetMatcherService] Successfully loaded {Count} assets and {MatcherCount} name matchers into memory.", _isinLookup.Count, _nameMatchers.Count);
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogErrorAsync(SettingKeys.MatcherChannel, ex, "[AssetMatcherService] Failed to load index from {Path}", _indexPath);
+        }
+    }
+
+    public List<MatchedCandidateAsset> MatchAssets(string title, string content, string? url = null)
+    {
+        var results = new Dictionary<string, MatchedCandidateAsset>(StringComparer.OrdinalIgnoreCase);
+        var fullText = (title + " " + content);
+
+        _indexLock.EnterReadLock();
+        try
+        {
+            // 1. URL Mining for ISINs (100% confidence)
+            if (!string.IsNullOrWhiteSpace(url))
+            {
+                var isinMatchesInUrl = IsinRegex.Matches(url);
+                foreach (Match m in isinMatchesInUrl)
+                {
+                    if (_isinLookup.TryGetValue(m.Value, out var asset))
+                    {
+                        results[asset.Isin] = new MatchedCandidateAsset(
+                            Isin: asset.Isin,
+                            Name: asset.Name,
+                            MentionCount: 1,
+                            InTitle: false,
+                            InUrl: true,
+                            InitialConfidence: 1.0
+                        );
+                    }
+                }
+            }
+
+            // 2. ISIN Regex in Title & Content
+            var isinMatches = IsinRegex.Matches(fullText);
+            foreach (Match m in isinMatches)
+            {
+                if (_isinLookup.TryGetValue(m.Value, out var asset))
+                {
+                    bool inTitle = title.Contains(m.Value, StringComparison.OrdinalIgnoreCase);
+                    int count = isinMatches.Count(x => x.Value.Equals(m.Value, StringComparison.OrdinalIgnoreCase));
+
+                    results[asset.Isin] = new MatchedCandidateAsset(
+                        Isin: asset.Isin,
+                        Name: asset.Name,
+                        MentionCount: count,
+                        InTitle: inTitle,
+                        InUrl: results.TryGetValue(asset.Isin, out var existing) && existing.InUrl,
+                        InitialConfidence: inTitle ? 1.0 : 0.95
+                    );
+                }
+            }
+
+            // 3. Name Matchers (Title & Content)
+            bool hasFinancialContext = FinancialContextRegex.IsMatch(fullText);
+
+            foreach (var matcher in _nameMatchers)
+            {
+                if (results.ContainsKey(matcher.Asset.Isin)) continue;
+
+                // Check Title First
+                bool titleMatch = matcher.WordRegex.IsMatch(title);
+                if (titleMatch)
+                {
+                    // If short/common, ensure financial context exists
+                    if (!matcher.IsShortOrCommon || hasFinancialContext)
+                    {
+                        results[matcher.Asset.Isin] = new MatchedCandidateAsset(
+                            Isin: matcher.Asset.Isin,
+                            Name: matcher.Asset.Name,
+                            MentionCount: 1,
+                            InTitle: true,
+                            InUrl: false,
+                            InitialConfidence: 0.9
+                        );
+                        continue;
+                    }
+                }
+
+                // Check Content
+                var contentMatches = matcher.WordRegex.Matches(content);
+                if (contentMatches.Count > 0)
+                {
+                    if (matcher.IsShortOrCommon)
+                    {
+                        // Short words require >= 2 mentions AND financial context
+                        if (contentMatches.Count >= 2 && hasFinancialContext)
+                        {
+                            results[matcher.Asset.Isin] = new MatchedCandidateAsset(
+                                Isin: matcher.Asset.Isin,
+                                Name: matcher.Asset.Name,
+                                MentionCount: contentMatches.Count,
+                                InTitle: false,
+                                InUrl: false,
+                                InitialConfidence: 0.75
+                            );
+                        }
+                    }
+                    else
+                    {
+                        results[matcher.Asset.Isin] = new MatchedCandidateAsset(
+                            Isin: matcher.Asset.Isin,
+                            Name: matcher.Asset.Name,
+                            MentionCount: contentMatches.Count,
+                            InTitle: false,
+                            InUrl: false,
+                            InitialConfidence: contentMatches.Count >= 2 ? 0.85 : 0.7
+                        );
+                    }
+                }
+            }
+        }
+        finally
+        {
+            _indexLock.ExitReadLock();
+        }
+
+        return results.Values.OrderByDescending(r => r.InitialConfidence).ThenByDescending(r => r.MentionCount).ToList();
+    }
+
+    private static string CleanCompanyName(string name)
+    {
+        if (string.IsNullOrWhiteSpace(name)) return string.Empty;
+        var stripped = LegalSuffixRegex.Replace(name, "").Trim();
+        return stripped;
+    }
+}

FinlyticNews/Services/AssetValidationService.cs

@@ -0,0 +1,151 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Dtos.Assets;
+using FinlyticCore.Models.Assets;
+using FinlyticCore.Services;
+using FinlyticNews.Util;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Service interface for validating candidate asset matches against FinlyticAssets metadata and sector clustering.
+/// </summary>
+public interface IAssetValidationService
+{
+    /// <summary>
+    /// Validates and filters candidate asset matches using FinlyticAssets metadata, sector clustering, and occurrence rules.
+    /// </summary>
+    Task<List<MatchedCandidateAsset>> ValidateCandidateAssetsAsync(
+        List<MatchedCandidateAsset> candidates,
+        string title,
+        string content,
+        CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Heuristic asset validation engine verifying asset existence, title priority, mention density, and sector coherence.
+/// </summary>
+public class AssetValidationService : IAssetValidationService
+{
+    private readonly IFinlyticLogger<AssetValidationService> _finlyticLogger;
+    private readonly NewsMqttClient _mqttClient;
+    private readonly ISettingsService _settingsService;
+
+    public AssetValidationService(
+        IFinlyticLogger<AssetValidationService> finlyticLogger,
+        NewsMqttClient mqttClient,
+        ISettingsService settingsService)
+    {
+        _finlyticLogger = finlyticLogger;
+        _mqttClient = mqttClient;
+        _settingsService = settingsService;
+    }
+
+    public async Task<List<MatchedCandidateAsset>> ValidateCandidateAssetsAsync(
+        List<MatchedCandidateAsset> candidates,
+        string title,
+        string content,
+        CancellationToken cancellationToken = default)
+    {
+        if (candidates.Count == 0) return [];
+
+        var verifiedAssets = new List<MatchedCandidateAsset>();
+        var assetDetails = new Dictionary<string, AssetDto>(StringComparer.OrdinalIgnoreCase);
+
+        // 1. Fetch Asset Details from FinlyticAssets via MQTT RPC
+        foreach (var candidate in candidates)
+        {
+            try
+            {
+                var req = new GetValidAssetRequest(candidate.Isin);
+                var assets = await _mqttClient.RequestAsync<GetValidAssetRequest, List<AssetDto>>("services/request/assets_Get", req);
+
+                if (assets != null && assets.Count > 0)
+                {
+                    var match = assets.FirstOrDefault(a => a.Isin.Equals(candidate.Isin, StringComparison.OrdinalIgnoreCase));
+                    if (match != null)
+                    {
+                        assetDetails[candidate.Isin] = match;
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                await _finlyticLogger.LogWarningAsync(SettingKeys.MatcherChannel, ex, "[AssetValidationService] Failed to query FinlyticAssets for ISIN {Isin}. Falling back to index data.", candidate.Isin);
+            }
+        }
+
+        // 2. Tag & Sector Frequency Analysis (Clustering)
+        var sectorCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+        foreach (var (_, dto) in assetDetails)
+        {
+            if (dto.Tags != null)
+            {
+                foreach (var tag in dto.Tags)
+                {
+                    if (tag.Type.Equals("sector", StringComparison.OrdinalIgnoreCase) ||
+                        tag.Type.Equals("industry", StringComparison.OrdinalIgnoreCase))
+                    {
+                        sectorCounts[tag.Name] = sectorCounts.GetValueOrDefault(tag.Name, 0) + 1;
+                    }
+                }
+            }
+        }
+
+        bool hasDominantSector = sectorCounts.Values.Any(count => count >= 2);
+        string? dominantSector = sectorCounts.OrderByDescending(kv => kv.Value).FirstOrDefault().Key;
+
+        // 3. Evaluate Each Candidate
+        foreach (var candidate in candidates)
+        {
+            // Rule A: Direct Title Match or URL Match -> Guaranteed Valid
+            if (candidate.InTitle || candidate.InUrl)
+            {
+                verifiedAssets.Add(candidate);
+                continue;
+            }
+
+            // Rule B: Multiple Mentions in Article Body
+            if (candidate.MentionCount >= 2)
+            {
+                verifiedAssets.Add(candidate);
+                continue;
+            }
+
+            // Rule C: Single Mention Validation with Sector Clustering
+            if (assetDetails.TryGetValue(candidate.Isin, out var dto))
+            {
+                bool matchesDominantSector = false;
+                if (hasDominantSector && dto.Tags != null)
+                {
+                    matchesDominantSector = dto.Tags.Any(t => t.Name.Equals(dominantSector, StringComparison.OrdinalIgnoreCase));
+                }
+
+                if (matchesDominantSector)
+                {
+                    // Reinforced by cluster
+                    verifiedAssets.Add(candidate);
+                }
+                else if (candidate.InitialConfidence >= 0.85)
+                {
+                    // High-confidence exact unique name match
+                    verifiedAssets.Add(candidate);
+                }
+                else
+                {
+                    await _finlyticLogger.LogDebugAsync(SettingKeys.MatcherChannel, "[AssetValidationService] Pruned low-confidence outlier candidate: {Name} ({Isin})", candidate.Name, candidate.Isin);
+                }
+            }
+            else
+            {
+                // If not found in FinlyticAssets, discard
+                await _finlyticLogger.LogDebugAsync(SettingKeys.MatcherChannel, "[AssetValidationService] Discarded candidate {Isin} as it does not exist in FinlyticAssets.", candidate.Isin);
+            }
+        }
+
+        return verifiedAssets;
+    }
+}

FinlyticNews/Services/N8nService.cs

@@ -1,141 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Net.Http;
-using System.Text.Json;
-using System.Threading;
-using System.Threading.Tasks;
-using FinlyticCore.Dtos.News;
-using FinlyticCore.Services;
-using FinlyticCore.Util;
-using FinlyticNews.Util;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-
-namespace FinlyticNews.Services;
-
-/// <summary>
-/// Defines integration operations with the external n8n AI workflow webhook.
-/// </summary>
-public interface IN8nService
-{
-    /// <summary>
-    /// Submits raw article content and pre-filtered assets to n8n, returning the parsed response metadata.
-    /// </summary>
-    Task<N8nResponsePayload?> AnalyzeArticleAsync(string content, List<FilteredAssetPayload> filteredAssets, CancellationToken ct = default);
-}
-
-/// <inheritdoc />
-public class N8nService : IN8nService
-{
-    private readonly HttpClient _httpClient;
-    private readonly IServiceScopeFactory _scopeFactory;
-    private readonly IConfiguration _configuration;
-    private readonly IFinlyticLogger<N8nService> _finlyticLogger;
-
-    public N8nService(
-        HttpClient httpClient,
-        IServiceScopeFactory scopeFactory,
-        IConfiguration configuration,
-        IFinlyticLogger<N8nService> finlyticLogger)
-    {
-        _httpClient = httpClient;
-        _scopeFactory = scopeFactory;
-        _configuration = configuration;
-        _finlyticLogger = finlyticLogger;
-    }
-
-    /// <inheritdoc />
-    public async Task<N8nResponsePayload?> AnalyzeArticleAsync(string content, List<FilteredAssetPayload> filteredAssets, CancellationToken ct = default)
-    {
-        string? targetUrl = null;
-
-        using (var scope = _scopeFactory.CreateScope())
-        {
-            var settingsService = scope.ServiceProvider.GetService<ISettingsService>();
-            if (settingsService != null)
-            {
-                var dynamicUrl = await settingsService.GetSettingAsync(SettingKeys.N8nArticleExtractionUrl, ct);
-                if (!string.IsNullOrWhiteSpace(dynamicUrl))
-                {
-                    targetUrl = dynamicUrl.Trim();
-                }
-            }
-
-            if (string.IsNullOrWhiteSpace(targetUrl))
-            {
-                var settingsDb = scope.ServiceProvider.GetService<ISettingsDbService>();
-                if (settingsDb != null)
-                {
-                    var settings = await settingsDb.GetSettingsAsync();
-                    targetUrl = settings?.N8nWebhookUrl;
-                }
-            }
-        }
-
-        if (string.IsNullOrWhiteSpace(targetUrl))
-        {
-            targetUrl = _configuration["N8N:ArticleExtractionUrl"] 
-                        ?? _configuration["N8N__ArticleExtractionUrl"];
-        }
-
-        if (string.IsNullOrWhiteSpace(targetUrl))
-        {
-            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, "[N8nService] N8nWebhookUrl is not configured in DB or application settings.");
-            return null;
-        }
-
-        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[N8nService] Posting article to n8n webhook pipeline at: {Url}", targetUrl);
-
-        var payload = new N8nRequestPayload(content, filteredAssets ?? []);
-
-        try
-        {
-            var jsonContent = JsonSerializer.Serialize(payload, FinlyticJsonSerializerContext.Default.N8nRequestPayload);
-            using var requestContent = new StringContent(jsonContent, System.Text.Encoding.UTF8, "application/json");
-
-            using var response = await _httpClient.PostAsync(targetUrl, requestContent, ct);
-
-            if (!response.IsSuccessStatusCode)
-            {
-                var errorMsg = await response.Content.ReadAsStringAsync(ct);
-                await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, "[N8nService] n8n webhook returned status code {StatusCode}. Error payload: {Error}", response.StatusCode, errorMsg);
-                return null;
-            }
-
-            using var responseStream = await response.Content.ReadAsStreamAsync(ct);
-            using var doc = await JsonDocument.ParseAsync(responseStream, cancellationToken: ct);
-            
-            var root = doc.RootElement;
-
-            if (root.ValueKind == JsonValueKind.Array)
-            {
-                if (root.GetArrayLength() == 0)
-                {
-                    await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[N8nService] n8n webhook returned an empty array.");
-                    return null;
-                }
-                root = root[0];
-            }
-
-            if (root.ValueKind == JsonValueKind.Object)
-            {
-                if (root.TryGetProperty("json", out var jsonChild) && jsonChild.ValueKind == JsonValueKind.Object)
-                    root = jsonChild;
-                else if (root.TryGetProperty("output", out var outChild) && outChild.ValueKind == JsonValueKind.Object)
-                    root = outChild;
-                else if (root.TryGetProperty("data", out var dataChild) && dataChild.ValueKind == JsonValueKind.Object)
-                    root = dataChild;
-                else if (root.TryGetProperty("body", out var bodyChild) && bodyChild.ValueKind == JsonValueKind.Object)
-                    root = bodyChild;
-            }
-
-            var result = root.Deserialize(FinlyticJsonSerializerContext.Default.N8nResponsePayload);
-            return result;
-        }
-        catch (Exception ex)
-        {
-            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[N8nService] Failed to communicate with or parse response from n8n webhook workflow.");
-            return null;
-        }
-    }
-}

FinlyticNews/Services/NewsBlocklistService.cs

@@ -0,0 +1,126 @@
+using System;
+using System.Collections.Concurrent;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FinlyticCore.Services;
+using FinlyticNews.Database;
+using FinlyticNews.Entities;
+using FinlyticNews.Util;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FinlyticNews.Services;
+
+/// <summary>
+/// Service interface for managing and querying blocked news article URLs (duplicates, missing content, or zero matched assets).
+/// </summary>
+public interface INewsBlocklistService
+{
+    /// <summary>
+    /// Checks whether the given URL is already recorded on the blocklist.
+    /// </summary>
+    bool IsBlocked(string url);
+
+    /// <summary>
+    /// Adds a URL to the database blocklist and in-memory cache.
+    /// </summary>
+    Task BlockUrlAsync(string url, string reason, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Loads all existing blocked URLs from the database into memory on startup.
+    /// </summary>
+    Task InitializeAsync(CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// High-performance in-memory and database-backed blocklist service.
+/// </summary>
+public class NewsBlocklistService : INewsBlocklistService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly IFinlyticLogger<NewsBlocklistService> _finlyticLogger;
+    private readonly ConcurrentDictionary<string, string> _blockedUrls = new(StringComparer.OrdinalIgnoreCase);
+    private bool _initialized;
+    private readonly SemaphoreSlim _initLock = new(1, 1);
+
+    public NewsBlocklistService(
+        IServiceScopeFactory scopeFactory,
+        IFinlyticLogger<NewsBlocklistService> finlyticLogger)
+    {
+        _scopeFactory = scopeFactory;
+        _finlyticLogger = finlyticLogger;
+    }
+
+    public async Task InitializeAsync(CancellationToken cancellationToken = default)
+    {
+        if (_initialized) return;
+
+        await _initLock.WaitAsync(cancellationToken);
+        try
+        {
+            if (_initialized) return;
+
+            using var scope = _scopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();
+
+            var urls = await dbContext.BlockedNewsUrls
+                .AsNoTracking()
+                .Select(b => new { b.Url, b.Reason })
+                .ToListAsync(cancellationToken);
+
+            foreach (var item in urls)
+            {
+                if (!string.IsNullOrWhiteSpace(item.Url))
+                {
+                    _blockedUrls[item.Url.Trim()] = item.Reason ?? string.Empty;
+                }
+            }
+
+            _initialized = true;
+            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsBlocklistService] Initialized with {Count} blocked URLs in memory cache.", _blockedUrls.Count);
+        }
+        finally
+        {
+            _initLock.Release();
+        }
+    }
+
+    public bool IsBlocked(string url)
+    {
+        if (string.IsNullOrWhiteSpace(url)) return true;
+        return _blockedUrls.ContainsKey(url.Trim());
+    }
+
+    public async Task BlockUrlAsync(string url, string reason, CancellationToken cancellationToken = default)
+    {
+        if (string.IsNullOrWhiteSpace(url)) return;
+        var cleanUrl = url.Trim();
+
+        _blockedUrls[cleanUrl] = reason;
+
+        try
+        {
+            using var scope = _scopeFactory.CreateScope();
+            var dbContext = scope.ServiceProvider.GetRequiredService<NewsDbContext>();
+
+            bool exists = await dbContext.BlockedNewsUrls.AnyAsync(b => b.Url == cleanUrl, cancellationToken);
+            if (!exists)
+            {
+                dbContext.BlockedNewsUrls.Add(new BlockedNewsUrlEntity
+                {
+                    Id = Guid.NewGuid(),
+                    Url = cleanUrl,
+                    Reason = reason,
+                    BlockedAtUtc = DateTime.UtcNow
+                });
+                await dbContext.SaveChangesAsync(cancellationToken);
+                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsBlocklistService] Added URL to blocklist: {Url} (Reason: {Reason})", cleanUrl, reason);
+            }
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[NewsBlocklistService] Failed to persist blocked URL: {Url}", cleanUrl);
+        }
+    }
+}

FinlyticNews/Services/NewsDbService.cs

@@ -22,7 +22,7 @@ public interface INewsDbService
     /// <summary>
     /// Phase 1: Discovers and locks a new article URL by setting its status to "Pending".
     /// </summary>
-    Task<NewsArticleEntity> CreatePendingArticleAsync(
+    Task<NewsArticleEntity?> CreatePendingArticleAsync(
         string url,
         List<string>? discoveredIsins = null,
         string? title = null,
@@ -35,15 +35,30 @@ public interface INewsDbService
     /// </summary>
     Task UpdateArticleStatusAsync(Guid id, string status);
 
+    /// <summary>
+    /// Deletes an article from the database entirely (e.g. when it fails or is a duplicate).
+    /// </summary>
+    Task DeleteArticleAsync(Guid id);
+
     /// <summary>
     /// Updates the target URL of an article if a redirect is resolved during the Processing phase.
     /// </summary>
     Task UpdateArticleUrlAsync(Guid id, string resolvedUrl);
 
     /// <summary>
-    /// Phase 4: Saves AI classification from n8n and sets the lifecycle status to "Completed".
+    /// Saves the processed article content, hashes, and matched assets, and transitions lifecycle status to "Completed".
     /// </summary>
-    Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets);
+    Task<NewsArticleEntity?> SaveProcessedArticleAsync(
+        Guid id,
+        string title,
+        string? author,
+        string? summary,
+        string contentRaw,
+        string? language,
+        DateTime? publishedAt,
+        string titleHash,
+        long simHash,
+        List<MatchedAssetEntity> matchedAssets);
 
     Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(string status);
 
@@ -95,7 +110,7 @@ public class NewsDbService : INewsDbService
     }
 
     /// <inheritdoc />
-    public async Task<NewsArticleEntity> CreatePendingArticleAsync(
+    public async Task<NewsArticleEntity?> CreatePendingArticleAsync(
         string url,
         List<string>? discoveredIsins = null,
         string? title = null,
@@ -112,7 +127,7 @@ public class NewsDbService : INewsDbService
         if (existingArticle != null)
         {
             await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Article URL already exists (Duplicate hit): {Url}", trimmedUrl);
-            return existingArticle;
+            return null;
         }
 
         var finalPublishedAt = publishedAt.HasValue
@@ -166,9 +181,9 @@ public class NewsDbService : INewsDbService
             var existing = await _context.NewsArticles.AsNoTracking().FirstOrDefaultAsync(a => a.SourceUrl == trimmedUrl || a.SourceUrl == cleanUrl || a.SourceUrl.StartsWith(cleanUrl));
             if (existing != null)
             {
-                return existing;
+                return null;
             }
-            return article;
+            return null;
         }
 
         return article;
@@ -191,6 +206,19 @@ public class NewsDbService : INewsDbService
         }
     }
 
+    /// <inheritdoc />
+    public async Task DeleteArticleAsync(Guid id)
+    {
+        var rowsAffected = await _context.NewsArticles
+            .Where(a => a.Id == id)
+            .ExecuteDeleteAsync();
+
+        if (rowsAffected > 0)
+        {
+            await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Deleted unused/failed article {Id} from database", id);
+        }
+    }
+
     /// <inheritdoc />
     public async Task UpdateArticleUrlAsync(Guid id, string resolvedUrl)
     {
@@ -209,37 +237,41 @@ public class NewsDbService : INewsDbService
     }
 
     /// <inheritdoc />
-    public async Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets)
+    public async Task<NewsArticleEntity?> SaveProcessedArticleAsync(
+        Guid id,
+        string title,
+        string? author,
+        string? summary,
+        string contentRaw,
+        string? language,
+        DateTime? publishedAt,
+        string titleHash,
+        long simHash,
+        List<MatchedAssetEntity> matchedAssets)
     {
-        DateTime? finalPublishedAt = null;
-        if (DateTime.TryParse(payload.PublishedAt, out var publishedDate))
-        {
-            finalPublishedAt = publishedDate.Kind == DateTimeKind.Unspecified
-                ? DateTime.SpecifyKind(publishedDate, DateTimeKind.Utc)
-                : publishedDate.ToUniversalTime();
-        }
-
-        DateTime? finalScrapedAt = null;
-        if (DateTime.TryParse(payload.ScrapedAt, out var scrapedDate))
-        {
-            finalScrapedAt = scrapedDate.ToUniversalTime();
-        }
+        var finalPublishedAt = publishedAt.HasValue
+            ? (publishedAt.Value.Kind == DateTimeKind.Unspecified
+                ? DateTime.SpecifyKind(publishedAt.Value, DateTimeKind.Utc)
+                : publishedAt.Value.ToUniversalTime())
+            : DateTime.UtcNow;
 
         var rowsAffected = await _context.NewsArticles
             .Where(a => a.Id == id)
             .ExecuteUpdateAsync(s => s
-                .SetProperty(a => a.Title, payload.Title)
-                .SetProperty(a => a.Author, payload.Author)
-                .SetProperty(a => a.Summary, payload.Summary)
-                .SetProperty(a => a.ContentRaw, payload.ContentRaw)
-                .SetProperty(a => a.Language, payload.Language)
+                .SetProperty(a => a.Title, title)
+                .SetProperty(a => a.Author, author)
+                .SetProperty(a => a.Summary, summary)
+                .SetProperty(a => a.ContentRaw, contentRaw)
+                .SetProperty(a => a.Language, language)
+                .SetProperty(a => a.TitleHash, titleHash)
+                .SetProperty(a => a.SimHash, simHash)
                 .SetProperty(a => a.Status, "Completed")
-                .SetProperty(a => a.PublishedAt, a => finalPublishedAt ?? a.PublishedAt)
-                .SetProperty(a => a.ScrapedAt, a => finalScrapedAt ?? a.ScrapedAt));
+                .SetProperty(a => a.PublishedAt, finalPublishedAt)
+                .SetProperty(a => a.ScrapedAt, DateTime.UtcNow));
 
         if (rowsAffected == 0)
         {
-            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Article with ID {Id} not found for classification update.", id);
+            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Article with ID {Id} not found for completion update.", id);
             return null;
         }
 
@@ -267,7 +299,7 @@ public class NewsDbService : INewsDbService
             .AsNoTracking()
             .FirstOrDefaultAsync(a => a.Id == id);
 
-        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[Lifecycle] Article {Id} successfully classified and marked 'Completed'. Title: '{Title}'", id, payload.Title);
+        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[Lifecycle] Article {Id} successfully processed and marked 'Completed'. Title: '{Title}'", id, title);
 
         return completedArticle;
     }
@@ -320,7 +352,7 @@ public class NewsDbService : INewsDbService
             var cleanSearch = searchQuery.Trim().ToLower();
             query = query.Where(a =>
                 a.Title.ToLower().Contains(cleanSearch) ||
-                a.Summary.ToLower().Contains(cleanSearch) ||
+                (a.Summary != null && a.Summary.ToLower().Contains(cleanSearch)) ||
                 a.MatchedAssets.Any(m => m.Name.ToLower().Contains(cleanSearch)));
         }
 

FinlyticNews/Services/NewsScraperBackgroundService.cs

@@ -1,62 +1,65 @@
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using System.Text.Json;
-using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
-using FinlyticAssets.Models;
-using FinlyticAssets.Util;
 using FinlyticCore.Dtos.News;
 using FinlyticCore.Services;
-using FinlyticCore.Util;
 using FinlyticNews.Entities;
 using FinlyticNews.Util;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
 namespace FinlyticNews.Services;
 
 /// <summary>
-/// A background worker service that orchestrates link discovery, Playwright scraping,
-/// pre-filtering, n8n AI enrichment, database persistence, and MQTT broadcasts.
+/// Background worker service orchestrating link discovery, Playwright scraping,
+/// non-AI deduplication, in-memory asset matching, plausibility validation, and MQTT broadcasts.
 /// </summary>
 public class NewsScraperBackgroundService : BackgroundService
 {
-    private record CompiledAssetMatcher(
-        AssetIndex Asset,
-        string CoreName,
-        Regex? WordRegex,
-        Regex? CoreWordRegex
-    );
-
     private readonly IServiceScopeFactory _scopeFactory;
     private readonly IFinlyticLogger<NewsScraperBackgroundService> _finlyticLogger;
     private readonly NewsMqttClient _mqttClient;
-    private readonly string _indexPath;
-
-    private List<CompiledAssetMatcher>? _cachedAssetMatchers;
-    private DateTime _lastIndexLoadTime = DateTime.MinValue;
+    private readonly INewsBlocklistService _blocklistService;
+    private readonly IArticleDeduplicationService _deduplicationService;
+    private readonly IAssetMatcherService _assetMatcherService;
+    private readonly IAssetValidationService _assetValidationService;
 
     public NewsScraperBackgroundService(
         IServiceScopeFactory scopeFactory,
         IFinlyticLogger<NewsScraperBackgroundService> finlyticLogger,
         NewsMqttClient mqttClient,
-        IConfiguration configuration)
+        INewsBlocklistService blocklistService,
+        IArticleDeduplicationService deduplicationService,
+        IAssetMatcherService assetMatcherService,
+        IAssetValidationService assetValidationService)
     {
         _scopeFactory = scopeFactory;
         _finlyticLogger = finlyticLogger;
         _mqttClient = mqttClient;
-        _indexPath = Path.Combine(Volumes.IndexRelativePath, "index.json");
+        _blocklistService = blocklistService;
+        _deduplicationService = deduplicationService;
+        _assetMatcherService = assetMatcherService;
+        _assetValidationService = assetValidationService;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] NewsScraperBackgroundService started.");
 
+        // Initialize in-memory blocklist and deduplication cache on startup
+        try
+        {
+            await _blocklistService.InitializeAsync(stoppingToken);
+            await _deduplicationService.InitializeAsync(stoppingToken);
+            await _assetMatcherService.ReloadIndexAsync(stoppingToken);
+        }
+        catch (Exception ex)
+        {
+            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Error during startup service initialization.");
+        }
+
         while (!stoppingToken.IsCancellationRequested)
         {
             try
@@ -76,7 +79,7 @@ public class NewsScraperBackgroundService : BackgroundService
             }
             catch (Exception ex) when (ex is not OperationCanceledException)
             {
-                await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] An unhandled exception occurred during news scraping cycle.");
+                await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Unhandled exception in news scraping cycle.");
             }
 
             int intervalMinutes = 15;
@@ -89,7 +92,7 @@ public class NewsScraperBackgroundService : BackgroundService
             catch { }
 
             var jitterSeconds = Random.Shared.Next(0, 60);
-            var nextRunDelay = TimeSpan.FromMinutes(intervalMinutes) + TimeSpan.FromSeconds(jitterSeconds);
+            var nextRunDelay = TimeSpan.FromMinutes(Math.Max(1, intervalMinutes)) + TimeSpan.FromSeconds(jitterSeconds);
             await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Scraping cycle completed. Next cycle in {Delay} (interval: {Minutes}m).", nextRunDelay, intervalMinutes);
 
             try
@@ -111,29 +114,26 @@ public class NewsScraperBackgroundService : BackgroundService
         var dbService = scope.ServiceProvider.GetRequiredService<INewsDbService>();
         var discoveryService = scope.ServiceProvider.GetRequiredService<IArticleDiscoveryService>();
         var scraperService = scope.ServiceProvider.GetRequiredService<IPlaywrightScraperService>();
-        var n8nService = scope.ServiceProvider.GetRequiredService<IN8nService>();
         var settings = scope.ServiceProvider.GetRequiredService<ISettingsService>();
 
         var maxArticlesPerFeed = await settings.GetSettingAsync(SettingKeys.MaxArticlesPerFeed, stoppingToken);
 
-        var assetMatchers = await GetOrLoadAssetMatchersAsync();
-        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Loaded {Count} asset index items for text pre-filtering.", assetMatchers.Count);
-
-        var failedArticles = await dbService.GetArticlesByStatusAsync("Scraping");
-        if (failedArticles.Count > 0)
+        // Process any previously interrupted articles
+        var pendingScraping = await dbService.GetArticlesByStatusAsync("Scraping");
+        if (pendingScraping.Count > 0)
         {
-            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Found {Count} articles in status 'Scraping' that failed to scrape previously. Retrying...", failedArticles.Count);
-            foreach (var article in failedArticles)
+            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Retrying {Count} pending scraping articles...", pendingScraping.Count);
+            foreach (var article in pendingScraping)
             {
                 if (stoppingToken.IsCancellationRequested) return;
-                await ProcessSingleArticleAsync(article, scraperService, n8nService, dbService, assetMatchers, stoppingToken);
+                await ProcessSingleArticleAsync(article, scraperService, dbService, stoppingToken);
             }
         }
 
         var sources = await dbService.GetSourcesAsync();
         if (sources.Count == 0)
         {
-            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] No article sources configured in database. Skipping cycle.");
+            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] No article sources configured in database.");
             return;
         }
 
@@ -141,30 +141,35 @@ public class NewsScraperBackgroundService : BackgroundService
         {
             if (stoppingToken.IsCancellationRequested) break;
 
-            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Starting article link discovery for source: {SourceName} ({Url})", source.Name, source.Source);
+            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Discovering articles from source: {Name} ({Url})", source.Name, source.Source);
             var discoveredArticles = await discoveryService.DiscoverLinksAsync(source.Source, source.Type, stoppingToken);
 
             if (discoveredArticles == null || discoveredArticles.Count == 0)
             {
-                await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] No links discovered from source: {SourceName}", source.Name);
                 continue;
             }
 
-            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Discovered {Count} potential article links from {SourceName}.", discoveredArticles.Count, source.Name);
-
+            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Discovered {Count} potential article links from {Name}.", discoveredArticles.Count, source.Name);
             var toProcess = discoveredArticles.Take(maxArticlesPerFeed > 0 ? maxArticlesPerFeed : 20);
 
             foreach (var discovered in toProcess)
             {
                 if (stoppingToken.IsCancellationRequested) break;
 
+                // 1. Fast Blocklist & Duplicate check before creating DB entry
+                if (_blocklistService.IsBlocked(discovered.Url))
+                {
+                    await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Skipping blocked article URL: {Url}", discovered.Url);
+                    continue;
+                }
+
                 if (await dbService.IsUrlDuplicateAsync(discovered.Url))
                 {
                     await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Skipping duplicate article URL: {Url}", discovered.Url);
                     continue;
                 }
 
-                NewsArticleEntity article;
+                NewsArticleEntity? article;
                 try
                 {
                     article = await dbService.CreatePendingArticleAsync(
@@ -178,17 +183,13 @@ public class NewsScraperBackgroundService : BackgroundService
                 }
                 catch (Exception ex)
                 {
-                    await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to register initial pending state for URL: {Url}. Skipping.", discovered.Url);
+                    await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to create pending article for {Url}", discovered.Url);
                     continue;
                 }
 
-                if (article.Id == Guid.Empty)
-                {
-                    await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Created pending article has invalid/empty ID for URL: {Url}. Skipping.", discovered.Url);
-                    continue;
-                }
+                if (article == null || article.Id == Guid.Empty) continue;
 
-                await ProcessSingleArticleAsync(article, scraperService, n8nService, dbService, assetMatchers, stoppingToken);
+                await ProcessSingleArticleAsync(article, scraperService, dbService, stoppingToken);
             }
         }
     }
@@ -196,25 +197,24 @@ public class NewsScraperBackgroundService : BackgroundService
     private async Task ProcessSingleArticleAsync(
         NewsArticleEntity article,
         IPlaywrightScraperService scraperService,
-        IN8nService n8nService,
         INewsDbService dbService,
-        List<CompiledAssetMatcher> assetMatchers,
         CancellationToken stoppingToken)
     {
         try
         {
             await dbService.UpdateArticleStatusAsync(article.Id, "Processing");
 
-            var (resolvedUrl, rawContent) = await scraperService.ScrapeArticleAsync(article.SourceUrl);
+            // 1. Playwright Headless Scrape & Redirect Resolution
+            var (resolvedUrl, scrapeResult) = await scraperService.ScrapeArticleAsync(article.SourceUrl);
 
-            if (!string.IsNullOrWhiteSpace(resolvedUrl) &&
-                !resolvedUrl.Equals(article.SourceUrl, StringComparison.OrdinalIgnoreCase))
+            // 2. Redirect Handling
+            if (!string.IsNullOrWhiteSpace(resolvedUrl) && !resolvedUrl.Equals(article.SourceUrl, StringComparison.OrdinalIgnoreCase))
             {
-                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Redirect detected. Initial: {OldUrl} -> Resolved: {NewUrl}", article.SourceUrl, resolvedUrl);
-                if (await dbService.IsUrlDuplicateAsync(resolvedUrl))
+                if (_blocklistService.IsBlocked(resolvedUrl) || await dbService.IsUrlDuplicateAsync(resolvedUrl))
                 {
-                    await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Redirected URL {ResolvedUrl} is a duplicate. Terminating processing.", resolvedUrl);
-                    await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
+                    await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Redirected URL {Url} is already blocked/duplicate.", resolvedUrl);
+                    await _blocklistService.BlockUrlAsync(article.SourceUrl, "RedirectToDuplicate", stoppingToken);
+                    await dbService.DeleteArticleAsync(article.Id);
                     return;
                 }
 
@@ -222,66 +222,84 @@ public class NewsScraperBackgroundService : BackgroundService
                 article.SourceUrl = resolvedUrl;
             }
 
-            if (string.IsNullOrWhiteSpace(rawContent) || rawContent.Length < 60)
+            // 3. Content Validity Check
+            var title = !string.IsNullOrWhiteSpace(scrapeResult.Title) ? scrapeResult.Title : article.Title;
+            var content = scrapeResult.TextContent;
+
+            if (string.IsNullOrWhiteSpace(content) || content.Length < 80 || string.IsNullOrWhiteSpace(title))
             {
-                await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
+                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Article {Id} has empty or insufficient content ({Length} chars). Adding to blocklist.", article.Id, content?.Length ?? 0);
+                await _blocklistService.BlockUrlAsync(article.SourceUrl, "EmptyOrInvalidContent", stoppingToken);
+                await dbService.DeleteArticleAsync(article.Id);
                 return;
             }
 
-            var discoveredIsins = article.MatchedAssets.Select(m => m.Isin).Where(i => !string.IsNullOrEmpty(i)).ToList();
-            var preFilteredAssets = PreFilterAssets(rawContent, article.Title, assetMatchers, discoveredIsins);
+            var publishedAt = scrapeResult.PublishedAt ?? article.PublishedAt;
 
-            if (preFilteredAssets.Count == 0)
+            // 4. Non-AI Content & Title Deduplication Check (SimHash + N-Gram)
+            var dupCheck = _deduplicationService.CheckDuplicate(title, content, publishedAt);
+            if (dupCheck.IsDuplicate)
             {
-                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Pre-filtering: Article {Id} does not reference any known assets. Terminating pipeline.", article.Id);
-                await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
+                await _finlyticLogger.LogInfoAsync(SettingKeys.DeduplicationChannel, "[NewsScraperBackgroundService] Article {Id} identified as duplicate of {OriginalId} (Reason: {Reason}). Blocking URL.", article.Id, dupCheck.DuplicateOfArticleId?.ToString() ?? "Unknown", dupCheck.Reason ?? "Duplicate content");
+                await _blocklistService.BlockUrlAsync(article.SourceUrl, $"Duplicate:{dupCheck.DuplicateOfArticleId}", stoppingToken);
+                await dbService.DeleteArticleAsync(article.Id);
                 return;
             }
 
-            await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Pre-filtering matched {Count} assets for article {Id}.", preFilteredAssets.Count, article.Id);
-
-            var n8nResponse = await n8nService.AnalyzeArticleAsync(rawContent, preFilteredAssets, stoppingToken);
-            if (n8nResponse == null)
+            // 5. In-Memory Asset Matching (ISIN Regex, Name Tokenizer, Suffix Trimming)
+            var candidateMatches = _assetMatcherService.MatchAssets(title, content, article.SourceUrl);
+            if (candidateMatches.Count == 0)
             {
-                await dbService.UpdateArticleStatusAsync(article.Id, "Failed");
+                await _finlyticLogger.LogInfoAsync(SettingKeys.MatcherChannel, "[NewsScraperBackgroundService] No candidate assets matched for article {Id} ('{Title}'). Blocking URL.", article.Id, title);
+                await _blocklistService.BlockUrlAsync(article.SourceUrl, "NoMatchedAssets", stoppingToken);
+                await dbService.DeleteArticleAsync(article.Id);
                 return;
             }
 
-            var matchedEntities = new List<MatchedAssetEntity>();
-            if (n8nResponse.MatchedAssets != null && n8nResponse.MatchedAssets.Count > 0)
+            // 6. Multi-Criteria Asset Validation (Sector Clustering, Title Weighting, Existence)
+            var validatedMatches = await _assetValidationService.ValidateCandidateAssetsAsync(candidateMatches, title, content, stoppingToken);
+            if (validatedMatches.Count == 0)
             {
-                foreach (var asset in n8nResponse.MatchedAssets)
-                {
-                    var preMatch = preFilteredAssets.FirstOrDefault(p => p.Name.Equals(asset.Name, StringComparison.OrdinalIgnoreCase));
-                    var isin = preMatch?.Isin ?? asset.Ticker ?? "";
-                    if (string.IsNullOrWhiteSpace(isin)) continue;
-
-                    matchedEntities.Add(new MatchedAssetEntity
-                    {
-                        Id = Guid.NewGuid(),
-                        NewsArticleId = article.Id,
-                        Isin = isin.Trim().ToUpperInvariant(),
-                        Name = !string.IsNullOrWhiteSpace(asset.Name) ? asset.Name.Trim() : isin.Trim().ToUpperInvariant()
-                    });
-                }
+                await _finlyticLogger.LogInfoAsync(SettingKeys.MatcherChannel, "[NewsScraperBackgroundService] All candidate matches pruned during validation for article {Id}. Blocking URL.", article.Id);
+                await _blocklistService.BlockUrlAsync(article.SourceUrl, "NoValidatedAssets", stoppingToken);
+                await dbService.DeleteArticleAsync(article.Id);
+                return;
             }
 
-            if (matchedEntities.Count == 0)
+            // 7. Prepare Matched Asset Entities
+            var matchedEntities = validatedMatches.Select(m => new MatchedAssetEntity
             {
-                foreach (var preMatch in preFilteredAssets)
-                {
-                    matchedEntities.Add(new MatchedAssetEntity
-                    {
-                        Id = Guid.NewGuid(),
-                        NewsArticleId = article.Id,
-                        Isin = preMatch.Isin,
-                        Name = preMatch.Name
-                    });
-                }
+                Id = Guid.NewGuid(),
+                NewsArticleId = article.Id,
+                Isin = m.Isin,
+                Name = m.Name
+            }).ToList();
+
+            // 8. Generate Clean Summary & Excerpt
+            var summary = scrapeResult.Excerpt;
+            if (string.IsNullOrWhiteSpace(summary))
+            {
+                summary = content.Length > 280 ? content[..280] + "..." : content;
             }
 
-            var updatedArticle = await dbService.SaveArticleClassificationAsync(article.Id, n8nResponse, matchedEntities);
+            // 9. Persist Completed Article
+            var updatedArticle = await dbService.SaveProcessedArticleAsync(
+                id: article.Id,
+                title: title,
+                author: scrapeResult.Author,
+                summary: summary,
+                contentRaw: content,
+                language: scrapeResult.Language ?? article.Language ?? "de",
+                publishedAt: publishedAt,
+                titleHash: dupCheck.TitleHash,
+                simHash: dupCheck.SimHash,
+                matchedAssets: matchedEntities
+            );
 
+            // 10. Register in deduplication memory cache
+            _deduplicationService.RegisterArticle(article.Id, dupCheck.TitleHash, dupCheck.SimHash, title, publishedAt);
+
+            // 11. Broadcast via MQTT
             if (updatedArticle != null)
             {
                 var dto = new NewsArticleDto
@@ -304,124 +322,13 @@ public class NewsScraperBackgroundService : BackgroundService
                 };
 
                 await _mqttClient.BroadcastArticleAsync(dto);
+                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Successfully processed and broadcasted article {Id} ('{Title}') with {Count} matched assets.", article.Id, title, matchedEntities.Count);
             }
         }
         catch (Exception ex)
         {
-            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to complete processing pipeline for article: {Url}. Transitioning to 'Scraping' for next cycle retry.", article.SourceUrl);
+            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Exception processing article {Url}. Flagging for next cycle retry.", article.SourceUrl);
             await dbService.UpdateArticleStatusAsync(article.Id, "Scraping");
         }
     }
-
-    private List<FilteredAssetPayload> PreFilterAssets(
-        string content,
-        string? title,
-        List<CompiledAssetMatcher> assetMatchers,
-        List<string>? priorityIsins = null)
-    {
-        if (assetMatchers.Count == 0) return new List<FilteredAssetPayload>();
-
-        var fullText = (title != null ? title + " " + content : content);
-        var matched = new Dictionary<string, FilteredAssetPayload>(StringComparer.OrdinalIgnoreCase);
-
-        if (priorityIsins != null && priorityIsins.Count > 0)
-        {
-            foreach (var isin in priorityIsins)
-            {
-                var match = assetMatchers.FirstOrDefault(m => string.Equals(m.Asset.Isin, isin, StringComparison.OrdinalIgnoreCase));
-                if (match != null && !matched.ContainsKey(match.Asset.Isin))
-                {
-                    matched[match.Asset.Isin] = new FilteredAssetPayload(match.Asset.Name, match.Asset.Isin);
-                }
-            }
-        }
-
-        foreach (var m in assetMatchers)
-        {
-            if (matched.ContainsKey(m.Asset.Isin)) continue;
-
-            if (fullText.Contains(m.Asset.Isin, StringComparison.OrdinalIgnoreCase))
-            {
-                matched[m.Asset.Isin] = new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin);
-                continue;
-            }
-
-            if (m.WordRegex != null && m.WordRegex.IsMatch(fullText))
-            {
-                matched[m.Asset.Isin] = new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin);
-                continue;
-            }
-
-            if (m.CoreWordRegex != null && m.CoreWordRegex.IsMatch(fullText))
-            {
-                matched[m.Asset.Isin] = new FilteredAssetPayload(m.Asset.Name, m.Asset.Isin);
-            }
-        }
-
-        return matched.Values.ToList();
-    }
-
-    private async Task<List<CompiledAssetMatcher>> GetOrLoadAssetMatchersAsync()
-    {
-        if (_cachedAssetMatchers != null && (DateTime.UtcNow - _lastIndexLoadTime).TotalMinutes < 60)
-        {
-            return _cachedAssetMatchers;
-        }
-
-        if (!File.Exists(_indexPath))
-        {
-            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsScraperBackgroundService] Asset index file not found at: {Path}. Pre-filtering will match 0 assets.", _indexPath);
-            return new List<CompiledAssetMatcher>();
-        }
-
-        try
-        {
-            using var stream = File.OpenRead(_indexPath);
-            var indexList = await JsonSerializer.DeserializeAsync<List<AssetIndex>>(stream);
-
-            if (indexList == null || indexList.Count == 0)
-            {
-                return new List<CompiledAssetMatcher>();
-            }
-
-            var compiled = new List<CompiledAssetMatcher>(indexList.Count);
-            foreach (var asset in indexList)
-            {
-                if (string.IsNullOrWhiteSpace(asset.Name) || string.IsNullOrWhiteSpace(asset.Isin))
-                    continue;
-
-                var rawName = asset.Name.Trim();
-                var coreName = ExtractCoreName(rawName);
-
-                Regex? wordRegex = null;
-                if (rawName.Length >= 4)
-                {
-                    wordRegex = new Regex($@"\b{Regex.Escape(rawName)}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
-                }
-
-                Regex? coreWordRegex = null;
-                if (!string.IsNullOrWhiteSpace(coreName) && coreName.Length >= 4 && !coreName.Equals(rawName, StringComparison.OrdinalIgnoreCase))
-                {
-                    coreWordRegex = new Regex($@"\b{Regex.Escape(coreName)}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
-                }
-
-                compiled.Add(new CompiledAssetMatcher(asset, coreName, wordRegex, coreWordRegex));
-            }
-
-            _cachedAssetMatchers = compiled;
-            _lastIndexLoadTime = DateTime.UtcNow;
-            return _cachedAssetMatchers;
-        }
-        catch (Exception ex)
-        {
-            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[NewsScraperBackgroundService] Failed to read or parse asset index file from {Path}.", _indexPath);
-            return _cachedAssetMatchers ?? new List<CompiledAssetMatcher>();
-        }
-    }
-
-    private static string ExtractCoreName(string rawName)
-    {
-        var cleaned = Regex.Replace(rawName, @"\b(AG|SE|SA|NV|PLC|INC|CORP|LLC|GMBH|CO|KG|HOLDING|GROUP|CLASS\s+[A-Z])\b", "", RegexOptions.IgnoreCase);
-        return cleaned.Trim(' ', '.', ',', '-');
-    }
 }

FinlyticNews/Services/PlaywrightScraperService.cs

@@ -11,16 +11,16 @@ using Microsoft.Playwright;
 namespace FinlyticNews.Services;
 
 /// <summary>
-/// Defines a headless scraping service for extracting text and resolving redirects from news sites.
+/// Defines a headless scraping service for extracting text, metadata, and resolving redirects from news sites.
 /// </summary>
 public interface IPlaywrightScraperService
 {
     /// <summary>
-    /// Scrapes the text body of an article, automatically following redirects and applying site-specific scraper adapters.
+    /// Scrapes an article, automatically following redirects and applying site-specific scraper adapters.
     /// </summary>
     /// <param name="url">The initial article URL.</param>
-    /// <returns>A tuple containing the final resolved URL and the extracted raw text content.</returns>
-    Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url);
+    /// <returns>A tuple containing the final resolved URL and the structured scrape result.</returns>
+    Task<(string ResolvedUrl, ScrapedArticleResult Result)> ScrapeArticleAsync(string url);
 }
 
 /// <inheritdoc />
@@ -45,9 +45,9 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
     }
 
     /// <inheritdoc />
-    public async Task<(string ResolvedUrl, string Content)> ScrapeArticleAsync(string url)
+    public async Task<(string ResolvedUrl, ScrapedArticleResult Result)> ScrapeArticleAsync(string url)
     {
-        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
+        await _finlyticLogger.LogInfoAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Launching browser context to scrape article: {Url}", url);
 
         var browser = await GetOrInitBrowserAsync();
         
@@ -73,8 +73,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
             }
 
             var finalUrl = page.Url;
-            await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Navigation completed. Initial final URL: {Url}", finalUrl);
-
             var host = new Uri(finalUrl).Host;
             var adapter = _scraperAdapters.FirstOrDefault(a => 
                 host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
@@ -82,14 +80,13 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
 
             if (adapter != null)
             {
-                await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Executing adapter redirect check for host: {Host}", adapter.Hostname);
                 try
                 {
                     var resolvedRedirectUrl = await adapter.TryResolveRedirectUrlAsync(page);
                     if (!string.IsNullOrWhiteSpace(resolvedRedirectUrl) && 
                         !resolvedRedirectUrl.Equals(finalUrl, StringComparison.OrdinalIgnoreCase))
                     {
-                        await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
+                        await _finlyticLogger.LogInfoAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Redirect resolved to target URL: {Url}", resolvedRedirectUrl);
                         
                         var redirectResponse = await page.GotoAsync(resolvedRedirectUrl, new PageGotoOptions
                         {
@@ -97,11 +94,6 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
                             Timeout = 30000
                         });
 
-                        if (redirectResponse == null)
-                        {
-                            await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[PlaywrightScraperService] Failed to load response for redirect URL: {Url}", resolvedRedirectUrl);
-                        }
-
                         finalUrl = page.Url;
                         host = new Uri(finalUrl).Host;
                         
@@ -112,26 +104,59 @@ public class PlaywrightScraperService : IPlaywrightScraperService, IAsyncDisposa
                 }
                 catch (Exception ex)
                 {
-                    await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter.Hostname);
+                    await _finlyticLogger.LogWarningAsync(SettingKeys.ScraperChannel, ex, "[PlaywrightScraperService] Failed to resolve redirect through adapter for host: {Host}. Continuing with current page.", adapter?.Hostname ?? host);
                 }
             }
 
-            string content;
-            if (adapter != null)
+            ScrapedArticleResult? result = null;
+            
+            for (int attempt = 1; attempt <= 2; attempt++)
             {
-                var result = await adapter.ExtractArticleContentAsync(page);
-                content = result?.TextContent ?? await FallbackExtractContentAsync(page);
-            }
-            else
-            {
-                content = await FallbackExtractContentAsync(page);
+                try
+                {
+                    if (adapter != null)
+                    {
+                        result = await adapter.ExtractArticleContentAsync(page);
+                    }
+
+                    if (result == null || string.IsNullOrWhiteSpace(result.TextContent))
+                    {
+                        var bodyText = await FallbackExtractContentAsync(page);
+                        result = new ScrapedArticleResult(
+                            Title: await page.TitleAsync(),
+                            TextContent: bodyText,
+                            HtmlContent: string.Empty,
+                            Author: null,
+                            Excerpt: null,
+                            FinalUrl: finalUrl
+                        );
+                    }
+                    
+                    break; // Extraction succeeded without execution context getting destroyed
+                }
+                catch (PlaywrightException ex) when (ex.Message.Contains("Execution context was destroyed") && attempt == 1)
+                {
+                    await _finlyticLogger.LogWarningAsync(SettingKeys.ScraperChannel, "[PlaywrightScraperService] Execution context destroyed (likely JS/Meta redirect). Waiting for new page load...");
+                    
+                    try
+                    {
+                        await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new PageWaitForLoadStateOptions { Timeout = 15000 });
+                    }
+                    catch (TimeoutException) { /* Ignored, try extracting anyway */ }
+
+                    finalUrl = page.Url;
+                    host = new Uri(finalUrl).Host;
+                    adapter = _scraperAdapters.FirstOrDefault(a => 
+                        host.EndsWith(a.Hostname, StringComparison.OrdinalIgnoreCase) ||
+                        a.Hostname.EndsWith(host, StringComparison.OrdinalIgnoreCase));
+                }
             }
 
-            return (finalUrl, content);
+            return (finalUrl, result!);
         }
         catch (Exception ex)
         {
-            await _finlyticLogger.LogErrorAsync(SettingKeys.NewsChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
+            await _finlyticLogger.LogErrorAsync(SettingKeys.ScraperChannel, ex, "[PlaywrightScraperService] Failed to scrape page content from URL: {Url}", url);
             throw;
         }
         finally

FinlyticNews/Services/SettingsDbService.cs

@@ -1,103 +0,0 @@
-using FinlyticNews.Database;
-using FinlyticNews.Entities;
-using Microsoft.EntityFrameworkCore;
-
-namespace FinlyticNews.Services;
-
-/// <summary>
-/// Service interface for retrieving and persisting runtime settings in PostgreSQL for FinlyticNews.
-/// </summary>
-public interface ISettingsDbService
-{
-    /// <summary>
-    /// Retrieves current news settings from database, initializing default values if empty.
-    /// </summary>
-    Task<NewsSettingsEntity> GetSettingsAsync();
-
-    /// <summary>
-    /// Persists updated settings to PostgreSQL.
-    /// </summary>
-    Task<NewsSettingsEntity> SaveSettingsAsync(NewsSettingsEntity settings);
-
-    /// <summary>
-    /// Updates settings from a key-value dictionary received via Admin Panel MQTT events.
-    /// </summary>
-    Task UpdateSettingsFromDictionaryAsync(Dictionary<string, string> dictionary);
-}
-
-/// <summary>
-/// EF Core implementation of <see cref="ISettingsDbService"/>.
-/// </summary>
-public class SettingsDbService : ISettingsDbService
-{
-    private readonly NewsDbContext _context;
-
-    /// <summary>
-    /// Initializes a new instance of the <see cref="SettingsDbService"/> class.
-    /// </summary>
-    /// <param name="context">The database context mapping database tables.</param>
-    public SettingsDbService(NewsDbContext context)
-    {
-        _context = context;
-    }
-
-    /// <inheritdoc />
-    public async Task<NewsSettingsEntity> GetSettingsAsync()
-    {
-        var settings = await _context.Settings.AsNoTracking().FirstOrDefaultAsync();
-        if (settings == null)
-        {
-            settings = new NewsSettingsEntity { Id = Guid.NewGuid() };
-            _context.Settings.Add(settings);
-            await _context.SaveChangesAsync();
-            _context.ChangeTracker.Clear();
-        }
-        return settings;
-    }
-
-    /// <inheritdoc />
-    public async Task<NewsSettingsEntity> SaveSettingsAsync(NewsSettingsEntity settings)
-    {
-        var existing = await _context.Settings.FirstOrDefaultAsync();
-        if (existing == null)
-        {
-            if (settings.Id == Guid.Empty) settings.Id = Guid.NewGuid();
-            _context.Settings.Add(settings);
-        }
-        else
-        {
-            existing.PollingFrequencyMinutes = settings.PollingFrequencyMinutes;
-            existing.ArticleRetentionDays = settings.ArticleRetentionDays;
-            existing.DefaultPageSize = settings.DefaultPageSize;
-            existing.ScrapingIntervalMinutes = settings.ScrapingIntervalMinutes;
-            existing.N8nWebhookUrl = settings.N8nWebhookUrl;
-            existing.UpdatedAt = settings.UpdatedAt;
-            _context.Settings.Update(existing);
-        }
-        await _context.SaveChangesAsync();
-        return settings;
-    }
-
-    /// <inheritdoc />
-    public async Task UpdateSettingsFromDictionaryAsync(Dictionary<string, string> dictionary)
-    {
-        var settings = await GetSettingsAsync();
-
-        foreach (var (key, value) in dictionary)
-        {
-            if (string.Equals(key, "PollingFrequencyMinutes", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var pfm))
-                settings.PollingFrequencyMinutes = pfm;
-            else if (string.Equals(key, "ArticleRetentionDays", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var ard))
-                settings.ArticleRetentionDays = ard;
-            else if (string.Equals(key, "DefaultPageSize", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var dps))
-                settings.DefaultPageSize = dps;
-            else if (string.Equals(key, "ScrapingIntervalMinutes", StringComparison.OrdinalIgnoreCase) && int.TryParse(value, out var sim))
-                settings.ScrapingIntervalMinutes = sim;
-            else if (string.Equals(key, "N8nWebhookUrl", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(value))
-                settings.N8nWebhookUrl = value.Trim();
-        }
-
-        settings.UpdatedAt = DateTime.UtcNow;
-        await SaveSettingsAsync(settings);
-    }
-}