Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

Browse Source
This commit is contained in:
Kleidukos
2026-08-24 21:35:33 +02:00
parent 44b161d509
commit 8112598602
400 changed files with 1812 additions and 113367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticNews/Services/NewsDbService.cs
+63 -31
View File
@@ -22,7 +22,7 @@ public interface INewsDbService
/// <summary>
/// Phase 1: Discovers and locks a new article URL by setting its status to "Pending".
/// </summary>
Task<NewsArticleEntity> CreatePendingArticleAsync(
Task<NewsArticleEntity?> CreatePendingArticleAsync(
string url,
List<string>? discoveredIsins = null,
string? title = null,
@@ -35,15 +35,30 @@ public interface INewsDbService
/// </summary>
Task UpdateArticleStatusAsync(Guid id, string status);
/// <summary>
/// Deletes an article from the database entirely (e.g. when it fails or is a duplicate).
/// </summary>
Task DeleteArticleAsync(Guid id);
/// <summary>
/// Updates the target URL of an article if a redirect is resolved during the Processing phase.
/// </summary>
Task UpdateArticleUrlAsync(Guid id, string resolvedUrl);
/// <summary>
/// Phase 4: Saves AI classification from n8n and sets the lifecycle status to "Completed".
/// Saves the processed article content, hashes, and matched assets, and transitions lifecycle status to "Completed".
/// </summary>
Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets);
Task<NewsArticleEntity?> SaveProcessedArticleAsync(
Guid id,
string title,
string? author,
string? summary,
string contentRaw,
string? language,
DateTime? publishedAt,
string titleHash,
long simHash,
List<MatchedAssetEntity> matchedAssets);
Task<List<NewsArticleEntity>> GetArticlesByStatusAsync(string status);
@@ -95,7 +110,7 @@ public class NewsDbService : INewsDbService
}
/// <inheritdoc />
public async Task<NewsArticleEntity> CreatePendingArticleAsync(
public async Task<NewsArticleEntity?> CreatePendingArticleAsync(
string url,
List<string>? discoveredIsins = null,
string? title = null,
@@ -112,7 +127,7 @@ public class NewsDbService : INewsDbService
if (existingArticle != null)
{
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Article URL already exists (Duplicate hit): {Url}", trimmedUrl);
return existingArticle;
return null;
}
var finalPublishedAt = publishedAt.HasValue
@@ -166,9 +181,9 @@ public class NewsDbService : INewsDbService
var existing = await _context.NewsArticles.AsNoTracking().FirstOrDefaultAsync(a => a.SourceUrl == trimmedUrl || a.SourceUrl == cleanUrl || a.SourceUrl.StartsWith(cleanUrl));
if (existing != null)
{
return existing;
return null;
}
return article;
return null;
}
return article;
@@ -191,6 +206,19 @@ public class NewsDbService : INewsDbService
}
}
/// <inheritdoc />
public async Task DeleteArticleAsync(Guid id)
{
var rowsAffected = await _context.NewsArticles
.Where(a => a.Id == id)
.ExecuteDeleteAsync();
if (rowsAffected > 0)
{
await _finlyticLogger.LogDebugAsync(SettingKeys.NewsChannel, "[Lifecycle] Deleted unused/failed article {Id} from database", id);
}
}
/// <inheritdoc />
public async Task UpdateArticleUrlAsync(Guid id, string resolvedUrl)
{
@@ -209,37 +237,41 @@ public class NewsDbService : INewsDbService
}
/// <inheritdoc />
public async Task<NewsArticleEntity?> SaveArticleClassificationAsync(Guid id, N8nResponsePayload payload, List<MatchedAssetEntity> matchedAssets)
public async Task<NewsArticleEntity?> SaveProcessedArticleAsync(
Guid id,
string title,
string? author,
string? summary,
string contentRaw,
string? language,
DateTime? publishedAt,
string titleHash,
long simHash,
List<MatchedAssetEntity> matchedAssets)
{
DateTime? finalPublishedAt = null;
if (DateTime.TryParse(payload.PublishedAt, out var publishedDate))
{
finalPublishedAt = publishedDate.Kind == DateTimeKind.Unspecified
? DateTime.SpecifyKind(publishedDate, DateTimeKind.Utc)
: publishedDate.ToUniversalTime();
}
DateTime? finalScrapedAt = null;
if (DateTime.TryParse(payload.ScrapedAt, out var scrapedDate))
{
finalScrapedAt = scrapedDate.ToUniversalTime();
}
var finalPublishedAt = publishedAt.HasValue
? (publishedAt.Value.Kind == DateTimeKind.Unspecified
? DateTime.SpecifyKind(publishedAt.Value, DateTimeKind.Utc)
: publishedAt.Value.ToUniversalTime())
: DateTime.UtcNow;
var rowsAffected = await _context.NewsArticles
.Where(a => a.Id == id)
.ExecuteUpdateAsync(s => s
.SetProperty(a => a.Title, payload.Title)
.SetProperty(a => a.Author, payload.Author)
.SetProperty(a => a.Summary, payload.Summary)
.SetProperty(a => a.ContentRaw, payload.ContentRaw)
.SetProperty(a => a.Language, payload.Language)
.SetProperty(a => a.Title, title)
.SetProperty(a => a.Author, author)
.SetProperty(a => a.Summary, summary)
.SetProperty(a => a.ContentRaw, contentRaw)
.SetProperty(a => a.Language, language)
.SetProperty(a => a.TitleHash, titleHash)
.SetProperty(a => a.SimHash, simHash)
.SetProperty(a => a.Status, "Completed")
.SetProperty(a => a.PublishedAt, a => finalPublishedAt ?? a.PublishedAt)
.SetProperty(a => a.ScrapedAt, a => finalScrapedAt ?? a.ScrapedAt));
.SetProperty(a => a.PublishedAt, finalPublishedAt)
.SetProperty(a => a.ScrapedAt, DateTime.UtcNow));
if (rowsAffected == 0)
{
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Article with ID {Id} not found for classification update.", id);
await _finlyticLogger.LogWarningAsync(SettingKeys.NewsChannel, "[NewsChannel] Article with ID {Id} not found for completion update.", id);
return null;
}
@@ -267,7 +299,7 @@ public class NewsDbService : INewsDbService
.AsNoTracking()
.FirstOrDefaultAsync(a => a.Id == id);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[Lifecycle] Article {Id} successfully classified and marked 'Completed'. Title: '{Title}'", id, payload.Title);
await _finlyticLogger.LogInfoAsync(SettingKeys.NewsChannel, "[Lifecycle] Article {Id} successfully processed and marked 'Completed'. Title: '{Title}'", id, title);
return completedArticle;
}
@@ -320,7 +352,7 @@ public class NewsDbService : INewsDbService
var cleanSearch = searchQuery.Trim().ToLower();
query = query.Where(a =>
a.Title.ToLower().Contains(cleanSearch) ||
a.Summary.ToLower().Contains(cleanSearch) ||
(a.Summary != null && a.Summary.ToLower().Contains(cleanSearch)) ||
a.MatchedAssets.Any(m => m.Name.ToLower().Contains(cleanSearch)));
}