feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Playwright;
|
||||
|
||||
namespace FinlyticNews.Adapters.Scraping;
|
||||
|
||||
/// <summary>
|
||||
/// Specialized article scraper adapter for IT-Times.de.
|
||||
/// </summary>
|
||||
public class ItTimesScraperAdapter : ArticleScraperAdapter
|
||||
{
|
||||
public override string Hostname => "it-times.de";
|
||||
|
||||
public override string ArticleBodySelector => "div.story_body, article.article, div.article_content";
|
||||
|
||||
public override async Task<ScrapedArticleResult?> ExtractArticleContentAsync(IPage page)
|
||||
{
|
||||
await page.EvaluateAsync(@"() => {
|
||||
document.querySelectorAll('.social_share, .related_stories, .banner, .ad').forEach(e => e.remove());
|
||||
}");
|
||||
|
||||
return await base.ExtractArticleContentAsync(page);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user