Kleidukos/Finlytic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

Browse Source
This commit is contained in:
Kleidukos
2026-08-24 21:35:33 +02:00
parent 44b161d509
commit 8112598602
400 changed files with 1812 additions and 113367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
FinlyticNews/Migrations/20260801073336_Init.Designer.cs
Generated
-174
View File
@@ -1,174 +0,0 @@
// <auto-generated />
using System;
using FinlyticNews.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticNews.Migrations
{
[DbContext(typeof(NewsDbContext))]
[Migration("20260801073336_Init")]
partial class Init
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticNews.Entities.ArticleSourceEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("ArticleSources");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Isin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("NewsArticleId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("Isin");
b.HasIndex("NewsArticleId");
b.ToTable("MatchedAssets");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsArticleEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Author")
.HasColumnType("text");
b.Property<string>("ContentRaw")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Language")
.HasColumnType("text");
b.Property<DateTime>("PublishedAt")
.HasColumnType("timestamp with time zone");
b.Property<DateTime>("ScrapedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("SourceUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Status")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Summary")
.HasColumnType("text");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("SourceUrl")
.IsUnique();
b.HasIndex("Status");
b.HasIndex("PublishedAt", "ScrapedAt");
b.ToTable("NewsArticles");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("ArticleRetentionDays")
.HasColumnType("integer");
b.Property<int>("DefaultPageSize")
.HasColumnType("integer");
b.Property<string>("N8nWebhookUrl")
.IsRequired()
.HasColumnType("text");
b.Property<int>("PollingFrequencyMinutes")
.HasColumnType("integer");
b.Property<int>("ScrapingIntervalMinutes")
.HasColumnType("integer");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.HasOne("FinlyticNews.Entities.NewsArticleEntity", "NewsArticle")
.WithMany("MatchedAssets")
.HasForeignKey("NewsArticleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("NewsArticle");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsArticleEntity", b =>
{
b.Navigation("MatchedAssets");
});
#pragma warning restore 612, 618
}
}
}
FinlyticNews/Migrations/20260813202646_CheckPendingNews.Designer.cs
Generated
-174
View File
@@ -1,174 +0,0 @@
// <auto-generated />
using System;
using FinlyticNews.Database;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable
namespace FinlyticNews.Migrations
{
[DbContext(typeof(NewsDbContext))]
[Migration("20260813202646_CheckPendingNews")]
partial class CheckPendingNews
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "10.0.9")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("FinlyticNews.Entities.ArticleSourceEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Source")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.ToTable("ArticleSources");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Isin")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<Guid>("NewsArticleId")
.HasColumnType("uuid");
b.HasKey("Id");
b.HasIndex("Isin");
b.HasIndex("NewsArticleId");
b.ToTable("MatchedAssets");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsArticleEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<string>("Author")
.HasColumnType("text");
b.Property<string>("ContentRaw")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Language")
.HasColumnType("text");
b.Property<DateTime>("PublishedAt")
.HasColumnType("timestamp with time zone");
b.Property<DateTime>("ScrapedAt")
.HasColumnType("timestamp with time zone");
b.Property<string>("SourceUrl")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Status")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Summary")
.HasColumnType("text");
b.Property<string>("Title")
.IsRequired()
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("SourceUrl")
.IsUnique();
b.HasIndex("Status");
b.HasIndex("PublishedAt", "ScrapedAt");
b.ToTable("NewsArticles");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("ArticleRetentionDays")
.HasColumnType("integer");
b.Property<int>("DefaultPageSize")
.HasColumnType("integer");
b.Property<string>("N8nWebhookUrl")
.IsRequired()
.HasColumnType("text");
b.Property<int>("PollingFrequencyMinutes")
.HasColumnType("integer");
b.Property<int>("ScrapingIntervalMinutes")
.HasColumnType("integer");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.HasOne("FinlyticNews.Entities.NewsArticleEntity", "NewsArticle")
.WithMany("MatchedAssets")
.HasForeignKey("NewsArticleId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("NewsArticle");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsArticleEntity", b =>
{
b.Navigation("MatchedAssets");
});
#pragma warning restore 612, 618
}
}
}
FinlyticNews/Migrations/20260813202646_CheckPendingNews.cs
-22
View File
@@ -1,22 +0,0 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticNews.Migrations
{
/// <inheritdoc />
public partial class CheckPendingNews : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
}
}
}
FinlyticNews/Migrations/20260815183946_AddDynamicSettings.cs
-43
View File
@@ -1,43 +0,0 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace FinlyticNews.Migrations
{
/// <inheritdoc />
public partial class AddDynamicSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DynamicSettings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
ValueJson = table.Column<string>(type: "text", nullable: false),
ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DynamicSettings", x => x.Id);
});
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DynamicSettings");
}
}
}
FinlyticNews/Migrations/20260815183946_AddDynamicSettings.Designer.cs → FinlyticNews/Migrations/20260818194224_Init.Designer.cs
Generated
+40 -32
View File
@@ -12,8 +12,8 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace FinlyticNews.Migrations
{
[DbContext(typeof(NewsDbContext))]
[Migration("20260815183946_AddDynamicSettings")]
partial class AddDynamicSettings
[Migration("20260818194224_Init")]
partial class Init
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
@@ -79,6 +79,33 @@ namespace FinlyticNews.Migrations
b.ToTable("ArticleSources");
});
modelBuilder.Entity("FinlyticNews.Entities.BlockedNewsUrlEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<DateTime>("BlockedAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("Reason")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b.HasKey("Id");
b.HasIndex("Url")
.IsUnique();
b.ToTable("BlockedNewsUrls");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.Property<Guid>("Id")
@@ -127,6 +154,9 @@ namespace FinlyticNews.Migrations
b.Property<DateTime>("ScrapedAt")
.HasColumnType("timestamp with time zone");
b.Property<long?>("SimHash")
.HasColumnType("bigint");
b.Property<string>("SourceUrl")
.IsRequired()
.HasColumnType("text");
@@ -142,48 +172,26 @@ namespace FinlyticNews.Migrations
.IsRequired()
.HasColumnType("text");
b.Property<string>("TitleHash")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasKey("Id");
b.HasIndex("SimHash");
b.HasIndex("SourceUrl")
.IsUnique();
b.HasIndex("Status");
b.HasIndex("TitleHash");
b.HasIndex("PublishedAt", "ScrapedAt");
b.ToTable("NewsArticles");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("ArticleRetentionDays")
.HasColumnType("integer");
b.Property<int>("DefaultPageSize")
.HasColumnType("integer");
b.Property<string>("N8nWebhookUrl")
.IsRequired()
.HasColumnType("text");
b.Property<int>("PollingFrequencyMinutes")
.HasColumnType("integer");
b.Property<int>("ScrapingIntervalMinutes")
.HasColumnType("integer");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.HasOne("FinlyticNews.Entities.NewsArticleEntity", "NewsArticle")
FinlyticNews/Migrations/20260801073336_Init.cs → FinlyticNews/Migrations/20260818194224_Init.cs
+59 -20
View File
@@ -25,6 +25,35 @@ namespace FinlyticNews.Migrations
table.PrimaryKey("PK_ArticleSources", x => x.Id);
});
migrationBuilder.CreateTable(
name: "BlockedNewsUrls",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Url = table.Column<string>(type: "character varying(2048)", maxLength: 2048, nullable: false),
Reason = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: false),
BlockedAtUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_BlockedNewsUrls", x => x.Id);
});
migrationBuilder.CreateTable(
name: "DynamicSettings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
ValueJson = table.Column<string>(type: "text", nullable: false),
ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DynamicSettings", x => x.Id);
});
migrationBuilder.CreateTable(
name: "NewsArticles",
columns: table => new
@@ -38,30 +67,15 @@ namespace FinlyticNews.Migrations
SourceUrl = table.Column<string>(type: "text", nullable: false),
ScrapedAt = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
PublishedAt = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
Status = table.Column<string>(type: "text", nullable: false)
Status = table.Column<string>(type: "text", nullable: false),
TitleHash = table.Column<string>(type: "character varying(64)", maxLength: 64, nullable: true),
SimHash = table.Column<long>(type: "bigint", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_NewsArticles", x => x.Id);
});
migrationBuilder.CreateTable(
name: "Settings",
columns: table => new
{
Id = table.Column<Guid>(type: "uuid", nullable: false),
PollingFrequencyMinutes = table.Column<int>(type: "integer", nullable: false),
ArticleRetentionDays = table.Column<int>(type: "integer", nullable: false),
DefaultPageSize = table.Column<int>(type: "integer", nullable: false),
ScrapingIntervalMinutes = table.Column<int>(type: "integer", nullable: false),
N8nWebhookUrl = table.Column<string>(type: "text", nullable: false),
UpdatedAt = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_Settings", x => x.Id);
});
migrationBuilder.CreateTable(
name: "MatchedAssets",
columns: table => new
@@ -82,6 +96,18 @@ namespace FinlyticNews.Migrations
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "IX_BlockedNewsUrls_Url",
table: "BlockedNewsUrls",
column: "Url",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_DynamicSettings_Key",
table: "DynamicSettings",
column: "Key",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_MatchedAssets_Isin",
table: "MatchedAssets",
@@ -97,6 +123,11 @@ namespace FinlyticNews.Migrations
table: "NewsArticles",
columns: new[] { "PublishedAt", "ScrapedAt" });
migrationBuilder.CreateIndex(
name: "IX_NewsArticles_SimHash",
table: "NewsArticles",
column: "SimHash");
migrationBuilder.CreateIndex(
name: "IX_NewsArticles_SourceUrl",
table: "NewsArticles",
@@ -107,6 +138,11 @@ namespace FinlyticNews.Migrations
name: "IX_NewsArticles_Status",
table: "NewsArticles",
column: "Status");
migrationBuilder.CreateIndex(
name: "IX_NewsArticles_TitleHash",
table: "NewsArticles",
column: "TitleHash");
}
/// <inheritdoc />
@@ -116,10 +152,13 @@ namespace FinlyticNews.Migrations
name: "ArticleSources");
migrationBuilder.DropTable(
name: "MatchedAssets");
name: "BlockedNewsUrls");
migrationBuilder.DropTable(
name: "Settings");
name: "DynamicSettings");
migrationBuilder.DropTable(
name: "MatchedAssets");
migrationBuilder.DropTable(
name: "NewsArticles");
FinlyticNews/Migrations/NewsDbContextModelSnapshot.cs
+38 -30
View File
@@ -76,6 +76,33 @@ namespace FinlyticNews.Migrations
b.ToTable("ArticleSources");
});
modelBuilder.Entity("FinlyticNews.Entities.BlockedNewsUrlEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<DateTime>("BlockedAtUtc")
.HasColumnType("timestamp with time zone");
b.Property<string>("Reason")
.IsRequired()
.HasMaxLength(256)
.HasColumnType("character varying(256)");
b.Property<string>("Url")
.IsRequired()
.HasMaxLength(2048)
.HasColumnType("character varying(2048)");
b.HasKey("Id");
b.HasIndex("Url")
.IsUnique();
b.ToTable("BlockedNewsUrls");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.Property<Guid>("Id")
@@ -124,6 +151,9 @@ namespace FinlyticNews.Migrations
b.Property<DateTime>("ScrapedAt")
.HasColumnType("timestamp with time zone");
b.Property<long?>("SimHash")
.HasColumnType("bigint");
b.Property<string>("SourceUrl")
.IsRequired()
.HasColumnType("text");
@@ -139,48 +169,26 @@ namespace FinlyticNews.Migrations
.IsRequired()
.HasColumnType("text");
b.Property<string>("TitleHash")
.HasMaxLength(64)
.HasColumnType("character varying(64)");
b.HasKey("Id");
b.HasIndex("SimHash");
b.HasIndex("SourceUrl")
.IsUnique();
b.HasIndex("Status");
b.HasIndex("TitleHash");
b.HasIndex("PublishedAt", "ScrapedAt");
b.ToTable("NewsArticles");
});
modelBuilder.Entity("FinlyticNews.Entities.NewsSettingsEntity", b =>
{
b.Property<Guid>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("uuid");
b.Property<int>("ArticleRetentionDays")
.HasColumnType("integer");
b.Property<int>("DefaultPageSize")
.HasColumnType("integer");
b.Property<string>("N8nWebhookUrl")
.IsRequired()
.HasColumnType("text");
b.Property<int>("PollingFrequencyMinutes")
.HasColumnType("integer");
b.Property<int>("ScrapingIntervalMinutes")
.HasColumnType("integer");
b.Property<DateTime>("UpdatedAt")
.HasColumnType("timestamp with time zone");
b.HasKey("Id");
b.ToTable("Settings");
});
modelBuilder.Entity("FinlyticNews.Entities.MatchedAssetEntity", b =>
{
b.HasOne("FinlyticNews.Entities.NewsArticleEntity", "NewsArticle")