feat(news): add scraper adapters, article deduplication, blocklist service, and remove tracked publish artifacts

2026-08-24 21:35:33 +02:00
parent 44b161d509
commit 8112598602
400 changed files with 1812 additions and 113367 deletions
@@ -0,0 +1,167 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace FinlyticNews.Migrations
+{
+    /// <inheritdoc />
+    public partial class Init : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "ArticleSources",
+                columns: table => new
+                {
+                    Id = table.Column<Guid>(type: "uuid", nullable: false),
+                    Source = table.Column<string>(type: "text", nullable: false),
+                    Name = table.Column<string>(type: "text", nullable: false),
+                    Type = table.Column<string>(type: "text", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_ArticleSources", x => x.Id);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "BlockedNewsUrls",
+                columns: table => new
+                {
+                    Id = table.Column<Guid>(type: "uuid", nullable: false),
+                    Url = table.Column<string>(type: "character varying(2048)", maxLength: 2048, nullable: false),
+                    Reason = table.Column<string>(type: "character varying(256)", maxLength: 256, nullable: false),
+                    BlockedAtUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_BlockedNewsUrls", x => x.Id);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "DynamicSettings",
+                columns: table => new
+                {
+                    Id = table.Column<Guid>(type: "uuid", nullable: false),
+                    Key = table.Column<string>(type: "character varying(150)", maxLength: 150, nullable: false),
+                    ValueJson = table.Column<string>(type: "text", nullable: false),
+                    ServiceIdentifier = table.Column<string>(type: "character varying(100)", maxLength: 100, nullable: false),
+                    LastUpdatedUtc = table.Column<DateTime>(type: "timestamp with time zone", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_DynamicSettings", x => x.Id);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "NewsArticles",
+                columns: table => new
+                {
+                    Id = table.Column<Guid>(type: "uuid", nullable: false),
+                    Title = table.Column<string>(type: "text", nullable: false),
+                    Author = table.Column<string>(type: "text", nullable: true),
+                    Summary = table.Column<string>(type: "text", nullable: true),
+                    ContentRaw = table.Column<string>(type: "text", nullable: false),
+                    Language = table.Column<string>(type: "text", nullable: true),
+                    SourceUrl = table.Column<string>(type: "text", nullable: false),
+                    ScrapedAt = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
+                    PublishedAt = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
+                    Status = table.Column<string>(type: "text", nullable: false),
+                    TitleHash = table.Column<string>(type: "character varying(64)", maxLength: 64, nullable: true),
+                    SimHash = table.Column<long>(type: "bigint", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_NewsArticles", x => x.Id);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "MatchedAssets",
+                columns: table => new
+                {
+                    Id = table.Column<Guid>(type: "uuid", nullable: false),
+                    NewsArticleId = table.Column<Guid>(type: "uuid", nullable: false),
+                    Isin = table.Column<string>(type: "text", nullable: false),
+                    Name = table.Column<string>(type: "text", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_MatchedAssets", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_MatchedAssets_NewsArticles_NewsArticleId",
+                        column: x => x.NewsArticleId,
+                        principalTable: "NewsArticles",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_BlockedNewsUrls_Url",
+                table: "BlockedNewsUrls",
+                column: "Url",
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_DynamicSettings_Key",
+                table: "DynamicSettings",
+                column: "Key",
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_MatchedAssets_Isin",
+                table: "MatchedAssets",
+                column: "Isin");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_MatchedAssets_NewsArticleId",
+                table: "MatchedAssets",
+                column: "NewsArticleId");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_NewsArticles_PublishedAt_ScrapedAt",
+                table: "NewsArticles",
+                columns: new[] { "PublishedAt", "ScrapedAt" });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_NewsArticles_SimHash",
+                table: "NewsArticles",
+                column: "SimHash");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_NewsArticles_SourceUrl",
+                table: "NewsArticles",
+                column: "SourceUrl",
+                unique: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_NewsArticles_Status",
+                table: "NewsArticles",
+                column: "Status");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_NewsArticles_TitleHash",
+                table: "NewsArticles",
+                column: "TitleHash");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "ArticleSources");
+
+            migrationBuilder.DropTable(
+                name: "BlockedNewsUrls");
+
+            migrationBuilder.DropTable(
+                name: "DynamicSettings");
+
+            migrationBuilder.DropTable(
+                name: "MatchedAssets");
+
+            migrationBuilder.DropTable(
+                name: "NewsArticles");
+        }
+    }
+}