diff --git a/Finlytic.sln b/Finlytic.sln index 4b4788a..1f50a64 100644 --- a/Finlytic.sln +++ b/Finlytic.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticAssets", "FinlyticAssets\FinlyticAssets.csproj", "{B5AC2C29-44D5-4538-815F-F02CDDE9D01F}" EndProject @@ -11,23 +11,116 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticCore", "FinlyticCor EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticNews", "FinlyticNews\FinlyticNews.csproj", "{03B4D920-6173-44E8-A1E2-8945D8393CEA}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticFundamentals", "FinlyticFundamentals\FinlyticFundamentals.csproj", "{D458A1B3-16CF-45E8-859D-87542A4A83A7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticSentiment", "FinlyticSentiment\FinlyticSentiment.csproj", "{9C3BB705-86AD-4A89-AA0F-A52C87A4950B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticTechnicalAnalysis", "FinlyticTechnicalAnalysis\FinlyticTechnicalAnalysis.csproj", "{A1C82F63-4482-4E99-9231-1184FA2E001F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticAnalyzer", "FinlyticAnalyzer\FinlyticAnalyzer.csproj", "{E9F7C091-62C4-417A-B981-8977DF82A1B0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticTrades", "FinlyticTrades\FinlyticTrades.csproj", "{57D84C2E-73E1-4231-A91B-6B620FCE5289}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FinlyticBackend", "FinlyticBackend\FinlyticBackend.csproj", "{C1A924B8-904E-436D-B07E-4E621F51C1AA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|x64.ActiveCfg = Debug|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|x64.Build.0 = Debug|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|x86.ActiveCfg = Debug|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Debug|x86.Build.0 = Debug|Any CPU {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|Any CPU.ActiveCfg = Release|Any CPU {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|Any CPU.Build.0 = Release|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|x64.ActiveCfg = Release|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|x64.Build.0 = Release|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|x86.ActiveCfg = Release|Any CPU + {B5AC2C29-44D5-4538-815F-F02CDDE9D01F}.Release|x86.Build.0 = Release|Any CPU {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|x64.ActiveCfg = Debug|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|x64.Build.0 = Debug|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|x86.ActiveCfg = Debug|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Debug|x86.Build.0 = Debug|Any CPU {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|Any CPU.ActiveCfg = Release|Any CPU {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|Any CPU.Build.0 = Release|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|x64.ActiveCfg = Release|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|x64.Build.0 = Release|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|x86.ActiveCfg = Release|Any CPU + {03E2ECBB-0FF9-43E4-994E-F6A522860AD5}.Release|x86.Build.0 = Release|Any CPU {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|x64.ActiveCfg = Debug|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|x64.Build.0 = Debug|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|x86.ActiveCfg = Debug|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Debug|x86.Build.0 = Debug|Any CPU {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|Any CPU.ActiveCfg = Release|Any CPU {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|Any CPU.Build.0 = Release|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|x64.ActiveCfg = Release|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|x64.Build.0 = Release|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|x86.ActiveCfg = Release|Any CPU + {03B4D920-6173-44E8-A1E2-8945D8393CEA}.Release|x86.Build.0 = Release|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Debug|x64.ActiveCfg = Debug|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Debug|x64.Build.0 = Debug|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Debug|x86.ActiveCfg = Debug|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Debug|x86.Build.0 = Debug|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Release|Any CPU.Build.0 = Release|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Release|x64.ActiveCfg = Release|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Release|x64.Build.0 = Release|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Release|x86.ActiveCfg = Release|Any CPU + {999DB199-69C5-46D2-BDAD-59C84F5769F1}.Release|x86.Build.0 = Release|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Debug|x64.ActiveCfg = Debug|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Debug|x64.Build.0 = Debug|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Debug|x86.ActiveCfg = Debug|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Debug|x86.Build.0 = Debug|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Release|Any CPU.Build.0 = Release|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Release|x64.ActiveCfg = Release|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Release|x64.Build.0 = Release|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Release|x86.ActiveCfg = Release|Any CPU + {D458A1B3-16CF-45E8-859D-87542A4A83A7}.Release|x86.Build.0 = Release|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Debug|x64.ActiveCfg = Debug|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Debug|x64.Build.0 = Debug|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Debug|x86.ActiveCfg = Debug|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Debug|x86.Build.0 = Debug|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Release|Any CPU.Build.0 = Release|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Release|x64.ActiveCfg = Release|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Release|x64.Build.0 = Release|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Release|x86.ActiveCfg = Release|Any CPU + {9C3BB705-86AD-4A89-AA0F-A52C87A4950B}.Release|x86.Build.0 = Release|Any CPU + {A1C82F63-4482-4E99-9231-1184FA2E001F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A1C82F63-4482-4E99-9231-1184FA2E001F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E9F7C091-62C4-417A-B981-8977DF82A1B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E9F7C091-62C4-417A-B981-8977DF82A1B0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E9F7C091-62C4-417A-B981-8977DF82A1B0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E9F7C091-62C4-417A-B981-8977DF82A1B0}.Release|Any CPU.Build.0 = Release|Any CPU + {57D84C2E-73E1-4231-A91B-6B620FCE5289}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {57D84C2E-73E1-4231-A91B-6B620FCE5289}.Debug|Any CPU.Build.0 = Debug|Any CPU + {57D84C2E-73E1-4231-A91B-6B620FCE5289}.Release|Any CPU.ActiveCfg = Release|Any CPU + {57D84C2E-73E1-4231-A91B-6B620FCE5289}.Release|Any CPU.Build.0 = Release|Any CPU + {C1A924B8-904E-436D-B07E-4E621F51C1AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C1A924B8-904E-436D-B07E-4E621F51C1AA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C1A924B8-904E-436D-B07E-4E621F51C1AA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C1A924B8-904E-436D-B07E-4E621F51C1AA}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal diff --git a/Project.md b/Project.md index 0a3802a..c34760e 100644 --- a/Project.md +++ b/Project.md @@ -1,66 +1,78 @@ -# Finlytic Assets Service +# Finlytic Enterprise System Architecture -Finlytic Assets is a scalable C# microservice designed for asset discovery, metadata ingestion, and quick querying. It interfaces directly with the Trade Republic API via WebSockets and exposes a high-performance RPC interface over MQTT to other services in the Finlytic ecosystem. +Finlytic is an enterprise financial intelligence platform composed of high-performance C# .NET 8 microservices, a web gateway (`FinlyticBackend`), a Flutter application (`FinlyticApp`), a React web interface (`FinlyticWeb`), and a real-time MQTT event mesh. --- -## Architecture Overview +## Ecosystem Architecture Overview ```mermaid graph TD - TR[Trade Republic WebSocket API] <-->|WS Protocol| TRS[TradeRepublicService] - TRS <-->|Ingest| ADS[AssetsFullScanService] - ADS <-->|Save / Update| DB[PostgreSQL Database] - DB -->|Trigger Index Update| AIS[AssetsIndexService] - AIS -->|Write Cache| Index[assets/index/index.json] - - MS[Other Finlytic Services] <-->|MQTT Request-Reply| MqttClient[AssetsMqttClient] - MqttClient <-->|Query Cache| DB + App[FinlyticApp (Flutter)] -->|HTTP REST & SignalR| Backend[FinlyticBackend] + Web[FinlyticWeb (React)] -->|HTTP REST & SignalR| Backend + + Backend <-->|MQTT Pub/Sub & RPC| Broker[MQTT Broker (EMQX / Mosquitto)] + + News[FinlyticNews Service] <-->|MQTT| Broker + Sentiment[FinlyticSentiment Service] <-->|MQTT| Broker + Assets[FinlyticAssets Service] <-->|MQTT| Broker + Fundamentals[FinlyticFundamentals Service] <-->|MQTT| Broker + TA[FinlyticTechnicalAnalysis Service] <-->|MQTT| Broker + Trades[FinlyticTrades Service] <-->|MQTT| Broker + Analyzer[FinlyticAnalyzer Service] <-->|MQTT| Broker + + TR[Trade Republic WS API] <--> Assets + N8N[n8n Webhook / FinBERT] <--> News + N8N <--> Sentiment ``` --- -## Core Features & Workflows +## Core System Principles & Rules -### 1. Automated WebSocket scraping -- **Continuous Scan**: The background worker (`AssetsFullScanService`) iterates through all support asset types: - - **Stocks** (`StockEntity`) - - **Crypto** (`CryptoEntity`) - - **Derivatives** (`DerivativeEntity`) - - **Bonds** (`BondEntity`) - - **Funds/ETFs** (`EtfEntity`) -- **Resilient Reconnection**: Uses a robust custom socket wrapper (`TradeRepublicClient`) that closes automatically after 5 minutes of inactivity to mimic human interaction profiles and reconnects dynamically when a request is made. +1. **Single Web Gateway (`FinlyticBackend`)**: + - `FinlyticBackend` is the **only** microservice hosting HTTP REST and SignalR WebSocket endpoints for external clients (`FinlyticApp`, `FinlyticWeb`). + - All background worker microservices (`FinlyticNews`, `FinlyticSentiment`, `FinlyticAssets`, `FinlyticFundamentals`, `FinlyticTechnicalAnalysis`, `FinlyticTrades`, `FinlyticAnalyzer`) operate strictly as `IHostedService` worker engines with zero Kestrel HTTP webservers. -### 2. Stealth Scheduling with Randomized Jitter -- **Dynamic Delays**: Scanner wait times are read dynamically from database configuration (`Settings`). -- **Jitter Offset**: High/low jitter values (randomized offsets between batches and asset types) are injected to avoid predictable traffic patterns and prevent Trade Republic rate limiting. +2. **Exclusive Inter-Service Communication via MQTT**: + - All background microservices communicate strictly over MQTT topics (Pub/Sub & RPC). + - High-performance, zero-allocation serialization is enforced using `.NET 8 JSON Source Generators` (`FinlyticJsonSerializerContext`). -### 3. Active vs. Dead Asset Tracking -- **Recency Filter**: Assets are continuously updated with their latest metadata. If an asset has not been updated within **14 days**, it is considered inactive, de-listed, or "dead". -- **Implicit Filtration**: Queries by other services only return active/valid records unless explicitly requested otherwise. - -### 4. MQTT RPC Interface -- **Get Asset by ISIN**: - - Subscribes to: `services/request/assets_Get/#` - - Returns a list of all active assets matching the specified ISIN across different instrument types (e.g., matching both a stock and its derivative tracker). -- **Omnibox Search**: - - Subscribes to: `services/request/assets_Search/#` - - Evaluates search keywords against local records (matching `ISIN`, `Name`, or tags like `Region`, `Country`, `Sector`, etc.). - - **JIT Fallback**: If a search query is formatted as an unknown valid ISIN, it performs a JIT-lookup directly against the Trade Republic API and registers the discovered asset before replying. - -### 5. Local Indexing File (`index.json`) -- **Automatic Regeneration**: On database modifications, `AssetsIndexService` is triggered to regenerate `assets/index/index.json`. -- **Pre-Filtering**: This lean index (containing only `ISIN` and `Name` properties) is written to a shared volume to allow other services to perform instant pre-filtering without querying the database or sending network requests. +3. **Absolute Prohibition of Mock/Demo Data**: + - No mock data, hardcoded fallback arrays, or fake dummy responses are permitted in any microservice or frontend client. + - Either real data is queried from database contexts (PostgreSQL) / external APIs, or empty result sets / explicit exceptions are returned. --- -## Technical Components +## Microservices Breakdown -| Component | Class / Interface | Responsibility | +| Project | Type | Description | | :--- | :--- | :--- | -| **Ingestion Worker** | `AssetsFullScanService` | Runs the main cron loop, scraping and paginating all Trade Republic asset types. | -| **Database Store** | [IAssetsDbService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/AssetsDbService.cs) | Handles CRUD operations, stateful merges, JIT lookups, and active record filtering. | -| **WebSocket Handler** | [ITradeRepublicService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/TradeRepublicService.cs) | Wraps the connection and request dispatching for the Trade Republic API. | -| **Local Indexer** | [IAssetsIndexService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/AssetsIndexService.cs) | Serializes the active assets list to `index.json` for external service indexing. | -| **Messaging Broker** | `AssetsMqttClient` | Listens for incoming MQTT requests, queries the database, and responds to RPC channels. | -| **Config Service** | [ISettingsDbService](file:///e:/Projects/FinlyticAssets/FinlyticAssets/Services/SettingsDbService.cs) | Manages scanner configurations (scan intervals, active pointers, offsets). | +| **`FinlyticCore`** | Class Library | Shared DTOs, domain models, MQTT infrastructure (`ManagedMqttClient`), and JSON Source Generator context. | +| **`FinlyticNews`** | Worker Service | Scraping (Playwright/RSS), deduplication, n8n AI enrichment, and news state persistence. | +| **`FinlyticSentiment`** | Worker Service | FinBERT AI sentiment evaluation, ISIN/Sector sentiment aggregation over MQTT. | +| **`FinlyticAssets`** | Worker Service | Trade Republic WebSocket full-scan ingestion, asset metadata indexing (`index.json`), and ISIN JIT lookup. | +| **`FinlyticFundamentals`** | Worker Service | Financial fundamentals scraping, SEC/Financial Modeling Prep integration, and corporate calendar events. | +| **`FinlyticTechnicalAnalysis`** | Worker Service | Real-time technical indicators (RSI, MACD, EMA, Supertrend) and chart pattern detection. | +| **`FinlyticTrades`** | Worker Service | Trade lifecycle management (Active, Closed, TTL worker, Feedback exporter). | +| **`FinlyticAnalyzer`** | Worker Service | 3-layer filter engine, VIX regime tracking, win-rate calculator, and trade signal generation. | +| **`FinlyticBackend`** | Web API / Gateway | ASP.NET Core REST API, JWT authentication, SignalR Hubs (`NewsHub`, `TradeHub`), and MQTT bridge. | +| **`FinlyticWeb`** | Web Application | React/Next.js dashboard web application. | +| **`FinlyticApp`** | Mobile/Cross-Platform App | Flutter application built with Clean Architecture (`models/`, `repositories/`, `bloc/`). | + +--- + +## Status of Implemented & Planned Features + +### Implemented Features +- [x] Zero-Allocation MQTT RPC & Event Mesh across all .NET 8 microservices. +- [x] Removal of Kestrel HTTP servers from all background worker services (`FinlyticAnalyzer`, `FinlyticTrades`, etc.). +- [x] Complete removal of all mock/demo fallbacks in backend and frontend. +- [x] PostgreSQL database indexes on `PublishedAt`, `Status`, `SourceUrl`, `Isin`, `CreatedAt`. +- [x] Clean Architecture migration across all 7 modules in `FinlyticApp` (Trades, Assets, Favorites, Auth, Admin, Calendar, Search). +- [x] SignalR Real-Time Hubs (`NewsHub`, `TradeHub`) with MQTT-to-SignalR broadcasting. + +### Planned Features +- [ ] Automated backtesting engine for multi-year strategy evaluation in `FinlyticAnalyzer`. +- [ ] Order Execution Integration (automated broker API order routing). +- [ ] Push Notifications for iOS/Android via Firebase Cloud Messaging in production deployment. diff --git a/Rules.md b/Rules.md index 71d02dd..86d6d33 100644 --- a/Rules.md +++ b/Rules.md @@ -1,6 +1,6 @@ # Development Rules & Guidelines -To ensure scalability, readability, and consistency across all microservices, the following architectural and coding rules must be adhered to: +To ensure scalability, readability, and consistency across all microservices and frontend clients, the following architectural and coding rules must be adhered to: ## 1. Service Interfaces and Implementations - **Single File Co-location**: Every service interface (e.g., `IAssetsDbService`) and its corresponding implementation class (e.g., `AssetsDbService`) **must** reside in the same file. @@ -16,5 +16,60 @@ To ensure scalability, readability, and consistency across all microservices, th - **Implementation Inheritdoc**: For class methods implementing interfaces or overriding base methods, use `/// ` to inherit documentation unless customization is needed. Note that any custom helper or private methods in the implementation must still have their own explicit XML comments. ## 3. Data Class & Model Architecture +- **Strict Data Class Usage**: ALL data processing, state passing, API payloads, MQTT message schemas, and internal data transfers **must strictly use strongly-typed Data Classes** (e.g., C# `record`, `class`, or immutable DTOs / Dart Data Classes). +- **Prohibition of Loose Types**: The use of untyped data containers such as `Dictionary`, `dynamic`, raw `JObject`/`JsonDocument`, or unstructured string-based JSON passing within internal logic is strictly forbidden. - **Core Placement**: All data classes, DTOs, and shared enums (e.g., `AssetType`) that are or could be used by multiple services must be defined in the `FinlyticCore` project. - **Service-Specific Exception**: Data classes and models may only reside in a specific service project (e.g., `FinlyticAssets`) if they strictly concern the internal operations of that service (e.g., raw API request/response structures for an integration that only that service manages). + +## 4. Absolute Prohibition of Demo Data & Mandatory Empty-State Transparency +- **No Mocks or Fallbacks**: Mock data, demo fallbacks, hardcoded fake arrays, placeholder graphs, or dummy fallback responses are strictly forbidden across all backend microservices and frontend clients. +- **Strict Data Reality**: Either real results are available from database queries/API responses or the system must return empty result sets / raise explicit exceptions. Silent mock fallbacks are prohibited. +- **Explicit UI Zero-Data Handling**: In the frontend (Flutter/Dart), if an API call returns no data, **never show placeholder content or fake metrics**. The UI must explicitly inform the user via an unambiguous Empty State view (e.g., *"No active recommendations available"* or *"No trades found for this period"*). The user must always clearly know when data is missing versus when real data is present. + +## 5. Inter-Service Communication via MQTT Only +- **Exclusive Protocol**: All internal communication between backend microservices **must** run exclusively over MQTT (Pub/Sub & RPC). +- **Single Web Gateway**: `FinlyticBackend` is the **only** microservice allowed to host Kestrel HTTP/WebSocket endpoints for external client access (`FinlyticApp`). Background microservices must never expose HTTP webservers or REST controllers. + +## 6. EF Core Migrations Management +- **Command-Only Creation**: Database migrations (EF Core Migrations) must strictly be created via CLI commands (e.g., `dotnet ef migrations add `). Manual hand-crafted creation of migration files is strictly prohibited. + +## 7. Security & API Authorization (Backend Gateway) +- **Mandatory Route Authorization**: Every single HTTP controller endpoint or WebSocket route in `FinlyticBackend` **must** be protected with `[Authorize]` attributes (or explicitly scoped authorization policies). +- **Explicit Whitelisting Only**: Endpoints without authentication (e.g., `/api/v1/auth/login`) are the *only* allowed exception and must be explicitly marked with `[AllowAnonymous]`. Unprotected endpoints without explicit anonymous authorization are forbidden. + +## 8. Dart / Flutter Frontend Architecture & JWT Handling +- **Mandatory JWT Injection**: Every outgoing HTTP/WebSocket request from the Flutter app (`FinlyticApp`) **must** include the JWT Bearer Token in the `Authorization` header (`Bearer `). This must be handled centrally via HTTP Interceptors (e.g., `Dio` interceptor or custom `HttpClient`). +- **Automated Authentication Invalidation (Auto-Logout)**: + - If any API request returns an unauthenticated response (`401 Unauthorized` or `403 Forbidden`), the frontend interceptor **must immediately invalidate** the stored local JWT. + - The application state must instantly trigger an automated logout, clear user tokens/cache, and redirect the user back to the Login screen. + - Under no circumstances should the client remain in an authenticated state after receiving an invalid or expired JWT error from the gateway. + +## 9. Asynchronous Programming Guidelines +- **Async All the Way**: Avoid blocking asynchronous code using `.Result`, `.Wait()`, or `.GetAwaiter().GetResult()`. Use `async`/`await` consistently throughout the call stack to prevent thread pool starvation. +- **Cancellation Tokens**: All asynchronous methods interacting with DBs, network/MQTT calls, or external APIs **must** accept a `CancellationToken` as their last parameter and pass it down to underlying async calls. +- **ValueTask Usage**: Prefer `ValueTask` over `Task` for hot-path methods that frequently complete synchronously (e.g., cached database lookups). + +## 10. Logging and Diagnostics +- **Channel-Based Logging**: Logging must be organized into logical, domain-specific channels (e.g., `MqttChannel`, `DatabaseChannel`, `AnalyzerChannel`, `StrategyChannel`, `RiskEngineChannel`). + - **Dynamic Toggle Control**: Every log channel must be configurable at runtime. Microservices must react to channel configuration changes without requiring a service restart (e.g., via `IOptionsMonitor` or MQTT configuration broadcast events). + - **Admin Panel Integration**: The state of each logging channel (Enabled / Disabled / LogLevel) must be exposed to and manageable from the `FinlyticBackend` Admin Panel. +- **Structured Logging**: Always use structured logging with named placeholders (e.g., `_logger.LogInformation("[{Channel}] Processing recommendation for asset {Symbol}", "AnalyzerChannel", symbol)` instead of string interpolation `$"{symbol}"`). +- **Log Levels Rules**: + - `Trace`/`Debug`: Fine-grained internal flow, payload dumps, and low-level MQTT events. + - `Information`: Business-relevant milestones (e.g., recommendation generated, manual trade validated). + - `Warning`: Recoverable issues or expected unexpected behavior (e.g., transient network retries, rate limits hit). + - `Error`: Unhandled exceptions, failed DB queries, or lost service connections requiring attention. +- **No Sensitive Data**: Never log raw authorization tokens, passwords, or personal credentials. + +## 11. Error Handling and Resilience +- **Explicit Exception Handling**: Catch specific exceptions rather than `System.Exception`. Always log the caught exception with full stack trace using `_logger.LogError(ex, "Message")`. +- **Global Error Handling in Gateway**: `FinlyticBackend` must handle failures gracefully using standardized error responses (e.g., Problem Details). Background services must fail fast or retry safely using resilience pipelines (e.g., Polly for MQTT reconnection/API retries). +- **Result Pattern**: For domain operations where failure is an expected outcome (e.g., trade validation failed due to missing inputs), prefer returning a typed Result object (`Result`) instead of throwing control-flow exceptions. + +## 12. Configuration & Environment Management +- **Strongly-Typed Settings**: All service configurations (e.g., MQTT broker settings, API keys, database connection strings) must be bound to strongly-typed options classes using `IOptions` or `IOptionsMonitor`. +- **No Hardcoded Secrets**: Secrets, connection strings, and private keys must never be hardcoded or checked into source control. Use environment variables or local user secrets (`appsettings.Local.json` excluded via `.gitignore`). + +## 13. Testing and Verification Standards +- **No Production Code Mocks**: Unit tests and integration tests must be located in dedicated test projects (e.g., `FinlyticCore.Tests`). Do not include test-only mock classes inside production assemblies. +- **Deterministic Testing**: Integration tests connecting to real databases or MQTT brokers must run against isolated local containers (e.g., Testcontainers or isolated local DB instances) to ensure non-destructive and predictable test execution. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 108540a..595f0ae 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,4 @@ -services: +services: finlyticassets: image: finlyticassets build: @@ -15,6 +15,7 @@ - MQTT__ClientId=finlytic_assets volumes: - C:\Users\larsh\Documents\docker\finlytic\assets\index:/app/assets/index + - C:\Users\larsh\Documents\docker\finlytic\assets\logos:/app/assets/logos finlyticnews: image: finlyticnews @@ -30,6 +31,105 @@ #- MQTT__Username=admin #- MQTT__Password=${MQTT_PASSWORD} - MQTT__ClientId=finlytic_news + - N8N__ArticleExtractionUrl=${ARTICLE_EXTRACTION_URL} + volumes: + - C:\Users\larsh\Documents\docker\finlytic\assets\index:/app/assets/index:ro + - C:\Users\larsh\Documents\docker\finlytic\data\summaries:/app/data/summaries:ro + + + finlyticfundamentals: + image: finlyticfundamentals + build: + context: . + dockerfile: FinlyticFundamentals/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_fundamentals;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + - MQTT__ClientId=finlytic_fundamentals + + finlyticsentiment: + image: finlyticsentiment + build: + context: . + dockerfile: FinlyticSentiment/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_sentimental;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + - MQTT__ClientId=finlytic_sentiment + - Webhooks__German=https://n8n.kleidukos.me/webhook/sentiment/de + - Webhooks__English=https://n8n.kleidukos.me/webhook/sentiment/en + volumes: + - C:\Users\larsh\Documents\docker\finlytic\data\summaries:/app/data/summaries + + finlytictechnicalanalysis: + image: finlytictechnicalanalysis + build: + context: . + dockerfile: FinlyticTechnicalAnalysis/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_ta;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + - MQTT__ClientId=finlytic_ta + + finlyticanalyzer: + image: finlyticanalyzer + build: + context: . + dockerfile: FinlyticAnalyzer/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_analyzer;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + - MQTT__ClientId=finlytic_analyzer + volumes: + - C:\Users\larsh\Documents\docker\finlytic\data\feedback:/app/data/feedback:ro + + finlytictrades: + image: finlytictrades + build: + context: . + dockerfile: FinlyticTrades/Dockerfile + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_trades;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + - MQTT__ClientId=finlytic_trades + volumes: + - C:\Users\larsh\Documents\docker\finlytic\data\feedback:/app/data/feedback + + finlyticbackend: + image: finlyticbackend + build: + context: . + dockerfile: FinlyticBackend/Dockerfile + ports: + - "5000:8080" + networks: + - postgres-network + environment: + - ConnectionStrings__DefaultConnection=Host=OmniDB;Database=finlytic_backend;Username=admin;Password=${DB_PASSWORD} + - MQTT__Host=host.docker.internal + - MQTT__Port=4545 + - MQTT__ClientId=finlytic_backend + - JWT__SecretKey=${JWT_SECRET_KEY:-FinlyticEnterpriseUltraSecureJwtSecretKey_2026_AtLeast32Chars!} + - ADMIN__DefaultPassword=${ADMIN_DEFAULT_PASSWORD:-AdminDefaultPassword2026!} + - Services__TradesServiceUrl=http://finlytictrades:8080/api/v1/trades/active + volumes: + - C:\Users\larsh\Documents\docker\finlytic\assets\index:/app/assets/index networks: postgres-network: external: true + diff --git a/rebuild-playwright-base.ps1 b/rebuild-playwright-base.ps1 new file mode 100644 index 0000000..52afadd --- /dev/null +++ b/rebuild-playwright-base.ps1 @@ -0,0 +1,47 @@ +# rebuild-playwright-base.ps1 +# ───────────────────────────────────────────────────────────────────────────── +# Rebuild the Playwright base image for FinlyticNews. +# +# Run this script ONLY when you update the Playwright NuGet package version. +# After running this, a normal `docker compose build` will be fast again. +# +# Usage: +# .\rebuild-playwright-base.ps1 +# .\rebuild-playwright-base.ps1 -Version 1.50.0 # when upgrading Playwright +# ───────────────────────────────────────────────────────────────────────────── + +param( + [string]$Version = "1.49.0" +) + +$ImageName = "finlytic-playwright-base:$Version" +$Dockerfile = "FinlyticNews/Dockerfile.playwright-base" +$Context = "FinlyticNews" + +Write-Host "" +Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Cyan +Write-Host " Building Playwright base image: $ImageName" -ForegroundColor Cyan +Write-Host " This only needs to run when the Playwright version changes." -ForegroundColor DarkGray +Write-Host "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ForegroundColor Cyan +Write-Host "" + +docker build ` + -f $Dockerfile ` + --build-arg PLAYWRIGHT_VERSION=$Version ` + -t $ImageName ` + $Context + +if ($LASTEXITCODE -eq 0) { + Write-Host "" + Write-Host "✅ Base image '$ImageName' built and cached locally." -ForegroundColor Green + Write-Host " You can now run 'docker compose build' as usual." -ForegroundColor Green + Write-Host "" + + # Also tag as 'latest' for convenience + docker tag $ImageName "finlytic-playwright-base:latest" + Write-Host " Tagged as 'finlytic-playwright-base:latest' as well." -ForegroundColor DarkGray +} else { + Write-Host "" + Write-Host "❌ Build failed. See errors above." -ForegroundColor Red + exit 1 +} diff --git a/update.py b/update.py new file mode 100644 index 0000000..fcb849f --- /dev/null +++ b/update.py @@ -0,0 +1,29 @@ +import re + +file_path = r"E:\Projects\Finlytic\FinlyticCore\Dtos\Fundamentals\AssetFundamentalsDto.cs" +with open(file_path, "r", encoding="utf-8") as f: + lines = f.readlines() + +out_lines = [] +has_using = any("using System.Text.Json.Serialization;" in l for l in lines) +if not has_using: + for i, line in enumerate(lines): + if "using System;" in line: + out_lines.append(line) + out_lines.append("using System.Text.Json.Serialization;\n") + lines = lines[i+1:] + break + +for i, line in enumerate(lines): + match = re.search(r'^(\s*)public (.+?) ([A-Z][a-zA-Z0-9_]*)( \{.*)$', line) + if match and " record " not in line and " class " not in line: + # Check if previous line has JsonPropertyName + if i == 0 or "JsonPropertyName" not in lines[i-1]: + indent = match.group(1) + prop_name = match.group(3) + camel_name = prop_name[0].lower() + prop_name[1:] + out_lines.append(f'{indent}[JsonPropertyName("{camel_name}")]\n') + out_lines.append(line) + +with open(file_path, "w", encoding="utf-8") as f: + f.writelines(out_lines)