feat!: Upgrade rtn for adult check handling

iPromKnight · Nov 19, 2024 · 5422baa · 5422baa
1 parent eab31ce
commit 5422baa
Show file tree

Hide file tree

Showing 41 changed files with 1,598 additions and 637 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -17,7 +17,7 @@
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http" Version="9.0.0" />
     <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
-    <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.0-rc.2" />
+    <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.0" />
     <PackageVersion Include="NSubstitute" Version="5.3.0" />
     <PackageVersion Include="pythonnet" Version="3.0.4" />
     <PackageVersion Include="Serilog.Sinks.Spectre" Version="0.5.0" />
@@ -32,7 +32,7 @@
     <PackageVersion Include="Testcontainers" Version="4.0.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.0.0" />
     <PackageVersion Include="KubernetesClient" Version="15.0.1" />
-    <PackageVersion Include="Verify.Xunit" Version="28.3.1" />
+    <PackageVersion Include="Verify.Xunit" Version="28.3.2" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions" Version="21.1.3" />
     <PackageVersion Include="xunit" Version="2.9.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2">

diff --git a/README.md b/README.md
@@ -14,52 +14,49 @@ The DMM import reruns on missing pages every hour.
 ```json
 {
   "Zilean": {
-    "ApiKey": "5c43b70d3be04308b72ada4f61515fb4e278b08c48ec4c8a87e954ec658f8e4e",
-    "FirstRun": false,
+    "ApiKey": "69f72d7eb22e48938fd889206ffcf911a514bcc2e3824b2e9e7549122fb16849",
+    "FirstRun": true,
     "Dmm": {
       "EnableScraping": true,
       "EnableEndpoint": true,
       "ScrapeSchedule": "0 * * * *",
       "MinimumReDownloadIntervalMinutes": 30,
       "MaxFilteredResults": 200,
-      "MinimumScoreMatch": 0.85,
-      "ImportBatched": false
+      "MinimumScoreMatch": 0.85
     },
     "Torznab": {
       "EnableEndpoint": true
     },
     "Database": {
-      "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;"
+      "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;"
     },
     "Torrents": {
-      "EnableEndpoint": true
+      "EnableEndpoint": false
     },
     "Imdb": {
       "EnableImportMatching": true,
       "EnableEndpoint": true,
       "MinimumScoreMatch": 0.85
     },
     "Ingestion": {
-      "ZurgInstances": [
-        {
-          "Url": "http://zurg:9999",
-          "EndpointType": 1
-        }
-      ],
+      "ZurgInstances": [],
       "ZileanInstances": [],
-      "EnableScraping": true,
+      "EnableScraping": false,
       "Kubernetes": {
         "EnableServiceDiscovery": false,
         "KubernetesSelectors": [],
         "KubeConfigFile": "/$HOME/.kube/config",
         "AuthenticationType": 0
       },
-      "BatchSize": 500,
-      "MaxChannelSize": 5000,
       "ScrapeSchedule": "0 * * * *",
       "ZurgEndpointSuffix": "/debug/torrents",
       "ZileanEndpointSuffix": "/torrents/all",
       "RequestTimeout": 10000
+    },
+    "Parsing": {
+      "IncludeAdult": false,
+      "IncludeTrash": true,
+      "BatchSize": 5000
     }
   }
 }
@@ -127,8 +124,6 @@ The `Ingestion` section in the JSON configuration defines the behavior and optio
     "KubeConfigFile": "/$HOME/.kube/config",
     "AuthenticationType": 0
   },
-  "BatchSize": 500,
-  "MaxChannelSize": 5000,
   "ScrapeSchedule": "0 * * * *",
   "ZurgEndpointSuffix": "/debug/torrents",
   "ZileanEndpointSuffix": "/torrents/all",
@@ -196,14 +191,6 @@ The `Ingestion` section in the JSON configuration defines the behavior and optio
     - **`KubeConfigFile`**: Path to the Kubernetes configuration file.
     - **`AuthenticationType`**: Authentication type for Kubernetes service discovery (0 = ConfigFile, 1 = RoleBased).
 
-### `BatchSize`
-- **Type**: `int`
-- **Description**: Number of torrents to process in a single batch.
-
-### `MaxChannelSize`
-- **Type**: `int`
-- **Description**: Maximum number of items to buffer in memory during ingestion.
-
 ### `ScrapeSchedule`
 - **Type**: `string` (CRON format)
 - **Description**: Schedule for automated scraping tasks.

diff --git a/eng/install-python-reqs-dmmscraper.ps1 b/eng/install-python-reqs-dmmscraper.ps1
@@ -1,3 +1,3 @@
-Remove-Item -Path ../src/Zilean.DmmScraper/python -Recurse -Force
-New-Item -Path ../src/Zilean.DmmScraper/python -ItemType Directory
-python -m pip install -r ../requirements.txt -t ../src/Zilean.DmmScraper/python/ --no-user
+Remove-Item -Path ../src/Zilean.Scraper/python -Recurse -Force
+New-Item -Path ../src/Zilean.Scraper/python -ItemType Directory
+python -m pip install -r ../requirements.txt -t ../src/Zilean.Scraper/python/ --no-user
diff --git a/eng/install-python-reqs-dmmscraper.sh b/eng/install-python-reqs-dmmscraper.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
-rm -rf ../src/Zilean.DmmScraper/python
-mkdir -p ../src/Zilean.DmmScraper/python
-python3.11 -m pip install -r ../requirements.txt -t ../src/Zilean.DmmScraper/python/
+rm -rf ../src/Zilean.Scraper/python
+mkdir -p ../src/Zilean.Scraper/python
+python3.11 -m pip install -r ../requirements.txt -t ../src/Zilean.Scraper/python/
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
 loguru==0.7.2
 rich==13.8.0
-rank-torrent-name==1.0.4
+rank-torrent-name==1.5.3