Merge pull request #19 from LimeDrive/dev

v1.3.0-alpha

README.md

@@ -45,6 +45,7 @@ Le scripte peut désormais être configuré en utilisant des variables d'environ
 - `GUNICORN_TIMEOUT`: Le délai d'attente pour les requêtes Gunicorn. Par défaut, il est défini sur '120'.
 - `LOG_PATH`: Le chemin du fichier journal pour le serveur proxy. Par défaut, il est défini sur '/app/config/logs/rss-proxy.log'. Il y a une rotaion de fichier journal déja configuré. Attention c'est le chemin dans le container.
 - `LOG_LEVEL`: Le niveau de journalisation pour le serveur proxy. Par défaut, il est défini sur 'INFO'.
+- `LOG_REDACTED`: Si les journaux doivent être anonymisés. Par défaut, il est défini sur 'True'.
 - `DB_PATH`: Le chemin de la base de données SQLite pour le serveur proxy. Par défaut, il est défini sur '/app/config/rss-proxy.db'. Attention c'est le chemin dans le container.
 - `SECRET_KEY`: La clé secrète utilisée pour la signature des cookies de session. Par défaut, il est défini sur 'superkey_that_can_be_changed'. Sécurité suplémentaire pour chiffré la base de donnée.
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ygg-rss-proxy"
-version = "1.0.0-alpha"
+version = "1.3.0-alpha"
 description = "Proxy for yggtorrent rss feeds"
 authors = ["LimeDrive <[email protected]>"]
 readme = "README.md"
@@ -18,6 +18,9 @@ flask-session = "^0.8.0"
 flask-sqlalchemy = "^3.1.1"
 orjson = "^3.10.4"
 stackprinter = "^0.2.12"
+timeout-decorator = "^0.5.0"
+tenacity = "^8.3.0"
+toml = "^0.10.2"
 
 
 [build-system]

ygg_rss_proxy/__init__.py

@@ -3,6 +3,7 @@
 from ygg_rss_proxy.app import app
 from ygg_rss_proxy.logging_config import logger
 from ygg_rss_proxy.run_gunicorn import GunicornApp
+from ygg_rss_proxy.version import get_version
 
 options = {
     "bind": f"{settings.gunicorn_binder}:{settings.gunicorn_port}",
@@ -11,11 +12,23 @@
     "timeout": settings.gunicorn_timeout,
 }
 
+try:
+    version = get_version()
+    logger.info("----------------------------------------------------------")
+    logger.info(f"ygg_rss_proxy version: {version}")
+except:
+    logger.info("----------------------------------------------------------")
+    logger.info("ygg_rss_proxy version: unknown")
+
 logger.info("----------------------------------------------------------")
 logger.info("Checking directories...")
 logger.info("----------------------------------------------------------")
 
-directories = ["/app/config", os.path.dirname(settings.db_path), os.path.dirname(settings.log_path)]
+directories = [
+    "/app/config",
+    os.path.dirname(settings.db_path),
+    os.path.dirname(settings.log_path),
+]
 for directory in directories:
     if not os.path.exists(directory):
         os.makedirs(directory)

ygg_rss_proxy/app.py

@@ -1,6 +1,7 @@
 from flask import Flask, request, jsonify, Response
 from flask_session import Session
 from flask_sqlalchemy import SQLAlchemy
+from timeout_decorator import TimeoutError
 from ygg_rss_proxy.rss import get_rss_feed, replace_torrent_links
 from ygg_rss_proxy.settings import settings
 from ygg_rss_proxy.logging_config import logger
@@ -21,6 +22,10 @@
 app.config["SESSION_USE_SIGNER"] = True
 app.config["SESSION_KEY_PREFIX"] = "session:"
 app.config["SECRET_KEY"] = settings.secret_key
+app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {
+    "connect_args": {"timeout": settings.db_timeout}
+}
+
 db = SQLAlchemy(app)
 app.config["SESSION_SQLALCHEMY"] = db
 
@@ -40,7 +45,10 @@ def proxy_rss():
     query_params = request.query_string.decode("utf-8")
     ygg_session = get_session()
 
-    response = get_rss_feed(query_params, requests_session=ygg_session)
+    try:
+        response = get_rss_feed(query_params, requests_session=ygg_session)
+    except TimeoutError as e:
+        logger.error(f"Timeout Err: {e}")
 
     if response.status_code in [401, 403, 307, 301]:
         # Session may have expired, re-authenticate and retry the request

ygg_rss_proxy/auth.py

@@ -1,4 +1,6 @@
 import requests
+import timeout_decorator
+from tenacity import retry, stop_after_attempt, wait_fixed
 from ygg_rss_proxy.fspy import FlareSolverr
 from ygg_rss_proxy.settings import settings
 from ygg_rss_proxy.logging_config import logger
@@ -72,10 +74,15 @@ def ygg_cloudflare_login(
         raise Exception("Failed to connect to FlareSolverr")
 
     response = fs_solver.request_get(url="https://www.ygg.re")
-    logger.debug(f"FlareSolverr response: {response}")
+    logger.debug(f"FlareSolverr message: {response.message}")
+    logger.debug(f"FlareSolverr status: {response.solution.status}")
+    logger.debug(f"FlareSolverr user-agent: {response.solution.user_agent}")
+    logger.debug(f"FlareSolverr cookies: {response.solution.cookies}")
 
     if not response.solution.cookies:
-        logger.error(f"Failed to get cookies from flaresolverr : {response.solution.cookies}")
+        logger.error(
+            f"Failed to get cookies from flaresolverr : {response.solution.cookies}"
+        )
         raise Exception("Failed to get cookies from flaresolverr")
 
     if response.message == "Challenge solved!":
@@ -110,7 +117,7 @@ def ygg_cloudflare_login(
                 break
         # Check if cf_clearance cookie is found
         if not cf_clearance_found:
-            logger.debug(f"Response : {response}")
+            logger.debug(f"Full flaresolverr Response : {response}")
             logger.error(f"Failed to get cf_clearance from flaresolverr")
             raise Exception("Failed to get cf_clearance from flaresolverr")
 
@@ -122,11 +129,19 @@ def ygg_cloudflare_login(
         return session
     else:
         logger.error(
-            f"Failed to authenticate to YGG with status code : {response.solution.status}"
+            f"Failed to authenticate to YGG using flaresolverr: {response.solution.status}"
         )
-        raise Exception("Failed to authenticate to YGG")
+        raise Exception("Failed to authenticate to YGG using flaresolverr")
 
 
+@retry(
+    stop=stop_after_attempt(3),
+    wait=wait_fixed(0.3),
+    retry_error_callback=lambda retry_state: Exception(
+        "Failed to connect to YGG after retries"
+    ),
+)
+@timeout_decorator.timeout(90, exception_message=f"Timeout after 90 seconds")
 def ygg_login(
     session=requests.Session(), ygg_playload: dict = ygg_playload
 ) -> requests.Session:
@@ -149,7 +164,7 @@ def ygg_login(
         logger.info("Cloudflare is enabled, using FlareSolverr")
         return ygg_cloudflare_login(session, ygg_playload)
     else:
-        logger.info("Cloudflare is disabled, using basic login")
+        logger.info("Cloudflare is disabled, using Basic Login")
         return ygg_basic_login(session, ygg_playload)
 
 

ygg_rss_proxy/logging_config.py

@@ -7,6 +7,7 @@
 import re
 import stackprinter
 
+REDACTED = settings.log_redacted
 
 class SecretFilter:
     def __init__(self, patterns):
@@ -25,11 +26,11 @@ def redact(self, message):
 
 
 patterns = [
-    r"passkey=[^&\s]+",
+    r"passkey=([^&\s]+)",
     r"'value': '([^']+)'",
-    r"value=\'[^\']+\'",
-    r"cf_clearance=[^;\s]+",
-    r"ygg_=[^;\s]+",
+    r"value='([^']+)'",
+    r"cf_clearance=([^;\s]+)",
+    r"ygg_=([^;\s]+)"
 ]
 
 logger.remove()
@@ -38,21 +39,24 @@ def redact(self, message):
 def format(record):
     format_ = "{time} {level} {function} {message}\n"
     pats = [
-        r"passkey=[^&\s]+",
+        r"passkey=([^&\s]+)",
         r"'value': '([^']+)'",
-        r"value=\'[^\']+\'",
-        r"cf_clearance=[^;\s]+",
-        r"ygg_=[^;\s]+",
-        r"[A-Za-z0-9]+\' \[GET\] of ygg_rss_proxy\.app>",
+        r"value='([^']+)'",
+        r"cf_clearance=([^;\s]+)",
+        r"ygg_=([^;\s]+)",
+        r"([A-Za-z0-9]+)' \[GET\] of ygg_rss_proxy\.app>",
+        r"session:[A-Za-z0-9_-]+",
+        r"(\{'session_data':\s*b'|<Session data b'|serialized_session_data\s*=\s*b')[\s\S]*?\.(?=\s)",
     ]
 
     if record["exception"] is not None:
         stack = stackprinter.format(
             record["exception"],
             suppressed_vars=[r".*ygg_playload.*", r".*query_params.*"],
         )
-        for pat in pats:
-            stack = re.sub(pat, "**<REDACTED>**", stack)
+        if REDACTED:
+            for pat in pats:
+                stack = re.sub(pat, "**<REDACTED>**", stack)
         record["extra"]["stack"] = stack
         format_ += "{extra[stack]}\n"
     return format_
@@ -63,7 +67,7 @@ def format(record):
     format=format,
     level=settings.log_level.value,
     colorize=True,
-    filter=SecretFilter(patterns),
+    filter=SecretFilter(patterns) if REDACTED else None,
 )
 
 logger.add(
@@ -74,7 +78,7 @@ def format(record):
     retention="5 days",
     compression="zip",
     enqueue=True,
-    filter=SecretFilter(patterns),
+    filter=SecretFilter(patterns) if REDACTED else None,
 )
 
 

ygg_rss_proxy/rss.py

@@ -2,19 +2,22 @@
 from lxml import etree
 from ygg_rss_proxy.settings import settings
 import requests
+import timeout_decorator
 
 # URLs
 URL_RSS: str = f"{settings.ygg_url}/rss"
 URL_TORRENTS: str = f"{settings.ygg_url}/rss/download"
 URL_PROXY = f"{settings.rss_shema}://{settings.rss_host}:{settings.rss_port}"
 
 
+@timeout_decorator.timeout(30, exception_message=f"Timeout after 30 seconds")
 def get_rss_feed(query_params, requests_session: requests.Session) -> requests.Response:
     rss_url_with_params = f"{URL_RSS}?{query_params}"
     response = requests_session.get(rss_url_with_params)
     return response
 
 
+@timeout_decorator.timeout(30, exception_message=f"Timeout after 30 seconds")
 def replace_torrent_links(rss_content) -> Any:
     parser = etree.XMLParser(recover=True)
     tree = etree.fromstring(rss_content, parser)

ygg_rss_proxy/run_gunicorn.py

@@ -2,6 +2,7 @@
 from gunicorn.app.base import BaseApplication
 from ygg_rss_proxy.app import app
 
+
 class GunicornApp(BaseApplication):
     def __init__(self, app, options=None):
         self.options = options or {}