Unify the logging config so that JSON logging works (#456)

* Unify the logging config so that JSON logging works

* Release v1.0.8

gunicorn.conf.py

@@ -1,40 +1,9 @@
 """Configuration for the Gunicorn server."""
 
-import sys
+from datadoc.logging_configuration.logging_config import get_log_config
 
 bind = "0.0.0.0:8050"
 workers = 1
 loglevel = "info"
 preload = True
-
-logconfig_dict = GUNICORN_LOG_CONFIG = {
-    "handlers": {
-        "console_stdout": {
-            "level": "DEBUG",
-            "class": "logging.StreamHandler",
-            "stream": sys.stdout,
-        },
-    },
-    "loggers": {
-        "": {"handlers": ["console_stdout"], "level": "INFO", "propagate": False},
-        "gunicorn": {
-            "handlers": ["console_stdout"],
-            "level": "INFO",
-            "propagate": False,
-        },
-        "gunicorn.access": {
-            "handlers": ["console_stdout"],
-            "level": "INFO",
-            "propagate": False,
-        },
-        "gunicorn.error": {
-            "handlers": ["console_stdout"],
-            "level": "INFO",
-            "propagate": False,
-        },
-    },
-    "root": {
-        "level": "INFO",
-        "handlers": ["console_stdout"],
-    },
-}
+logconfig_dict = get_log_config()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ssb-datadoc"
-version = "1.0.7"
+version = "1.0.8"
 description = "Document dataset metadata. For use in Statistics Norway's metadata system."
 authors = ["Statistics Norway <[email protected]>"]
 license = "MIT"
@@ -161,6 +161,9 @@ max-complexity = 15
 [tool.ruff.lint.pydocstyle]
 convention = "google" # You can also use "numpy".
 
+[tool.ruff.lint.flake8-pytest-style]
+fixture-parentheses = false
+
 [tool.ruff.lint.pep8-naming]
 classmethod-decorators = [
     "classmethod",

src/datadoc/app.py

@@ -24,12 +24,12 @@
 from datadoc.frontend.components.control_bars import build_controls_bar
 from datadoc.frontend.components.control_bars import build_footer_control_bar
 from datadoc.frontend.components.control_bars import header
-from datadoc.logging_configuration.logging_config import configure_logging
+from datadoc.logging_configuration.logging_config import get_log_config
 from datadoc.utils import get_app_version
 from datadoc.utils import pick_random_port
 from datadoc.utils import running_in_notebook
 
-configure_logging()
+logging.config.dictConfig(get_log_config())
 logger = logging.getLogger(__name__)
 
 

src/datadoc/config.py

@@ -2,24 +2,16 @@
 
 from __future__ import annotations
 
-import logging
 import os
-import sys
 from pathlib import Path
-from pprint import pformat
 from typing import Literal
 
 from dapla_metadata.datasets import enums
-from dotenv import dotenv_values
 from dotenv import load_dotenv
 
 from datadoc.constants import DAPLA_MANUAL_TEXT
 from datadoc.frontend.components.builders import build_link_object
 
-logging.basicConfig(level=logging.DEBUG, force=True, stream=sys.stdout)
-
-logger = logging.getLogger(__name__)
-
 DOT_ENV_FILE_PATH = Path(__file__).parent.joinpath(".env")
 
 JUPYTERHUB_USER = "JUPYTERHUB_USER"
@@ -34,18 +26,12 @@ def _load_dotenv_file() -> None:
     if not env_loaded and DOT_ENV_FILE_PATH.exists():
         load_dotenv(DOT_ENV_FILE_PATH)
         env_loaded = True
-        logger.info(
-            "Loaded .env file with config keys: \n%s",
-            pformat(list(dotenv_values(DOT_ENV_FILE_PATH).keys())),
-        )
 
 
 def _get_config_item(item: str) -> str | None:
     """Get a config item. Makes sure all access is logged."""
     _load_dotenv_file()
-    value = os.getenv(item)
-    logger.debug("Config accessed. %s", item)
-    return value
+    return os.getenv(item)
 
 
 def get_jupyterhub_user() -> str | None:

src/datadoc/logging_configuration/gunicorn_access_log_filter.py

@@ -0,0 +1,11 @@
+import logging
+
+EXCLUDED_PATHS = ["/healthz", "/_dash-", "/assets"]
+
+
+class GunicornAccessLoggerHealthProbeFilter(logging.Filter):
+    """Filter out any Gunicorn access logs on Liveness or Readiness probes."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Filter health probes on the /healthz endpoints."""
+        return all(path not in record.getMessage() for path in EXCLUDED_PATHS)

src/datadoc/logging_configuration/logging_config.py

@@ -1,55 +1,69 @@
 from __future__ import annotations
 
-import logging.config
 from typing import Any
 
 from datadoc.config import get_log_formatter
 from datadoc.config import get_log_level
+from datadoc.logging_configuration.gunicorn_access_log_filter import (
+    GunicornAccessLoggerHealthProbeFilter,
+)
 
 
-def configure_logging(config: dict[str, Any] | None = None) -> None:
+def get_log_config() -> dict[str, Any]:
     """Configure logging for the application."""
-    if not config:
-        config = {
-            "version": 1,
-            "disable_existing_loggers": False,
-            "formatters": {
-                "simple": {
-                    "format": "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: %(message)s",
+    return {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "simple": {
+                "format": "%(asctime)s.%(msecs)03d %(levelname)s %(name)s: %(message)s",
+                "logger": "name",
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+            },
+            "json": {
+                "()": "datadoc.logging_configuration.json_formatter.DatadocJSONFormatter",
+                "fmt_keys": {
+                    "level": "levelname",
+                    "message": "message",
+                    "timestamp": "timestamp",
                     "logger": "name",
-                    "datefmt": "%Y-%m-%d %H:%M:%S",
-                },
-                "json": {
-                    "()": "datadoc.logging_configuration.json_formatter.DatadocJSONFormatter",
-                    "fmt_keys": {
-                        "level": "levelname",
-                        "message": "message",
-                        "timestamp": "timestamp",
-                        "logger": "name",
-                        "module": "module",
-                        "function": "funcName",
-                        "line": "lineno",
-                        "thread_name": "threadName",
-                    },
+                    "module": "module",
+                    "function": "funcName",
+                    "line": "lineno",
+                    "thread_name": "threadName",
                 },
             },
-            "handlers": {
-                "stdout": {
-                    "class": "logging.StreamHandler",
-                    "level": get_log_level(),
-                    "formatter": get_log_formatter(),
-                    "stream": "ext://sys.stdout",
-                },
+        },
+        "handlers": {
+            "stdout": {
+                "class": "logging.StreamHandler",
+                "level": get_log_level(),
+                "formatter": get_log_formatter(),
+                "stream": "ext://sys.stdout",
             },
-            "loggers": {
-                "root": {
-                    "level": get_log_level(),
-                    "handlers": [
-                        "stdout",
-                    ],
-                },
+        },
+        "loggers": {
+            "gunicorn": {
+                "handlers": ["stdout"],
+                "level": "INFO",
+                "propagate": False,
             },
-        }
-
-    logging.config.dictConfig(config)
-    logging.getLogger("faker").setLevel(logging.ERROR)
+            "gunicorn.access": {
+                "handlers": ["stdout"],
+                "level": "INFO",
+                "propagate": False,
+                "filters": [GunicornAccessLoggerHealthProbeFilter()],
+            },
+            "gunicorn.error": {
+                "handlers": ["stdout"],
+                "level": "INFO",
+                "propagate": False,
+            },
+        },
+        "root": {
+            "level": get_log_level(),
+            "handlers": [
+                "stdout",
+            ],
+        },
+    }