Merge branch 'main' into feature/mula/rate-limit-2

* main:
  Add sectxt dependency (#1610)
  Refactor environment settings, names, and documentation (#1517)
  Add pipeline to check if there are new translation strings (#1606)
  Translations update from Hosted Weblate (#1604)
  Update scheduler documentation (#1476)
  Add community install/update scripts (#1309)
  Bump actions/checkout from 1 to 3 (#1598)
  Run docker-compose pull in make pull (#1585)
  Configure github actions in dependabot (#1594)
  fix many ports open normalizer (#1592)
  Fix human-readable name for ImageMetadata (#1558)
  Upgrade FastAPI (#1576)
  OOI Detail page: Remember page position after clicking the "show inheritance" link (#1590)
  Fix `rstcheck` hook (#1584)

.env-defaults

@@ -0,0 +1,28 @@
+# Container entrypoints will run database migrations if set to "true"
+DATABASE_MIGRATION=true
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG=True
+
+# SECURITY WARNING: enable two factor authentication in production!
+TWOFACTOR_ENABLED=False
+
+# --- Endpoints --- #
+OCTOPOES_API=http://octopoes_api:80
+SCHEDULER_API=http://scheduler:8000
+KEIKO_API=http://keiko:8000
+KATALOGUS_API=http://katalogus:8000
+XTDB_URI=http://crux:3000
+
+# Bytes uses JWT for authentication
+BYTES_API=http://bytes:8000
+
+# --- Rocky --- #
+DJANGO_SUPERUSER_EMAIL=superuser@localhost
+DJANGO_SUPERUSER_FULL_NAME="KAT Superuser"
+
+# https://docs.openkat.nl/technical_design/hardening.html#django-allowed-hosts
+DJANGO_ALLOWED_HOSTS=127.0.0.1,localhost
+
+# https://docs.openkat.nl/technical_design/hardening.html#django-csrf-trusted-origins
+DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1

.env-dist

@@ -1,95 +1,82 @@
-# Postgresql
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD={%POSTGRES_PASSWORD}
+# === Notes === #
+
+# {% ... } values are generated dynamically for security reasons when this file is copied through `make env`.
+# After removing your `.env` file, you must run `make reset` to wipe the old container credentials.
+
+# `.env` overrides variables from `.env-defaults` if included
 
-# General
-DATABASE_MIGRATION=true
 # If you use docker-compose.release-example.yml as base for a docker compose
 # based setup you can use this variable to define the version of OpenKAT you
-# want to use.
+# want to use. Example value: v1.9.0
 # KAT_VERSION=
 
-# Rocky
-SECRET_KEY={%SECRET_KEY}
-DEBUG=True
-TWOFACTOR_ENABLED=True
-DJANGO_ALLOWED_HOSTS=127.0.0.1,localhost
-DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1
-DJANGO_SUPERUSER_EMAIL=superuser@localhost
-DJANGO_SUPERUSER_FULL_NAME="KAT Superuser"
+# --- PostgreSQL --- #
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD={%POSTGRES_PASSWORD}
+
+# --- RabbitMQ --- #
+RABBITMQ_DEFAULT_VHOST=kat
+RABBITMQ_DEFAULT_USER={%QUEUE_USERNAME}
+RABBITMQ_DEFAULT_PASS={%QUEUE_PASSWORD}
+
+# --- OpenTelemetry --- #
+# Uncomment to enable OpenTelemetry https://docs.openkat.nl/technical_design/localinstall.html#opentelemetry
+# SPAN_EXPORT_GRPC_ENDPOINT=http://jaeger:4317
+
+# --- Octopoes, Boefjes & Bytes shared --- #
+QUEUE_URI=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/${RABBITMQ_DEFAULT_VHOST}
+BYTES_USERNAME={%BYTES_USERNAME}
+BYTES_PASSWORD={%BYTES_PASSWORD}
+
+### --- MODULE SPECIFIC SETTINGS --- ###
+
+# --- Rocky --- #
+# See `rocky/rocky/settings.py`
+
 DJANGO_SUPERUSER_PASSWORD={%DJANGO_SUPERUSER_PASSWORD}
 
-# Rocky DB
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY={%SECRET_KEY}
+
 ROCKY_DB=rocky
 ROCKY_DB_USER=rocky_app
-ROCKY_DB_PASSWORD={%ROCKY_DB_PASSWORD}
 ROCKY_DB_HOST=postgres
 ROCKY_DB_PORT=5432
-ROCKY_DB_USER_CREATEDB=CREATEDB
-# ROCKY_DB_DSN=
+ROCKY_DB_PASSWORD={%ROCKY_DB_PASSWORD}
 
-# Queueing
-RABBITMQ_DEFAULT_VHOST=kat
-RABBITMQ_DEFAULT_USER={%QUEUE_USERNAME}
-RABBITMQ_DEFAULT_PASS={%QUEUE_PASSWORD}
 
-# Boefjes and Normalizers
-QUEUE_URI=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/${RABBITMQ_DEFAULT_VHOST}
+# --- Boefjes --- #
+# See `boefjes/boefjes/config.py`
+# See https://docs.openkat.nl/technical_design/environmentvariables.html#boefjes
 
-# Endpoints
-OCTOPOES_API=http://octopoes_api:80
-BYTES_API=http://bytes:8000
-SCHEDULER_API=http://scheduler:8000
-KEIKO_API=http://keiko:8000
+KATALOGUS_DB=katalogus
+KATALOGUS_DB_USER=katalogus_app
+KATALOGUS_DB_PASSWORD={%KATALOGUS_DB_PASSWORD}
+KATALOGUS_DB_URI=postgresql://${KATALOGUS_DB_USER}:${KATALOGUS_DB_PASSWORD}@postgres:5432/${KATALOGUS_DB}
 
-# Bytes API
-SECRET={%SECRET}
-ACCESS_TOKEN_EXPIRE_MINUTES=1000
-BYTES_USERNAME={%BYTES_USERNAME}
-BYTES_PASSWORD={%BYTES_PASSWORD}
+# --- Bytes --- #
+# See `bytes/bytes/config.py`
+# See https://docs.openkat.nl/technical_design/environmentvariables.html#bytes
+
+BYTES_SECRET={%SECRET}
 
-# Bytes DB
 BYTES_DB=bytes
 BYTES_DB_USER=bytes_app
 BYTES_DB_PASSWORD={%BYTES_DB_PASSWORD}
 BYTES_DB_URI=postgresql://${BYTES_DB_USER}:${BYTES_DB_PASSWORD}@postgres:5432/${BYTES_DB}
-BYTES_METRICS_TTL_SECONDS=0
 
-# Bytes FileSystem
-BYTES_FILE_PERMISSION=640
-BYTES_FOLDER_PERMISSION=740
-ENCRYPTION_MIDDLEWARE=IDENTITY
 
-# KATalogus API
-KATALOGUS_API=http://katalogus:8000
+# --- Keiko --- #
+# See `keiko/keiko/settings.py`
+
+# --- Octopoes --- #
+# See `octopoes/octopoes/config/settings.py`
 
-# KATalogus DB
-KATALOGUS_DB=katalogus
-KATALOGUS_DB_USER=katalogus_app
-KATALOGUS_DB_PASSWORD={%KATALOGUS_DB_PASSWORD}
-KATALOGUS_DB_URI=postgresql://${KATALOGUS_DB_USER}:${KATALOGUS_DB_PASSWORD}@postgres:5432/${KATALOGUS_DB}
 
-# Octopoes
-OCTOPOES_SCAN_LEVEL_RECALCULATION_INTERVAL=60
-XTDB_URI=http://crux:3000
-XTDB_TYPE=xtdb-multinode
+# --- Mula --- #
+# See `mula/scheduler/config/settings.py`
 
-# Scheduler
-SCHEDULER_PQ_MAXSIZE=1000
-SCHEDULER_PQ_INTERVAL=60
-SCHEDULER_RABBITMQ_DSN=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/${RABBITMQ_DEFAULT_VHOST}
 SCHEDULER_DB=scheduler
 SCHEDULER_DB_USER=scheduler_app
 SCHEDULER_DB_PASSWORD={%SCHEDULER_DB_PASSWORD}
-SCHEDULER_DB_DSN=postgresql://${SCHEDULER_DB_USER}:${SCHEDULER_DB_PASSWORD}@postgres:5432/${SCHEDULER_DB}
-
-# Keiko
-KEIKO_DEBUG=False
-KEIKO_LOG_CFG=logging.json
-KEIKO_TEMPLATES_FOLDER=templates
-KEIKO_GLOSSARIES_FOLDER=glossaries
-KEIKO_REPORTS_FOLDER=/reports
-KEIKO_ASSETS_FOLDER=assets
-
-# OpenTracing
-# SPAN_EXPORT_GRPC_ENDPOINT=http://jaeger:4317
+SCHEDULER_DB_URI=postgresql://${SCHEDULER_DB_USER}:${SCHEDULER_DB_PASSWORD}@postgres:5432/${SCHEDULER_DB}

.github/dependabot.yml

@@ -12,3 +12,7 @@ updates:
   ignore:
   - dependency-name: "*"
     update-types: ["version-update:semver-minor"]
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "monthly"

.github/workflows/build_docs_on_pr.yml

@@ -28,7 +28,7 @@ jobs:
         run: pip install -r requirements.txt
 
       - name: Compile static HTML
-        run: sphinx-build -b html docs/source docs/_build
+        run: make docs
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3

.github/workflows/bytes_tests.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
 
       - name: Run unit tests
         run: make utest build_args='--build-arg PYTHON_VERSION=${{ matrix.version }}'

.github/workflows/deploy_docs.yml

@@ -39,7 +39,7 @@ jobs:
         run: pip install -r requirements.txt
 
       - name: Compile static HTML
-        run: sphinx-build -b html docs/source docs/_build
+        run: make docs
 
       - name: Setup Pages
         uses: actions/configure-pages@v2

.github/workflows/keiko_itest.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/keiko_test.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.version }}
         uses: actions/setup-python@v4

.github/workflows/keiko_tex_linters.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
 
       - name: Install chktex
         run: sudo apt-get install chktex

.github/workflows/octopoes_tests.yml

@@ -34,7 +34,7 @@ jobs:
         run: pytest --cov-report=xml
         working-directory: ./octopoes
 
-      - if: ${{ matrix.version == '3.8' }}
+      - if: ${{ matrix.version == '3.11' }}
         name: Upload coverage as artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/rocky_makelang.yml

@@ -0,0 +1,40 @@
+name: Rocky Check if translations are up to date
+
+on:
+  push:
+    paths:
+      - rocky/**
+  pull_request:
+    paths:
+      - rocky/**
+
+jobs:
+  makelang:
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        id: py311
+        with:
+          python-version: 3.11
+          cache: pip
+
+      - name: Install GNU gettext utilities
+        run: sudo apt-get install -y --no-install-recommends gettext
+
+      - name: Install requirements.txt
+        run: pip install -r requirements.txt
+        working-directory: ./rocky
+
+      - name: Install Octopoes
+        run: pip install wheel && cd octopoes && python setup.py bdist_wheel && pip install dist/octopoes*.whl
+
+      - name: Generate the .po files if they changed
+        run: SECRET_KEY="whatever" make lang
+        working-directory: ./rocky
+
+      - name: Check if there are any changed files
+        run: git diff --exit-code

.github/workflows/test_debian_packages_on_ubuntu.yml

@@ -180,7 +180,7 @@ jobs:
 
       - name: Update configs for rabbitmq
         run: |
-          sudo sed -i "s/SCHEDULER_RABBITMQ_DSN=/SCHEDULER_RABBITMQ_DSN=amqp:\/\/kat:rabbit@localhost:5672\/kat/g" /etc/kat/mula.conf
+          sudo sed -i "s/QUEUE_URI=/QUEUE_URI=amqp:\/\/kat:rabbit@localhost:5672\/kat/g" /etc/kat/mula.conf
           sudo sed -i "s/QUEUE_URI=/QUEUE_URI=amqp:\/\/kat:rabbit@localhost:5672\/kat/g" /etc/kat/rocky.conf
           sudo sed -i "s/QUEUE_URI=/QUEUE_URI=amqp:\/\/kat:rabbit@localhost:5672\/kat/g" /etc/kat/bytes.conf
           sudo sed -i "s/QUEUE_URI=/QUEUE_URI=amqp:\/\/kat:rabbit@localhost:5672\/kat/g" /etc/kat/boefjes.conf
@@ -204,7 +204,7 @@ jobs:
 
       - name: Migrations for Mula
         run: |
-          sudo sed -i "s/SCHEDULER_DB_DSN=/SCHEDULER_DB_DSN=postgresql:\/\/mula:postgres@localhost\/mula_db/g" /etc/kat/mula.conf
+          sudo sed -i "s/SCHEDULER_DB_URI=/SCHEDULER_DB_URI=postgresql:\/\/mula:postgres@localhost\/mula_db/g" /etc/kat/mula.conf
           sudo -u kat update-mula-db
 
       - name: Setup Bytes credentials
@@ -213,9 +213,6 @@ jobs:
           sudo sed -i "s/BYTES_PASSWORD=\$/BYTES_PASSWORD=$(sudo grep BYTES_PASSWORD /etc/kat/bytes.conf | awk -F'=' '{ print $2 }')/" /etc/kat/boefjes.conf
           sudo sed -i "s/BYTES_PASSWORD=\$/BYTES_PASSWORD=$(sudo grep BYTES_PASSWORD /etc/kat/bytes.conf | awk -F'=' '{ print $2 }')/" /etc/kat/mula.conf
 
-      - name: Setup Octopoes to use xtdb-multinode
-        run: sudo sed -i "s/XTDB_TYPE=\"xtdb\"/XTDB_TYPE=\"xtdb-multinode\"/g" /etc/kat/octopoes.conf
-
       - name: Restart KAT
         run: sudo systemctl restart kat-rocky kat-mula kat-bytes kat-boefjes kat-normalizers kat-katalogus kat-keiko kat-octopoes kat-octopoes-worker
 

.gitignore

@@ -436,3 +436,6 @@ deployment/github_config.sh
 
 nl-kat-*
 /_version.py
+
+# Automatically generated markdown files for the environment variables
+/docs/source/technical_design/environment_settings/*.md

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
     hooks:
       - id: rstcheck
         # https://github.com/rstcheck/rstcheck-core/issues/4
-        args: ["--ignore-messages", "Hyperlink target .* is not referenced"]
+        args: ["--ignore-messages", "Hyperlink target .* is not referenced", "--ignore-directives", "mermaid"]
         additional_dependencies: ["rstcheck[sphinx]"]
 
   - repo: https://github.com/MarketSquare/robotframework-tidy

Makefile

@@ -12,6 +12,15 @@ UNAME := $(shell uname)
 export DOCKER_BUILDKIT=1
 export COMPOSE_DOCKER_CLI_BUILD=1
 
+define build-settings-doc
+	echo "# $$(echo "$(3)" | sed 's/.*/\u&/')" > docs/source/technical_design/environment_settings/$(3).md
+	DOCS=True PYTHONPATH=./$(1) settings-doc generate \
+	-f markdown -m $(2) \
+	--templates docs/settings-doc-templates \
+	>> docs/source/technical_design/environment_settings/$(3).md
+endef
+
+
 # Build and bring up all containers (default target)
 kat: env-if-empty build up
 	@echo
@@ -51,6 +60,7 @@ fetch:
 # Pull the latest changes from the default upstream
 pull:
 	git pull
+	docker-compose pull
 
 # Upgrade to the latest release without losing persistent data. Usage: `make upgrade version=v1.5.0` (version is optional)
 VERSION?=$(shell curl -sSf "https://api.github.com/repos/minvws/nl-kat-coordination/tags" | jq -r '[.[].name | select(. | contains("rc") | not)][0]')
@@ -97,6 +107,11 @@ ubuntu22.04-build-image:
 	docker build -t kat-ubuntu22.04-build-image packaging/ubuntu22.04
 
 docs:
+	$(call build-settings-doc,keiko,keiko.settings,keiko)
+	$(call build-settings-doc,octopoes,octopoes.config.settings,octopoes)
+	$(call build-settings-doc,boefjes,boefjes.config,boefjes)
+	$(call build-settings-doc,bytes,bytes.config,bytes)
+	$(call build-settings-doc,mula/scheduler,config.settings,mula)
 	sphinx-build -b html docs/source docs/_build
 
 poetry-dependencies: