Merge branch 'feature/full-traceback-in-raw' of github.com:minvws/nl-…

…kat-coordination into feature/full-traceback-in-raw

.github/workflows/build-rdo-package.yml

@@ -5,7 +5,7 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  create:
+  push:
     tags:
       - v*
 

.github/workflows/keiko_test.yml

@@ -28,7 +28,17 @@ jobs:
           cache: 'pip' # caching pip dependencies
 
       - name: Install Latex
-        run: sudo apt-get install -y --no-install-recommends texlive-latex-base texlive-latex-extra texlive-latex-recommended texlive-lang-european texlive-fonts-recommended
+        run: |
+          sudo apt-get install -y --no-install-recommends \
+            chktex \
+            fonts-recommended \
+            latexmk \
+            texlive-fonts-recommended \
+            texlive-lang-european \
+            texlive-latex-base \
+            texlive-latex-extra \
+            texlive-latex-recommended \
+            texlive-xetex
 
       - name: Install requirements-dev.txt
         run: pip install -r requirements-dev.txt

Makefile

@@ -1,94 +1,82 @@
 SHELL := bash
 .ONESHELL:
+.NOTPARALLEL:
 
 # use HIDE to run commands invisibly, unless VERBOSE defined
 HIDE:=$(if $(VERBOSE),,@)
 UNAME := $(shell uname)
 
-.PHONY: kat rebuild update clean migrate build itest debian-build-image ubuntu-build-image
+.PHONY: $(MAKECMDGOALS)
 
 # Export Docker buildkit options
 export DOCKER_BUILDKIT=1
 export COMPOSE_DOCKER_CLI_BUILD=1
 
-kat: env-if-empty clean # This should give you a clean install
-	make build
-	make up
+# Build and bring up all containers (default target)
+kat: env-if-empty build up
 
-rebuild: clean
-	make build
-	make up
+# Update using git pull and bring up containers
+update: pull kat
 
-update: down pull
-	make build
-	make up
-
-clean: down # This should clean up all persistent data
-	-docker volume rm nl-kat-coordination_postgres-data nl-kat-coordination_bytes-data nl-kat-coordination_xtdb-data
-
-export version
-
-upgrade: fetch down # Upgrade to the latest release without losing persistent data. Usage: `make upgrade version=v1.5.0` (version is optional)
-ifeq ($(version),)
-	version=$(shell curl --silent  "https://api.github.com/repos/minvws/nl-kat-coordination/tags" | jq -r '.[].name' | grep -v "rc" | head -n 1)
-	make upgrade version=$$version
-else
-	make checkout branch=$(version)
-	make build-all
-	make up
-endif
-
-reset:
-	-docker-compose down --remove-orphans --volumes --timeout 0
-	make up
-	make -C boefjes build
-	make -C rocky almost-flush
+# Bring down containers, remove all volumes, and bring them up again
+reset: clean kat
 
+# Bring up containers
 up:
-	docker-compose up -d --force-recreate
+	docker-compose up --detach
 
+# Bring down containers without data loss
 down:
 	-docker-compose down
 
+# Bring down containers and remove all volumes
+clean:
+	-docker-compose down --timeout 0 --volumes --remove-orphans
+
+# Fetch the latest changes from the Git remote
 fetch:
-	-git fetch
+	git fetch --all --prune --tags
 
+# Pull the latest changes from the default upstream
 pull:
-	-git pull
+	git pull
+
+# Upgrade to the latest release without losing persistent data. Usage: `make upgrade version=v1.5.0` (version is optional)
+VERSION?=$(shell curl -sSf "https://api.github.com/repos/minvws/nl-kat-coordination/tags" | jq -r '[.[].name | select(. | contains("rc") | not)][0]')
+upgrade: down fetch
+	git checkout $(VERSION)
+	make kat
 
+# Create .env file only if it does not exist
 env-if-empty:
 ifeq ("$(wildcard .env)","")
 	make env
 endif
 
-env:  # Create .env file from the env-dist with randomly generated credentials from vars annotated by "{%EXAMPLE_VAR}"
-	$(HIDE) cp .env-dist .env
+# Create .env file from the env-dist with randomly generated credentials from vars annotated by "{%EXAMPLE_VAR}"
+env:
+	cp .env-dist .env
+	echo "Initializing .env with random credentials"
 ifeq ($(UNAME), Darwin)  # Different sed on MacOS
 	$(HIDE) grep -o "{%\([_A-Z]*\)}" .env-dist | sort -u | while read v; do sed -i '' "s/$$v/$$(openssl rand -hex 25)/g" .env; done
 else
 	$(HIDE) grep -o "{%\([_A-Z]*\)}" .env-dist | sort -u | while read v; do sed -i "s/$$v/$$(openssl rand -hex 25)/g" .env; done
 endif
 
-checkout: # Usage: `make checkout branch=develop`
-	-git checkout $(branch)
-
-pull-reset:
-	-git reset --hard HEAD
-	-git pull
-
-build-all:  # Build should prepare all other services: migrate them, seed them, etc.
-ifeq ($(UNAME), Darwin)
-	docker-compose build --build-arg USER_UID="$$(id -u)"
+# Build will prepare all services: migrate them, seed them, etc.
+build:
+ifeq ($(UNAME),Darwin)
+	docker-compose build --pull --parallel --build-arg USER_UID="$$(id -u)"
 else
-	docker-compose build --build-arg USER_UID="$$(id -u)" --build-arg USER_GID="$$(id -g)"
+	docker-compose build --pull --parallel --build-arg USER_UID="$$(id -u)" --build-arg USER_GID="$$(id -g)"
 endif
-
-build: build-all
 	make -C rocky build
 	make -C boefjes build
 
+# Build Debian build image
 debian-build-image:
 	docker build -t kat-debian-build-image packaging/debian
 
+# Build Ubuntu build image
 ubuntu-build-image:
 	docker build -t kat-ubuntu-build-image packaging/ubuntu

docker-compose.yml

@@ -117,14 +117,7 @@ services:
   rocky:
     restart: unless-stopped
     depends_on:
-      - octopoes_api_worker
-      - octopoes_api
       - postgres
-      - boefje
-      - normalizer
-      - katalogus
-      - scheduler
-      - keiko
     ports:
       - "127.0.0.1:8000:80"
     build:

docs/source/technical_design/localinstall.rst

@@ -107,6 +107,20 @@ Go to the directory containing openkat:
 	$ cd nl-kat-coordination
 	$ make update
 
+Clean reinstallation
+--------------------
+
+If you to start over with a clean slate, you can do so with the following commands:
+
+.. code-block:: sh
+
+	$ cd nl-kat-coordination
+	$ make reset
+
+This removes all Docker containers and volumes, and then brings up the containers again.
+
+Optionally, first remove the ``.env`` file (``rm .env``) before running ``make reset`` to also reset all configuration in environment variables. This should also resolve issues such as database authentication errors (``password authentication failed``).
+
 OpenTelemetry
 =============
 

keiko/Dockerfile

@@ -14,8 +14,15 @@ ENV PATH=/home/keiko/.local/bin:${PATH}
 # LateX dependencies
 RUN apt update -y \
     && apt install -y --no-install-recommends \
-        texlive-latex-base texlive-latex-extra texlive-latex-recommended \
-        texlive-lang-european texlive-fonts-recommended chktex \
+        chktex \
+        fonts-recommended \
+        latexmk \
+        texlive-fonts-recommended \
+        texlive-lang-european \
+        texlive-latex-base \
+        texlive-latex-extra \
+        texlive-latex-recommended \
+        texlive-xetex \
     && rm -rf /var/lib/apt/lists/*
 
 ENV KEIKO_REPORTS_FOLDER=/reports

keiko/debian/control

@@ -7,6 +7,6 @@ Section: python
 Priority: optional
 Architecture: any
 Pre-Depends: ${misc:Pre-Depends}
-Depends: ${python}, texlive-latex-base, texlive-latex-extra, texlive-latex-recommended, texlive-lang-european, texlive-fonts-recommended, chktex, ${misc:Depends}, ${shlibs:Depends}
+Depends: ${python}, chktex, fonts-recommended, latexmk, texlive-fonts-recommended, texlive-lang-european, texlive-latex-base, texlive-latex-extra, texlive-latex-recommended, texlive-xetex, ${misc:Depends}, ${shlibs:Depends}
 Description: Keiko
  The PDF report generation tool for KAT.

keiko/keiko/keiko.py

@@ -5,11 +5,11 @@
 import shutil
 import subprocess
 import tempfile
-from logging import getLogger
+from logging import DEBUG, ERROR, getLogger
 from pathlib import Path
-from typing import Dict, Set, Tuple
+from typing import Any, Dict, Set, Tuple
 
-from jinja2 import Environment, FileSystemLoader, select_autoescape
+from jinja2 import Environment, FileSystemLoader
 from opentelemetry import trace
 from opentelemetry.trace import Status, StatusCode
 
@@ -22,6 +22,39 @@
 
 DATA_SHAPE_CLASS_NAME = "DataShape"
 
+LATEX_SPECIAL_CHARS = str.maketrans(
+    {
+        "&": r"\&",
+        "%": r"\%",
+        "$": r"\$",
+        "#": r"\#",
+        "_": r"\_",
+        "{": r"\{",
+        "}": r"\}",
+        "~": r"\textasciitilde{}",
+        "^": r"\^{}",
+        "\\": r"\textbackslash{}",
+        "\n": "\\newline%\n",
+        "-": r"{-}",
+        "\xA0": "~",  # Non-breaking space
+        "[": r"{[}",
+        "]": r"{]}",
+    }
+)
+
+
+def latex_escape(input: Any) -> str:
+    """Escape characters that are special in LaTeX.
+
+    References:
+    - https://github.com/JelteF/PyLaTeX/blob/ecc1e6e339a5a7be958c328403517cd547873d7e/pylatex/utils.py#L68-L100
+    - http://tex.stackexchange.com/a/34586/43228
+    - http://stackoverflow.com/a/16264094/2570866
+    """
+    if not isinstance(input, str):
+        input = str(input)
+    return input.translate(LATEX_SPECIAL_CHARS)
+
 
 def baretext(input_: str) -> str:
     """Remove non-alphanumeric characters from a string."""
@@ -58,10 +91,10 @@ def generate_report(
     # init jinja2 template
     env = Environment(
         loader=FileSystemLoader(settings.templates_folder),
-        autoescape=select_autoescape(),
         variable_start_string="@@{",
         variable_end_string="}@@",
     )
+    env.filters["latex_escape"] = latex_escape
     template = env.get_template(f"{template_name}/template.tex")
 
     if not template.filename:
@@ -135,32 +168,41 @@ def generate_report(
 
         # run pdflatex
         cmd = [
-            "pdflatex",
+            "latexmk",
+            "-xelatex",
             "-synctex=1",
             "-interaction=nonstopmode",
             preprocessed_tex_path.as_posix(),
         ]
         env = {**os.environ, "TEXMFVAR": directory}
-        for i in (1, 2):
-            output = subprocess.run(cmd, cwd=directory, env=env, capture_output=True, check=False)
-            current_span.add_event(f"Completed pdflatex run {i}")
+
+        def log_output(level, output):
+            if not logger.isEnabledFor(level):
+                return
+            # prefix all lines in output
+            for line in output.decode("utf-8").splitlines():
+                logger.log(level, "latexmk [report_id=%s] output: %s", report_id, line)
+
+        try:
+            # capture all output to stdout, so that lines from stdout+stderr are in correct relative order
+            output = subprocess.run(
+                cmd, cwd=directory, env=env, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+            )
+            current_span.add_event("Completed latexmk")
             logger.info(
-                "pdflatex [run=%d] [report_id=%s] [template=%s] [command=%s]",
-                i,
+                "latexmk [report_id=%s] [template=%s] [command=%s]",
                 report_id,
                 template_name,
                 " ".join(cmd),
             )
-            if output.returncode:
-                logger.error("stdout: %s", output.stdout.decode("utf-8"))
-                logger.error("stderr: %s", output.stderr.decode("utf-8"))
-                ex = Exception("Error in pdflatex run %d", i)
-                current_span.set_status(Status(StatusCode.ERROR))
-                current_span.record_exception(ex)
-                raise ex
-            else:
-                logger.debug(output.stdout.decode("utf-8"))
-                logger.debug(output.stderr.decode("utf-8"))
+            log_output(DEBUG, output.stdout)
+        except subprocess.CalledProcessError as ex:
+            log_output(ERROR, ex.stdout)
+            err = Exception("Error in latexmk")
+            err.__cause__ = ex
+            current_span.set_status(Status(StatusCode.ERROR))
+            current_span.record_exception(err)
+            raise err
 
         # copy result back to output folder
         shutil.copyfile(