Merge branch 'refs/heads/main' into chore/docs-on-oci-image-building-…

…design

# Conflicts:
#	docs/source/developer_documentation/boefjes-runner.md

.env-defaults

@@ -30,3 +30,7 @@ DJANGO_CSRF_TRUSTED_ORIGINS=http://localhost,http://127.0.0.1
 
 # This allows running pytest inside the container
 ROCKY_DB_USER_CREATEDB=CREATEDB
+
+# This is normally False when DEBUG is true, but we override that in settings.py
+# so it possible to set DEBUG to True in production like environments.
+COMPRESS_ENABLED=False

.github/workflows/boefjes_tests.yml

@@ -39,7 +39,7 @@ jobs:
         run: python3 -m pip install --upgrade pip
 
       - name: Install dev requirements
-        run: pip install -r requirements-dev.txt
+        run: grep -v git+https:// requirements-dev.txt | pip install -r /dev/stdin && grep git+https:// requirements-dev.txt | pip install -r /dev/stdin
         working-directory: ./boefjes
 
       - name: Install requirements

.github/workflows/build-rdo-package.yml

@@ -187,7 +187,7 @@ jobs:
         run: python3.10 -m venv /var/www/html/.venv
 
       - name: Install requirements
-        run: source .venv/bin/activate; pip install --upgrade pip; find . -name requirements.txt | xargs -L 1 pip install -r; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
+        run: source .venv/bin/activate; pip install --upgrade pip; grep -v git+https:// requirements.txt | pip install -r /dev/stdin ; grep git+https:// requirements.txt | pip install -r /dev/stdin; pip install ${{ github.workspace }}/octopoes/dist/octopoes*.whl
         working-directory: /var/www/html
 
       - name: Create archive

.github/workflows/deploy_docs.yml

@@ -42,7 +42,7 @@ jobs:
         run: make docs
 
       - name: Setup Pages
-        uses: actions/configure-pages@v4
+        uses: actions/configure-pages@v5
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.pre-commit-config.yaml

@@ -36,12 +36,12 @@ repos:
     files: pyproject.toml$
 
 - repo: https://github.com/rstcheck/rstcheck
-  rev: v6.2.0
+  rev: v6.2.1
   hooks:
   - id: rstcheck
     # https://github.com/rstcheck/rstcheck-core/issues/4
-    args: ["--ignore-messages", "Hyperlink target .* is not referenced", "--ignore-directives", "mermaid"]
-    additional_dependencies: ["rstcheck[sphinx]"]
+    args: ["--ignore-messages", "Hyperlink target .* is not referenced", "--ignore-directives", "mermaid,automodule"]
+    additional_dependencies: ["rstcheck[sphinx]", "autodoc-pydantic==2.1.0"]
 
 - repo: https://github.com/MarketSquare/robotframework-tidy
   rev: "4.11.0"
@@ -55,10 +55,11 @@ repos:
     exclude: |
       /tests/
 
-- repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: 'v0.3.4'
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: 'v0.3.5'
   hooks:
   - id: ruff
+  - id: ruff-format
 
 - repo: https://github.com/asottile/pyupgrade
   rev: v3.15.2
@@ -72,11 +73,6 @@ repos:
   - id: django-upgrade
     args: [--target-version, "4.2"]
 
-- repo: https://github.com/psf/black
-  rev: "24.3.0"
-  hooks:
-  - id: black
-
 - repo: https://github.com/pre-commit/mirrors-mypy
   rev: v1.9.0
   hooks:
@@ -145,7 +141,7 @@ repos:
     exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
 - repo: https://github.com/thibaudcolas/pre-commit-stylelint
-  rev: v16.3.0
+  rev: v16.3.1
   hooks:
     - id: stylelint
       args: [ --fix ]

Makefile

@@ -113,7 +113,9 @@ docs:
 	$(call build-settings-doc,boefjes,boefjes.config,boefjes,Boefjes)
 	$(call build-settings-doc,bytes,bytes.config,bytes,Bytes)
 	$(call build-settings-doc,mula/scheduler,config.settings,mula,Mula)
-	sphinx-build -b html docs/source docs/_build
+
+	PYTHONPATH=$(PYTHONPATH):boefjes/:bytes/:keiko/:mula/:octopoes/ sphinx-build -b html docs/source docs/_build
+
 
 poetry-dependencies:
 	files=$$(find . -name pyproject.toml -maxdepth 2); \

boefjes/.ci/docker-compose.yml

@@ -14,6 +14,6 @@ services:
     - .:/app/boefjes
 
   ci_katalogus-db:
-    image: postgres:15
+    image: docker.io/library/postgres:15
     env_file:
       - .ci/.env.test

boefjes/Dockerfile

@@ -18,8 +18,14 @@ COPY boefjes/requirements-dev.txt boefjes/requirements.txt .
 
 RUN --mount=type=cache,target=/root/.cache \
     pip install --upgrade pip \
-    && pip install -r requirements.txt \
-    && if [ "$ENVIRONMENT" = "dev" ]; then pip install -r requirements-dev.txt; fi
+    && if [ "$ENVIRONMENT" = "dev" ]; \
+    then \
+    grep -v git+https:// requirements-dev.txt | pip install -r /dev/stdin ; \
+    grep git+https:// requirements-dev.txt | pip install -r /dev/stdin ; \
+    else \
+    grep -v git+https:// requirements.txt | pip install -r /dev/stdin ;\
+    grep git+https:// requirements.txt | pip install -r /dev/stdin ; \
+    fi
 
 FROM dev
 

boefjes/Makefile

@@ -6,7 +6,7 @@ MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 # Makefile Reference: https://tech.davis-hansson.com/p/make/
 
-.PHONY: help sql migrate migrations debian ubuntu clean
+.PHONY: help sql migrate migrations debian ubuntu clean images
 
 # use HIDE to run commands invisibly, unless VERBOSE defined
 HIDE:=$(if $(VERBOSE),,@)
@@ -16,6 +16,10 @@ export revid	# Revision id to generate raw sql for
 export rev1		# Previous revision id for generating migrations
 export rev2		# New revision id for the new migration file
 
+# We set this to build images with the right target platform in the `images` target. This prevents arm systems such as
+# systems with an Apple silicon chip to build images they cannot use.
+export DOCKER_DEFAULT_PLATFORM=$(shell docker system info --format '{{.OSType}}/{{.Architecture}}')
+
 ##
 ##|------------------------------------------------------------------------|
 ##			Help
@@ -28,11 +32,19 @@ help: ## Show this help.
 ##|------------------------------------------------------------------------|
 ##			Development
 ##|------------------------------------------------------------------------|
-build: seed
+
+build: seed images
 
 seed:  # Seed the katalogus database
 	-docker compose run --rm katalogus python -m boefjes.seed
 
+images:  # Build the images for the containerized boefjes
+  # Dns-records is disabled for now, see the discussion in https://github.com/minvws/nl-kat-coordination/pull/2709
+	# docker build -f images/base.Dockerfile -t openkat/dns-records --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dns .
+	docker build -f ./boefjes/plugins/kat_dnssec/boefje.Dockerfile -t openkat/dns-sec .
+	docker build -f ./boefjes/plugins/kat_nmap_tcp/boefje.Dockerfile -t openkat/nmap  .
+
+
 ##
 ##|------------------------------------------------------------------------|
 ##			Migrations

boefjes/boefjes/api.py

@@ -73,7 +73,7 @@ def get_scheduler_client():
 
 def get_bytes_client():
     return BytesAPIClient(
-        settings.bytes_api,
+        str(settings.bytes_api),
         username=settings.bytes_username,
         password=settings.bytes_password,
     )
@@ -156,7 +156,8 @@ def create_boefje_meta(task, local_repository):
 
     organization = task.p_item.data.organization
     input_ooi = task.p_item.data.input_ooi
-    arguments = {}
+    arguments = {"oci_arguments": boefje_resource.oci_arguments}
+
     if input_ooi:
         reference = Reference.from_str(input_ooi)
         try:

boefjes/boefjes/clients/bytes_client.py

@@ -5,7 +5,7 @@
 from typing import Any
 from uuid import UUID
 
-from httpx import Client, HTTPError, HTTPStatusError, HTTPTransport, Response
+from httpx import Client, HTTPStatusError, HTTPTransport, Response
 
 from boefjes.job_models import BoefjeMeta, NormalizerMeta, RawDataMeta
 
@@ -51,7 +51,7 @@ def login(self) -> None:
     def _verify_response(response: Response) -> None:
         try:
             response.raise_for_status()
-        except HTTPError as error:
+        except HTTPStatusError as error:
             if error.response.status_code != 401:
                 logger.error(response.text)
             else:

boefjes/boefjes/docker_boefjes_runner.py

@@ -51,18 +51,20 @@ def run(self) -> None:
                 network=settings.docker_network,
             )
 
-            # save container log (stderr) to bytes
-            self.bytes_api_client.login()
-            self.bytes_api_client.save_raw(task_id, container_logs, stderr_mime_types)
+            task = self.scheduler_client.get_task(task_id)
 
             # if status is still running the container didn't call the output API endpoint, so set to status to failed
-            task = self.scheduler_client.get_task(task_id)
             if task.status == TaskStatus.RUNNING:
+                self.boefje_meta.ended_at = datetime.now(timezone.utc)
+                self.bytes_api_client.save_boefje_meta(self.boefje_meta)  # The task didn't create a boefje_meta object
+                self.bytes_api_client.save_raw(task_id, container_logs, stderr_mime_types.union({"error/boefje"}))
                 self.scheduler_client.patch_task(task_id, TaskStatus.FAILED)
+
                 # have to raise exception to prevent _start_working function from setting status to completed
                 raise RuntimeError("Boefje did not call output API endpoint")
         except ContainerError as e:
             logger.exception("Container error")
+
             # save container log (stderr) to bytes
             self.bytes_api_client.login()
             self.boefje_meta.ended_at = datetime.now(timezone.utc)

boefjes/boefjes/job_handler.py

@@ -167,11 +167,11 @@ def handle(self, boefje_meta: BoefjeMeta) -> None:
                         else:
                             valid_mimetypes.add(mimetype)
                     raw_file_id = self.bytes_client.save_raw(boefje_meta.id, output, mime_types.union(valid_mimetypes))
-                    logger.debug(
+                    logger.info(
                         "Saved raw file %s for boefje %s[%s]", raw_file_id, boefje_meta.boefje.id, boefje_meta.id
                     )
-
-            logger.info("Done with boefje for %s[%s]", boefje_meta.boefje.id, str(boefje_meta.id))
+            else:
+                logger.info("No results for boefje %s[%s]", boefje_meta.boefje.id, boefje_meta.id)
 
 
 class NormalizerHandler(Handler):

boefjes/boefjes/katalogus/local_repository.py

@@ -8,7 +8,6 @@
 from boefjes.plugins.models import (
     BOEFJE_DEFINITION_FILE,
     BOEFJES_DIR,
-    ENTRYPOINT_BOEFJES,
     ENTRYPOINT_NORMALIZERS,
     NORMALIZER_DEFINITION_FILE,
     BoefjeResource,
@@ -62,20 +61,23 @@ def schema(self, id_: str) -> dict | None:
 
     def cover_path(self, id_: str) -> Path:
         boefjes = self.resolve_boefjes()
+        normalizers = self.resolve_normalizers()
+        default_cover_path = self.default_cover_path()
 
-        if id_ not in boefjes:
-            return self.default_cover_path()
-
-        boefje = boefjes[id_]
-        path = boefje.path / "cover.jpg"
-
-        if not path.exists():
-            logger.debug("Did not find cover for boefje %s", boefje)
-            return self.default_cover_path()
+        if id_ in boefjes:
+            plugin = boefjes[id_]
+            cover_path = plugin.path / "cover.jpg"
+        elif id_ in normalizers:
+            plugin = normalizers[id_]
+            cover_path = plugin.path / "normalizer_cover.jpg"
+        else:
+            cover_path = default_cover_path
 
-        logger.debug("Found cover for boefje %s", boefje)
+        if not cover_path.exists():
+            logger.debug("Did not find cover for plugin %s", plugin)
+            return default_cover_path
 
-        return path
+        return cover_path
 
     def default_cover_path(self) -> Path:
         return self.path / "default_cover.jpg"
@@ -92,7 +94,7 @@ def resolve_boefjes(self) -> dict[str, BoefjeResource]:
         if self._cached_boefjes:
             return self._cached_boefjes
 
-        paths_and_packages = self._find_packages_in_path_containing_files([BOEFJE_DEFINITION_FILE, ENTRYPOINT_BOEFJES])
+        paths_and_packages = self._find_packages_in_path_containing_files([BOEFJE_DEFINITION_FILE])
         boefje_resources = []
 
         for path, package in paths_and_packages:
@@ -124,7 +126,7 @@ def resolve_normalizers(self) -> dict[str, NormalizerResource]:
 
         return self._cached_normalizers
 
-    def _find_packages_in_path_containing_files(self, files: list[str]) -> list[tuple[Path, str]]:
+    def _find_packages_in_path_containing_files(self, required_files: list[str]) -> list[tuple[Path, str]]:
         prefix = self.create_relative_import_statement_from_cwd(self.path)
         paths = []
 
@@ -134,10 +136,10 @@ def _find_packages_in_path_containing_files(self, files: list[str]) -> list[tupl
                 continue
 
             path = self.path / package.name.replace(prefix, "").replace(".", "/")
-            not_present_files = [file for file in files if not (path / file).exists()]
+            missing_files = [file for file in required_files if not (path / file).exists()]
 
-            if not_present_files:
-                logging.debug("Files %s not found for %s", not_present_files, package.name)
+            if missing_files:
+                logging.debug("Files %s not found for %s", missing_files, package.name)
                 continue
 
             paths.append((path, package.name))

boefjes/boefjes/katalogus/models.py

@@ -42,6 +42,7 @@ class Boefje(Plugin):
     options: list[str] | None = None
     runnable_hash: str | None = None
     oci_image: str | None = None
+    oci_arguments: list[str] = Field(default_factory=list)
 
 
 class Normalizer(Plugin):

boefjes/boefjes/katalogus/tests/boefjes_test_dir/kat_test/kat_test_4/boefje.json

@@ -0,0 +1,13 @@
+{
+  "id": "kat_test_4",
+  "name": "Kat test 4 name",
+  "description": "Testing KAT",
+  "consumes": [
+    "DNSZone"
+  ],
+  "produces": [
+    "text/html"
+  ],
+  "environment_keys": [],
+  "scan_level": 1
+}

boefjes/boefjes/katalogus/tests/boefjes_test_dir/kat_test/kat_test_4/schema.json

@@ -0,0 +1,19 @@
+{
+  "title": "Arguments",
+  "type": "object",
+  "properties": {
+    "api_key": {
+      "title": "Api Key",
+      "maxLength": 128,
+      "type": "integer"
+    },
+    "optional_key": {
+      "title": "Optional Key",
+      "maxLength": 128,
+      "type": "string"
+    }
+  },
+  "required": [
+    "api_key"
+  ]
+}