Feature/boefje normalizer config models (#3118)

Signed-off-by: Donny Peeters <[email protected]>
Co-authored-by: stephanie0x00 <[email protected]>
Co-authored-by: Jan Klopper <[email protected]>

boefjes/.ci/docker-compose.yml

@@ -5,7 +5,7 @@ services:
       dockerfile: boefjes/Dockerfile
       args:
         - ENVIRONMENT=dev
-    command: sh -c 'python -m pytest -v boefjes/katalogus/tests/integration'
+    command: sh -c 'python -m pytest -v tests/integration'
     depends_on:
       - ci_katalogus-db
     env_file:

boefjes/boefjes/api.py

@@ -15,7 +15,7 @@
 from boefjes.config import settings
 from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector, serialize_ooi
 from boefjes.job_models import BoefjeMeta
-from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
+from boefjes.local_repository import LocalPluginRepository, get_local_repository
 from boefjes.plugins.models import _default_mime_types
 from octopoes.models import Reference
 from octopoes.models.exception import ObjectNotFoundException

boefjes/boefjes/app.py

@@ -17,8 +17,8 @@
 )
 from boefjes.config import Settings
 from boefjes.job_handler import BoefjeHandler, NormalizerHandler, bytes_api_client
-from boefjes.katalogus.local_repository import get_local_repository
 from boefjes.local import LocalBoefjeJobRunner, LocalNormalizerJobRunner
+from boefjes.local_repository import get_local_repository
 from boefjes.runtime_interfaces import Handler, WorkerManager
 
 logger = logging.getLogger(__name__)

boefjes/boefjes/config.py

@@ -7,7 +7,7 @@
 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
 from pydantic_settings.sources import EnvSettingsSource
 
-from boefjes.katalogus.models import EncryptionMiddleware
+from boefjes.models import EncryptionMiddleware
 
 BASE_DIR: Path = Path(__file__).parent.resolve()
 

boefjes/boefjes/katalogus/dependencies/plugins.py → boefjes/boefjes/dependencies/plugins.py

@@ -9,20 +9,19 @@
 from jsonschema.validators import validate
 from sqlalchemy.orm import Session
 
-from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
-from boefjes.katalogus.models import Boefje, FilterParameters, Normalizer, PaginationParameters, PluginType
-from boefjes.katalogus.storage.interfaces import (
+from boefjes.local_repository import LocalPluginRepository, get_local_repository
+from boefjes.models import Boefje, FilterParameters, Normalizer, PaginationParameters, PluginType
+from boefjes.sql.config_storage import create_config_storage
+from boefjes.sql.db import session_managed_iterator
+from boefjes.sql.plugin_storage import create_plugin_storage
+from boefjes.storage.interfaces import (
+    ConfigStorage,
     ExistingPluginId,
     NotFound,
-    PluginEnabledStorage,
+    PluginNotFound,
     PluginStorage,
     SettingsNotConformingToSchema,
-    SettingsStorage,
 )
-from boefjes.sql.db import session_managed_iterator
-from boefjes.sql.plugin_enabled_storage import create_plugin_enabled_storage
-from boefjes.sql.plugin_storage import create_plugin_storage
-from boefjes.sql.setting_storage import create_setting_storage
 
 logger = logging.getLogger(__name__)
 
@@ -31,33 +30,29 @@ class PluginService:
     def __init__(
         self,
         plugin_storage: PluginStorage,
-        plugin_enabled_store: PluginEnabledStorage,
-        settings_storage: SettingsStorage,
+        config_storage: ConfigStorage,
         local_repo: LocalPluginRepository,
     ):
         self.plugin_storage = plugin_storage
-        self.plugin_enabled_store = plugin_enabled_store
-        self.settings_storage = settings_storage
+        self.config_storage = config_storage
         self.local_repo = local_repo
 
     def __enter__(self):
-        self.plugin_enabled_store.__enter__()
         self.plugin_storage.__enter__()
-        self.settings_storage.__enter__()
+        self.config_storage.__enter__()
 
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.plugin_enabled_store.__exit__(exc_type, exc_val, exc_tb)
         self.plugin_storage.__exit__(exc_type, exc_val, exc_tb)
-        self.settings_storage.__exit__(exc_type, exc_val, exc_tb)
+        self.config_storage.__exit__(exc_type, exc_val, exc_tb)
 
     def get_all(self, organisation_id: str) -> list[PluginType]:
-        all_plugins = self.get_all_without_enabled()
+        all_plugins = self._get_all_without_enabled()
 
         return [self._set_plugin_enabled(plugin, organisation_id) for plugin in all_plugins.values()]
 
-    def get_all_without_enabled(self):
+    def _get_all_without_enabled(self):
         all_plugins = {plugin.id: plugin for plugin in self.local_repo.get_all()}
 
         for plugin in self.plugin_storage.get_all():
@@ -88,24 +83,25 @@ def by_plugin_ids(self, plugin_ids: list[str], organisation_id: str) -> list[Plu
         return found_plugins
 
     def get_all_settings(self, organisation_id: str, plugin_id: str):
-        return self.settings_storage.get_all(organisation_id, plugin_id)
+        return self.config_storage.get_all_settings(organisation_id, plugin_id)
 
     def clone_settings_to_organisation(self, from_organisation: str, to_organisation: str):
         # One requirement is that only boefjes enabled in the from_organisation end up being enabled for the target.
-        for plugin_id in self.plugin_enabled_store.get_all_enabled(to_organisation):
+        for plugin_id in self.config_storage.get_enabled_boefjes(to_organisation):
             self.set_enabled_by_id(plugin_id, to_organisation, enabled=False)
 
         for plugin in self.get_all(from_organisation):
             if all_settings := self.get_all_settings(from_organisation, plugin.id):
                 self.upsert_settings(all_settings, to_organisation, plugin.id)
 
-        for plugin_id in self.plugin_enabled_store.get_all_enabled(from_organisation):
+        for plugin_id in self.config_storage.get_enabled_boefjes(from_organisation):
             self.set_enabled_by_id(plugin_id, to_organisation, enabled=True)
 
-    def upsert_settings(self, values: dict, organisation_id: str, plugin_id: str):
-        self._assert_settings_match_schema(values, organisation_id, plugin_id)
+    def upsert_settings(self, settings: dict, organisation_id: str, plugin_id: str):
+        self._assert_settings_match_schema(settings, organisation_id, plugin_id)
+        self._put_boefje(plugin_id)
 
-        return self.settings_storage.upsert(values, organisation_id, plugin_id)
+        return self.config_storage.upsert(organisation_id, plugin_id, settings=settings)
 
     def create_boefje(self, boefje: Boefje) -> None:
         try:
@@ -121,8 +117,38 @@ def create_normalizer(self, normalizer: Normalizer) -> None:
         except KeyError:
             self.plugin_storage.create_normalizer(normalizer)
 
+    def _put_boefje(self, boefje_id: str) -> None:
+        """Check existence of a boefje, and insert a database entry if it concerns a local boefje"""
+
+        try:
+            self.plugin_storage.boefje_by_id(boefje_id)
+        except PluginNotFound:
+            try:
+                plugin = self.local_repo.by_id(boefje_id)
+            except KeyError:
+                raise
+
+            if plugin.type != "boefje":
+                raise
+            self.plugin_storage.create_boefje(plugin)
+
+    def _put_normalizer(self, normalizer_id: str) -> None:
+        """Check existence of a normalizer, and insert a database entry if it concerns a local normalizer"""
+
+        try:
+            self.plugin_storage.normalizer_by_id(normalizer_id)
+        except PluginNotFound:
+            try:
+                plugin = self.local_repo.by_id(normalizer_id)
+            except KeyError:
+                raise
+
+            if plugin.type != "normalizer":
+                raise
+            self.plugin_storage.create_normalizer(plugin)
+
     def delete_settings(self, organisation_id: str, plugin_id: str):
-        self.settings_storage.delete(organisation_id, plugin_id)
+        self.config_storage.delete(organisation_id, plugin_id)
 
         try:
             self._assert_settings_match_schema({}, organisation_id, plugin_id)
@@ -154,14 +180,15 @@ def description(self, plugin_id: str, organisation_id: str) -> str:
 
     def set_enabled_by_id(self, plugin_id: str, organisation_id: str, enabled: bool):
         if enabled:
-            all_settings = self.settings_storage.get_all(organisation_id, plugin_id)
+            all_settings = self.get_all_settings(organisation_id, plugin_id)
             self._assert_settings_match_schema(all_settings, organisation_id, plugin_id)
 
-        self.plugin_enabled_store.update_or_create_by_id(
-            plugin_id,
-            enabled,
-            organisation_id,
-        )
+        try:
+            self._put_boefje(plugin_id)
+        except PluginNotFound:
+            self._put_normalizer(plugin_id)
+
+        self.config_storage.upsert(organisation_id, plugin_id, enabled=enabled)
 
     def _assert_settings_match_schema(self, all_settings: dict, organisation_id: str, plugin_id: str):
         schema = self.schema(plugin_id)
@@ -174,7 +201,7 @@ def _assert_settings_match_schema(self, all_settings: dict, organisation_id: str
 
     def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> PluginType:
         with contextlib.suppress(KeyError, NotFound):
-            plugin.enabled = self.plugin_enabled_store.get_by_id(plugin.id, organisation_id)
+            plugin.enabled = self.config_storage.is_enabled_by_id(plugin.id, organisation_id)
 
         return plugin
 
@@ -183,8 +210,7 @@ def get_plugin_service(organisation_id: str) -> Iterator[PluginService]:
     def closure(session: Session):
         return PluginService(
             create_plugin_storage(session),
-            create_plugin_enabled_storage(session),
-            create_setting_storage(session),
+            create_config_storage(session),
             get_local_repository(),
         )
 

boefjes/boefjes/docker_boefjes_runner.py

@@ -9,7 +9,7 @@
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
 from boefjes.job_models import BoefjeMeta
-from boefjes.katalogus.models import Boefje
+from boefjes.models import Boefje
 
 logger = logging.getLogger(__name__)
 

boefjes/boefjes/job_handler.py

@@ -13,7 +13,7 @@
 from boefjes.config import settings
 from boefjes.docker_boefjes_runner import DockerBoefjesRunner
 from boefjes.job_models import BoefjeMeta, NormalizerMeta, SerializedOOI, SerializedOOIValue
-from boefjes.katalogus.local_repository import LocalPluginRepository
+from boefjes.local_repository import LocalPluginRepository
 from boefjes.plugins.models import _default_mime_types
 from boefjes.runtime_interfaces import BoefjeJobRunner, Handler, NormalizerJobRunner
 from octopoes.api.models import Affirmation, Declaration, Observation

boefjes/boefjes/katalogus/api/organisations.py → boefjes/boefjes/katalogus/organisations.py

@@ -1,9 +1,9 @@
 from fastapi import APIRouter, Depends, HTTPException, status
 
-from boefjes.katalogus.dependencies.organisations import get_organisations_store
-from boefjes.katalogus.models import Organisation
-from boefjes.katalogus.storage.interfaces import OrganisationNotFound, OrganisationStorage
+from boefjes.models import Organisation
 from boefjes.sql.db import ObjectNotFoundException
+from boefjes.sql.organisation_storage import get_organisations_store
+from boefjes.storage.interfaces import OrganisationNotFound, OrganisationStorage
 
 router = APIRouter(prefix="/organisations", tags=["organisations"])
 

boefjes/boefjes/katalogus/api/plugins.py → boefjes/boefjes/katalogus/plugins.py

@@ -5,16 +5,16 @@
 from fastapi.responses import FileResponse, JSONResponse, Response
 from pydantic import BaseModel, Field
 
-from boefjes.katalogus.api.organisations import check_organisation_exists
-from boefjes.katalogus.dependencies.plugins import (
+from boefjes.dependencies.plugins import (
     PluginService,
     get_pagination_parameters,
     get_plugin_service,
     get_plugins_filter_parameters,
 )
-from boefjes.katalogus.models import FilterParameters, PaginationParameters, PluginType
-from boefjes.katalogus.storage.interfaces import PluginStorage
+from boefjes.katalogus.organisations import check_organisation_exists
+from boefjes.models import FilterParameters, PaginationParameters, PluginType
 from boefjes.sql.plugin_storage import get_plugin_storage
+from boefjes.storage.interfaces import PluginStorage
 
 router = APIRouter(
     prefix="/organisations/{organisation_id}",

boefjes/boefjes/katalogus/api/root.py → boefjes/boefjes/katalogus/root.py

@@ -15,10 +15,10 @@
 from pydantic import BaseModel, Field
 
 from boefjes.config import settings
-from boefjes.katalogus.api import organisations, plugins
-from boefjes.katalogus.api import settings as settings_router
-from boefjes.katalogus.storage.interfaces import NotFound, StorageError
+from boefjes.katalogus import organisations, plugins
+from boefjes.katalogus import settings as settings_router
 from boefjes.katalogus.version import __version__
+from boefjes.storage.interfaces import NotAllowed, NotFound, StorageError
 
 with settings.log_cfg.open() as f:
     logging.config.dictConfig(json.load(f))
@@ -59,6 +59,14 @@ def entity_not_found_handler(request: Request, exc: NotFound):
     )
 
 
+@app.exception_handler(NotAllowed)
+def not_allowed_handler(request: Request, exc: NotAllowed):
+    return JSONResponse(
+        status_code=status.HTTP_400_BAD_REQUEST,
+        content={"message": exc.message},
+    )
+
+
 @app.exception_handler(StorageError)
 def storage_error_handler(request: Request, exc: StorageError):
     return JSONResponse(

boefjes/boefjes/katalogus/api/settings.py → boefjes/boefjes/katalogus/settings.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, Depends
 
-from boefjes.katalogus.api.organisations import check_organisation_exists
-from boefjes.katalogus.dependencies.plugins import PluginService, get_plugin_service
+from boefjes.dependencies.plugins import PluginService, get_plugin_service
+from boefjes.katalogus.organisations import check_organisation_exists
 
 router = APIRouter(
     prefix="/organisations/{organisation_id}/{plugin_id}/settings",