From 5fef8d3a7d0b1968140ee68c6a23487a12f4787c Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 22 Mar 2022 12:02:49 +0100 Subject: [PATCH 01/43] test: add tests --- karp/tests/e2e/test_resources_api.py | 17 +++++++++++++++++ karp/webapp/routes/resources_api.py | 2 +- karp/webapp/tasks.py | 7 +++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/karp/tests/e2e/test_resources_api.py b/karp/tests/e2e/test_resources_api.py index 1b1f8988..387f544e 100644 --- a/karp/tests/e2e/test_resources_api.py +++ b/karp/tests/e2e/test_resources_api.py @@ -22,6 +22,10 @@ def new_resource() -> ResourceCreate: class TestResourcesRoutes: + def test_get_resource_exist(self, fa_data_client): + response = fa_data_client.get('/resources/places') + assert response.status_code != status.HTTP_404_NOT_FOUND + def test_get_resources_exist(self, fa_data_client): response = fa_data_client.get('/resources') assert response.status_code != status.HTTP_404_NOT_FOUND @@ -34,6 +38,8 @@ def test_get_resource_permissionss_exist(self, fa_data_client): response = fa_data_client.get('/resources/permissions') assert response.status_code != status.HTTP_404_NOT_FOUND + +class TestGetResourcePermissions: def test_get_resources(self, fa_data_client): response = fa_data_client.get('/resources/permissions') @@ -99,3 +105,14 @@ def test_get_resource_by_resource_id(self, fa_data_client): resource = ResourcePublic(**response.json()) assert resource.resource_id == 'test_resource' + +class TestGetResources: + def test_get_resources(self, fa_data_client): + response = fa_data_client.get('/resources/') + response_data = response.json() + print(f'{response_data=}') + assert response.status_code == status.HTTP_200_OK + + assert len(response_data) >= 3 + for resource_dict in response_data: + ResourcePublic(**resource_dict) diff --git a/karp/webapp/routes/resources_api.py b/karp/webapp/routes/resources_api.py index bbceb697..51cdccb4 100644 --- a/karp/webapp/routes/resources_api.py +++ b/karp/webapp/routes/resources_api.py @@ -39,7 +39,7 @@ def list_resource_permissions( def get_all_resources( get_resources: lex.GetResources = Depends( deps.inject_from_req(lex.GetResources)), -) -> list[dict]: +) -> list[lex.ResourceDto]: return get_resources.query() diff --git a/karp/webapp/tasks.py b/karp/webapp/tasks.py index deed6445..5d739c01 100644 --- a/karp/webapp/tasks.py +++ b/karp/webapp/tasks.py @@ -1,17 +1,24 @@ +import logging from typing import Callable from fastapi import FastAPI from karp.db_infrastructure import tasks as db_tasks from karp.main import config + +logger = logging.getLogger(__name__) + + def create_start_app_handler(app: FastAPI) -> Callable: async def start_app() -> None: + logger.debug('start_app') app.state.db = db_tasks.connect_to_db(config.DATABASE_URL) return start_app def create_stop_app_handler(app: FastAPI) -> Callable: async def stop_app() -> None: + logger.debug('stop_app') db_tasks.close_db_connection(app.state.db) app.state.app_context = None return stop_app From 338df62fc46bbfbcd1b952543a3518ffc2b83340 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 22:21:04 +0100 Subject: [PATCH 02/43] test: add test for sql_entry_uow_creator --- .../tests/integration/test_sql_entries_uow.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 karp/tests/integration/test_sql_entries_uow.py diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py new file mode 100644 index 00000000..3fb509fe --- /dev/null +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -0,0 +1,40 @@ +from unittest import mock + +import pytest + +from karp.foundation.events import EventBus +from karp import lex +from karp.lex_infrastructure import SqlEntryUowCreator +from karp.tests.unit.lex import factories + + +@pytest.fixture +def example_uow() -> lex.CreateEntryRepository: + return factories.CreateEntryRepositoryFactory() + + +@pytest.fixture +def sql_entry_uow_v1_creator(sqlite_session_factory) -> SqlEntryUowCreator: + return SqlEntryUowCreator( + event_bus=mock.Mock(spec=EventBus), + session_factory=sqlite_session_factory, + ) + + +class TestSqlEntryUowV1: + def test_creator_repository_type( + self, + sql_entry_uow_v1_creator: SqlEntryUowCreator, + ): + assert sql_entry_uow_v1_creator.repository_type == 'sql_entries_v1' + + def test_uow_repository_type( + self, + sql_entry_uow_v1_creator: SqlEntryUowCreator, + example_uow: lex.CreateEntryRepository, + ): + entry_uow = sql_entry_uow_v1_creator( + **example_uow.dict(exclude={'repository_type'}) + ) + assert entry_uow.repository_type == 'sql_entries_v1' + From 63cce1d9b97f25c23153c4e320ca417a09e85d9a Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 22:24:40 +0100 Subject: [PATCH 03/43] test: add test for tablename --- karp/tests/integration/test_sql_entries_uow.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index 3fb509fe..2dc5853a 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -38,3 +38,14 @@ def test_uow_repository_type( ) assert entry_uow.repository_type == 'sql_entries_v1' + def test_repo_table_name( + self, + sql_entry_uow_v1_creator: SqlEntryUowCreator, + example_uow: lex.CreateEntryRepository, + ): + entry_uow = sql_entry_uow_v1_creator( + **example_uow.dict(exclude={'repository_type'}) + ) + with entry_uow as uw: + assert uw.repo.history_model.__tablename__ == example_uow.name + From 3a499e8ec4ccf669da83d2c49610c78a6bbd2c5f Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 22:28:08 +0100 Subject: [PATCH 04/43] test: add test for tablename --- .../tests/integration/test_sql_entries_uow.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index 2dc5853a..93ea3a7d 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -49,3 +49,31 @@ def test_repo_table_name( with entry_uow as uw: assert uw.repo.history_model.__tablename__ == example_uow.name + +class TestSqlEntryUowV2: + def test_creator_repository_type( + self, + sql_entry_uow_v2_creator: SqlEntryUowCreator, + ): + assert sql_entry_uow_v2_creator.repository_type == 'sql_entries_v2' + + def test_uow_repository_type( + self, + sql_entry_uow_v2_creator: SqlEntryUowCreator, + example_uow: lex.CreateEntryRepository, + ): + entry_uow = sql_entry_uow_v2_creator( + **example_uow.dict(exclude={'repository_type'}) + ) + assert entry_uow.repository_type == 'sql_entries_v2' + + def test_repo_table_name( + self, + sql_entry_uow_v2_creator: SqlEntryUowCreator, + example_uow: lex.CreateEntryRepository, + ): + entry_uow = sql_entry_uow_v2_creator( + **example_uow.dict(exclude={'repository_type'}) + ) + with entry_uow as uw: + assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{str(example_uow.entity_id)} From e427fc3a644a6d4db7735e9b59266194bbd459f8 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 22:31:38 +0100 Subject: [PATCH 05/43] test: add test for sql_entries_v2 --- karp/tests/integration/test_sql_entries_uow.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index 93ea3a7d..15f49ad0 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -4,7 +4,7 @@ from karp.foundation.events import EventBus from karp import lex -from karp.lex_infrastructure import SqlEntryUowCreator +from karp.lex_infrastructure import SqlEntryUowCreator, SqlEntryUowV2Creator from karp.tests.unit.lex import factories @@ -21,6 +21,14 @@ def sql_entry_uow_v1_creator(sqlite_session_factory) -> SqlEntryUowCreator: ) +@pytest.fixture +def sql_entry_uow_v2_creator(sqlite_session_factory) -> SqlEntryUowV2Creator: + return SqlEntryUowV2Creator( + event_bus=mock.Mock(spec=EventBus), + session_factory=sqlite_session_factory, + ) + + class TestSqlEntryUowV1: def test_creator_repository_type( self, @@ -76,4 +84,4 @@ def test_repo_table_name( **example_uow.dict(exclude={'repository_type'}) ) with entry_uow as uw: - assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{str(example_uow.entity_id)} + assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{str(example_uow.entity_id)}' From fdaa84e116e07aa7460129745fcc91fa7d47cbfc Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 23:29:05 +0100 Subject: [PATCH 06/43] fix: use entity_id in table_name in sql_entries_v2 --- karp/lex_infrastructure/__init__.py | 8 +-- .../repositories/__init__.py | 2 +- .../repositories/sql_entries.py | 49 ++++++++++++++++--- .../tests/integration/test_sql_entries_uow.py | 20 ++++---- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/karp/lex_infrastructure/__init__.py b/karp/lex_infrastructure/__init__.py index 302a2cd3..923ab69d 100644 --- a/karp/lex_infrastructure/__init__.py +++ b/karp/lex_infrastructure/__init__.py @@ -43,7 +43,8 @@ ) from karp.lex_infrastructure.repositories import ( SqlEntryUowRepositoryUnitOfWork, - SqlEntryUowCreator, + SqlEntryUowV1Creator, + SqlEntryUowV2Creator, SqlResourceUnitOfWork, ) @@ -102,8 +103,9 @@ def resources_uow( @injector.multiprovider def entry_uow_creator_map(self) -> Dict[str, EntryUnitOfWorkCreator]: return { - 'default': SqlEntryUowCreator, - SqlEntryUowCreator.repository_type: SqlEntryUowCreator, + 'default': SqlEntryUowV2Creator, + SqlEntryUowV1Creator.repository_type: SqlEntryUowV1Creator, + SqlEntryUowV2Creator.repository_type: SqlEntryUowV2Creator, } diff --git a/karp/lex_infrastructure/repositories/__init__.py b/karp/lex_infrastructure/repositories/__init__.py index 9fca5cfd..09d0d8c5 100644 --- a/karp/lex_infrastructure/repositories/__init__.py +++ b/karp/lex_infrastructure/repositories/__init__.py @@ -1,3 +1,3 @@ from .sql_entry_uows import SqlEntryUowRepository, SqlEntryUowRepositoryUnitOfWork -from .sql_entries import SqlEntryUowCreator +from .sql_entries import SqlEntryUowV1Creator, SqlEntryUowV2Creator from .sql_resources import SqlResourceRepository, SqlResourceUnitOfWork diff --git a/karp/lex_infrastructure/repositories/sql_entries.py b/karp/lex_infrastructure/repositories/sql_entries.py index 35fb9420..2f575d29 100644 --- a/karp/lex_infrastructure/repositories/sql_entries.py +++ b/karp/lex_infrastructure/repositories/sql_entries.py @@ -2,7 +2,7 @@ import inspect import logging import typing -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Generic, TypeVar from uuid import UUID import injector @@ -564,7 +564,7 @@ class SqlEntryUnitOfWork( SqlUnitOfWork, repositories.EntryUnitOfWork, ): - repository_type: str = 'sql_entries_v1' + repository_type: str = 'sql_entries_base' def __init__( self, @@ -585,12 +585,15 @@ def _begin(self): self._session = self.session_factory() if self._entries is None: self._entries = SqlEntryRepository.from_dict( - name=self.name, + name=self.table_name(), resource_config=self.config, session=self._session ) return self + def table_name(self) -> str: + return self.name + @property def repo(self) -> SqlEntryRepository: if self._entries is None: @@ -606,6 +609,18 @@ def collect_new_events(self) -> typing.Iterable: return super().collect_new_events() else: return [] + + +class SqlEntryUnitOfWorkV1(SqlEntryUnitOfWork): + repository_type: str = 'sql_entries_v1' + + +class SqlEntryUnitOfWorkV2(SqlEntryUnitOfWork): + repository_type: str = 'sql_entries_v2' + + def table_name(self) -> str: + return f"{self.name}_{self.entity_id.hex}" + # ===== Value objects ===== # class SqlEntryRepositorySettings(EntryRepositorySettings): # def __init__(self, *, table_name: str, config: Dict): @@ -623,9 +638,11 @@ def collect_new_events(self) -> typing.Iterable: # runtime_table_name, history_model, settings.config # ) # return SqlEntryRepository(history_model, runtime_model, settings.config) +SqlEntryUowType = TypeVar('SqlEntryUowType', bound=SqlEntryUnitOfWork) -class SqlEntryUowCreator: - repository_type: str = SqlEntryUnitOfWork.repository_type + +class SqlEntryUowCreator(Generic[SqlEntryUowType]): + repository_type: str = "repository_type" @injector.inject def __init__( @@ -646,9 +663,9 @@ def __call__( user: str, message: str, timestamp: float, - ) -> SqlEntryUnitOfWork: + ) -> SqlEntryUowType: if entity_id not in self.cache: - self.cache[entity_id] = SqlEntryUnitOfWork( + self.cache[entity_id] = self._create_uow( entity_id=entity_id, name=name, config=config, @@ -660,3 +677,21 @@ def __call__( event_bus=self.event_bus, ) return self.cache[entity_id] + + +class SqlEntryUowV1Creator( + SqlEntryUowCreator[SqlEntryUnitOfWorkV1] +): + repository_type: str = "sql_entries_v1" + + def _create_uow(self, **kwargs) -> SqlEntryUnitOfWorkV1: + return SqlEntryUnitOfWorkV1(**kwargs) + + +class SqlEntryUowV2Creator( + SqlEntryUowCreator[SqlEntryUnitOfWorkV2] +): + repository_type: str = "sql_entries_v2" + + def _create_uow(self, **kwargs) -> SqlEntryUnitOfWorkV2: + return SqlEntryUnitOfWorkV2(**kwargs) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index 15f49ad0..591141fd 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -4,7 +4,7 @@ from karp.foundation.events import EventBus from karp import lex -from karp.lex_infrastructure import SqlEntryUowCreator, SqlEntryUowV2Creator +from karp.lex_infrastructure import SqlEntryUowV1Creator, SqlEntryUowV2Creator from karp.tests.unit.lex import factories @@ -14,8 +14,8 @@ def example_uow() -> lex.CreateEntryRepository: @pytest.fixture -def sql_entry_uow_v1_creator(sqlite_session_factory) -> SqlEntryUowCreator: - return SqlEntryUowCreator( +def sql_entry_uow_v1_creator(sqlite_session_factory) -> SqlEntryUowV1Creator: + return SqlEntryUowV1Creator( event_bus=mock.Mock(spec=EventBus), session_factory=sqlite_session_factory, ) @@ -32,13 +32,13 @@ def sql_entry_uow_v2_creator(sqlite_session_factory) -> SqlEntryUowV2Creator: class TestSqlEntryUowV1: def test_creator_repository_type( self, - sql_entry_uow_v1_creator: SqlEntryUowCreator, + sql_entry_uow_v1_creator: SqlEntryUowV1Creator, ): assert sql_entry_uow_v1_creator.repository_type == 'sql_entries_v1' def test_uow_repository_type( self, - sql_entry_uow_v1_creator: SqlEntryUowCreator, + sql_entry_uow_v1_creator: SqlEntryUowV1Creator, example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v1_creator( @@ -48,7 +48,7 @@ def test_uow_repository_type( def test_repo_table_name( self, - sql_entry_uow_v1_creator: SqlEntryUowCreator, + sql_entry_uow_v1_creator: SqlEntryUowV1Creator, example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v1_creator( @@ -61,13 +61,13 @@ def test_repo_table_name( class TestSqlEntryUowV2: def test_creator_repository_type( self, - sql_entry_uow_v2_creator: SqlEntryUowCreator, + sql_entry_uow_v2_creator: SqlEntryUowV2Creator, ): assert sql_entry_uow_v2_creator.repository_type == 'sql_entries_v2' def test_uow_repository_type( self, - sql_entry_uow_v2_creator: SqlEntryUowCreator, + sql_entry_uow_v2_creator: SqlEntryUowV2Creator, example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v2_creator( @@ -77,11 +77,11 @@ def test_uow_repository_type( def test_repo_table_name( self, - sql_entry_uow_v2_creator: SqlEntryUowCreator, + sql_entry_uow_v2_creator: SqlEntryUowV2Creator, example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v2_creator( **example_uow.dict(exclude={'repository_type'}) ) with entry_uow as uw: - assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{str(example_uow.entity_id)}' + assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{example_uow.entity_id.hex}' From 6c7115603d76a76cf5c0aa2e9b15ef31647b2a11 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 28 Mar 2022 11:06:28 +0200 Subject: [PATCH 07/43] fix: shorten the table name --- karp/lex_infrastructure/repositories/sql_entries.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/karp/lex_infrastructure/repositories/sql_entries.py b/karp/lex_infrastructure/repositories/sql_entries.py index 2f575d29..0a46376b 100644 --- a/karp/lex_infrastructure/repositories/sql_entries.py +++ b/karp/lex_infrastructure/repositories/sql_entries.py @@ -1,16 +1,14 @@ """SQL repositories for entries.""" -import inspect import logging import typing -from typing import Dict, List, Optional, Tuple, Generic, TypeVar -from uuid import UUID +from typing import Dict, List, Optional, Generic, TypeVar import injector import regex import sqlalchemy as sa from sqlalchemy import sql from sqlalchemy.orm import sessionmaker -import logging +import ulid from karp.foundation.value_objects import UniqueId from karp.foundation.events import EventBus @@ -190,7 +188,7 @@ def _save(self, entry: Entry): { 'entry_by_entry_id': entry_by_entry_id, 'entry_by_entity_id': entry_by_entity_id, - 'entry': entry.dict(), + 'entry': entry.dict(), } ) raise RuntimeError(f'entry = {entry.dict()}') @@ -619,7 +617,9 @@ class SqlEntryUnitOfWorkV2(SqlEntryUnitOfWork): repository_type: str = 'sql_entries_v2' def table_name(self) -> str: - return f"{self.name}_{self.entity_id.hex}" + u = ulid.from_uuid(self.entity_id) + random_part = u.randomness().str + return f"{self.name}_{random_part}" # ===== Value objects ===== # class SqlEntryRepositorySettings(EntryRepositorySettings): From eaa062fe618be5cdeb6e685b963004a73a8b7c71 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 28 Mar 2022 11:48:10 +0200 Subject: [PATCH 08/43] test: fix sql_entries_v2 test --- karp/tests/integration/test_sql_entries_uow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index 591141fd..33dd96ed 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -1,6 +1,7 @@ from unittest import mock import pytest +import ulid from karp.foundation.events import EventBus from karp import lex @@ -83,5 +84,6 @@ def test_repo_table_name( entry_uow = sql_entry_uow_v2_creator( **example_uow.dict(exclude={'repository_type'}) ) + random_part = ulid.from_uuid(entry_uow.entity_id).randomness().str with entry_uow as uw: - assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{example_uow.entity_id.hex}' + assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{random_part}' From 1d5c02da4282e8e97ca04df40c8999691ad91b47 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 28 Mar 2022 11:24:05 +0000 Subject: [PATCH 09/43] =?UTF-8?q?=F0=9F=93=9D=20Update=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 122a0c10..5ae210ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Latest Changes +* Avoid name clashes in SqlEntryRepository. PR [#194](https://github.com/spraakbanken/karp-backend/pull/194) by [@kod-kristoff](https://github.com/kod-kristoff). * Fix correct fetching from repos. PR [#193](https://github.com/spraakbanken/karp-backend/pull/193) by [@kod-kristoff](https://github.com/kod-kristoff). ## 6.0.17 From 1ef394631caf2887b4f45eb8a2d1d5b718e5aa7a Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 13:46:53 +0100 Subject: [PATCH 10/43] feat: extend resource show output --- karp/cliapp/subapps/resource_subapp.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/karp/cliapp/subapps/resource_subapp.py b/karp/cliapp/subapps/resource_subapp.py index 557a8ab5..a5008699 100644 --- a/karp/cliapp/subapps/resource_subapp.py +++ b/karp/cliapp/subapps/resource_subapp.py @@ -222,14 +222,11 @@ def show( raise typer.Exit(3) typer.echo( - """ - Resource: {resource.resource_id} - EntityId: {resource.entity_id} - Version: {resource.version} - Discarded: {resource.discarded} - Config: {resource.config} - """.format( - resource=resource + tabulate( + ( + (key, value) + for key, value in resource.dict().items() + ) ) ) From 85bd349857debc7dcb2dab0f4eb3d0a59ad2b675 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 28 Mar 2022 09:04:30 +0200 Subject: [PATCH 11/43] stub: add delete entry-repo --- karp/cliapp/subapps/entry_repo_subapp.py | 18 ++++++++++++++++-- .../tests/unit/lex/test_entry_repo_handlers.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/karp/cliapp/subapps/entry_repo_subapp.py b/karp/cliapp/subapps/entry_repo_subapp.py index ead63096..ce5cde19 100644 --- a/karp/cliapp/subapps/entry_repo_subapp.py +++ b/karp/cliapp/subapps/entry_repo_subapp.py @@ -15,13 +15,11 @@ @subapp.command() def create(infile: typer.FileBinaryRead, ctx: typer.Context): - typer.echo(infile.name) try: data = json.load(infile) except Exception as err: typer.echo(f"Error reading file '{infile.name}': {str(err)}") raise typer.Exit(123) - typer.echo('after json.load') create_entry_repo = CreateEntryRepository.from_dict( data, user='local admin', @@ -39,6 +37,22 @@ def create(infile: typer.FileBinaryRead, ctx: typer.Context): ) +@subapp.command() +def delete( + entity_id: UniqueId, + ctx: typer.Context, + user: Optional[str] = typer.Option(None), +): + + bus = inject_from_ctx(CommandBus, ctx) + + delete_entry_repo = DeleteEntryRepo( + entity_id=entity_id, + user=user or "local admin" + ) + typer.echo(f"Entry repository with id '{entity_id}' deleted.") + + @subapp.command() def list(ctx: typer.Context): query = inject_from_ctx(ListEntryRepos, ctx) diff --git a/karp/tests/unit/lex/test_entry_repo_handlers.py b/karp/tests/unit/lex/test_entry_repo_handlers.py index 1ca4ba73..1c9671e5 100644 --- a/karp/tests/unit/lex/test_entry_repo_handlers.py +++ b/karp/tests/unit/lex/test_entry_repo_handlers.py @@ -18,3 +18,16 @@ def test_create_entry_repository( EntryUowRepositoryUnitOfWork) assert entry_uow_repo_uow.was_committed assert len(entry_uow_repo_uow.repo) == 1 + + +class TestDeleteEntryRepository: + def test_delete_entry_repository_succeeds( + self, + lex_ctx: adapters.UnitTestContext, + ): + cmd = factories.CreateEntryRepositoryFactory() + lex_ctx.command_bus.dispatch(cmd) + + entry_uow_repo_uow = lex_ctx.container.get( + EntryUowRepositoryUnitOfWork) + assert entry_uow_repo_uow.was_committed From 4467e3f49b5118704a741e73b3683304518333d6 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 22:21:04 +0100 Subject: [PATCH 12/43] test: add test for sql_entry_uow_creator --- .../tests/integration/test_sql_entries_uow.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index 33dd96ed..a6857d9f 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -35,7 +35,7 @@ def test_creator_repository_type( self, sql_entry_uow_v1_creator: SqlEntryUowV1Creator, ): - assert sql_entry_uow_v1_creator.repository_type == 'sql_entries_v1' + assert sql_entry_uow_v1_creator.repository_type == "sql_entries_v1" def test_uow_repository_type( self, @@ -43,9 +43,9 @@ def test_uow_repository_type( example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v1_creator( - **example_uow.dict(exclude={'repository_type'}) + **example_uow.dict(exclude={"repository_type"}) ) - assert entry_uow.repository_type == 'sql_entries_v1' + assert entry_uow.repository_type == "sql_entries_v1" def test_repo_table_name( self, @@ -53,7 +53,7 @@ def test_repo_table_name( example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v1_creator( - **example_uow.dict(exclude={'repository_type'}) + **example_uow.dict(exclude={"repository_type"}) ) with entry_uow as uw: assert uw.repo.history_model.__tablename__ == example_uow.name @@ -64,7 +64,7 @@ def test_creator_repository_type( self, sql_entry_uow_v2_creator: SqlEntryUowV2Creator, ): - assert sql_entry_uow_v2_creator.repository_type == 'sql_entries_v2' + assert sql_entry_uow_v2_creator.repository_type == "sql_entries_v2" def test_uow_repository_type( self, @@ -72,9 +72,9 @@ def test_uow_repository_type( example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v2_creator( - **example_uow.dict(exclude={'repository_type'}) + **example_uow.dict(exclude={"repository_type"}) ) - assert entry_uow.repository_type == 'sql_entries_v2' + assert entry_uow.repository_type == "sql_entries_v2" def test_repo_table_name( self, @@ -82,8 +82,11 @@ def test_repo_table_name( example_uow: lex.CreateEntryRepository, ): entry_uow = sql_entry_uow_v2_creator( - **example_uow.dict(exclude={'repository_type'}) + **example_uow.dict(exclude={"repository_type"}) ) random_part = ulid.from_uuid(entry_uow.entity_id).randomness().str with entry_uow as uw: - assert uw.repo.history_model.__tablename__ == f'{example_uow.name}_{random_part}' + assert ( + uw.repo.history_model.__tablename__ + == f"{example_uow.name}_{random_part}" + ) From 7c01c250ff0db1f076748d858d8412d82be555e2 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 22:31:38 +0100 Subject: [PATCH 13/43] test: add test for sql_entries_v2 --- karp/tests/integration/test_sql_entries_uow.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/karp/tests/integration/test_sql_entries_uow.py b/karp/tests/integration/test_sql_entries_uow.py index a6857d9f..93deccdc 100644 --- a/karp/tests/integration/test_sql_entries_uow.py +++ b/karp/tests/integration/test_sql_entries_uow.py @@ -30,6 +30,14 @@ def sql_entry_uow_v2_creator(sqlite_session_factory) -> SqlEntryUowV2Creator: ) +@pytest.fixture +def sql_entry_uow_v2_creator(sqlite_session_factory) -> SqlEntryUowV2Creator: + return SqlEntryUowV2Creator( + event_bus=mock.Mock(spec=EventBus), + session_factory=sqlite_session_factory, + ) + + class TestSqlEntryUowV1: def test_creator_repository_type( self, From 89c4975e8a450ea35abdcfb7c6b109e841e17547 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 25 Mar 2022 23:29:05 +0100 Subject: [PATCH 14/43] fix: use entity_id in table_name in sql_entries_v2 --- .../repositories/sql_entries.py | 188 +++++++++--------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/karp/lex_infrastructure/repositories/sql_entries.py b/karp/lex_infrastructure/repositories/sql_entries.py index 0a46376b..1dbcb6a8 100644 --- a/karp/lex_infrastructure/repositories/sql_entries.py +++ b/karp/lex_infrastructure/repositories/sql_entries.py @@ -14,9 +14,13 @@ from karp.foundation.events import EventBus from karp.lex.domain import errors from karp.lex.application import repositories + # from karp.domain.errors import NonExistingField, RepositoryError from karp.lex.domain.entities.entry import ( # EntryRepositorySettings,; EntryRepository,; create_entry_repository, - Entry, EntryOp, EntryStatus) + Entry, + EntryOp, + EntryStatus, +) from karp.db_infrastructure import db from karp.lex_infrastructure.sql import sql_models from karp.db_infrastructure.sql_repository import SqlRepository @@ -29,9 +33,7 @@ NO_PROPERTY_PATTERN = regex.compile(r"has no property '(\w+)'") -class SqlEntryRepository( - repositories.EntryRepository, SqlRepository -): +class SqlEntryRepository(repositories.EntryRepository, SqlRepository): def __init__( self, history_model, @@ -72,9 +74,8 @@ def from_dict( # history_model = create_history_entry_table(table_name) # history_model.create(bind=db.engine, checkfirst=True) - logger.warning({'table_name': table_name}) - history_model = sql_models.get_or_create_entry_history_model( - table_name) + logger.warning({"table_name": table_name}) + history_model = sql_models.get_or_create_entry_history_model(table_name) # runtime_table = db.get_table(runtime_table_name) # if runtime_table is None: @@ -91,8 +92,7 @@ def from_dict( if session: runtime_model.__table__.create(bind=session.bind, checkfirst=True) for child_model in runtime_model.child_tables.values(): - child_model.__table__.create( - bind=session.bind, checkfirst=True) + child_model.__table__.create(bind=session.bind, checkfirst=True) return cls( history_model=history_model, runtime_model=runtime_model, @@ -138,84 +138,86 @@ def _save(self, entry: Entry): logger.error( { - 'entry_by_entry_id': entry_by_entry_id, - 'entry_by_entity_id': entry_by_entity_id, - 'entry': entry.dict(), + "entry_by_entry_id": entry_by_entry_id, + "entry_by_entity_id": entry_by_entity_id, + "entry": entry.dict(), } ) if not entry_by_entry_id: if not entry_by_entity_id: - logger.warning('adding') - return self._session.add( - self.runtime_model( - **runtime_entry_raw - ) - ) + logger.warning("adding") + return self._session.add(self.runtime_model(**runtime_entry_raw)) else: - logger.warning('entity_id but no entry_id') + logger.warning("entity_id but no entry_id") entry_by_entity_id.discarded = True - return self._session.add( - self.runtime_model( - **runtime_entry_raw - ) - ) + return self._session.add(self.runtime_model(**runtime_entry_raw)) return else: if not entry_by_entity_id: - logger.warning('entry_id but no entity_id') + logger.warning("entry_id but no entity_id") for key, value in runtime_entry_raw.items(): setattr(entry_by_entry_id, key, value) return else: - logger.warning('updating') + logger.warning("updating") for key, value in runtime_entry_raw.items(): setattr(entry_by_entry_id, key, value) return if not entry_by_entity_id: # if entry discarded and replaced - logger.error({'entry_by_entry_id': entry_by_entry_id, - 'entry_by_entity_id': entry_by_entity_id}) + logger.error( + { + "entry_by_entry_id": entry_by_entry_id, + "entry_by_entity_id": entry_by_entity_id, + } + ) assert entry_by_entry_id.discarded - raise RuntimeError(f'entry = {entry.dict()}') + raise RuntimeError(f"entry = {entry.dict()}") if entry_by_entry_id.entity_id != entry.entity_id: - logger.warn('entity_id changed', extra={'entry_by_entry_id': entry_by_entry_id, - 'entry': entry.dict()}) + logger.warn( + "entity_id changed", + extra={ + "entry_by_entry_id": entry_by_entry_id, + "entry": entry.dict(), + }, + ) # raise RuntimeError(f'entry = {entry.dict()}') if entry_by_entry_id.entry_id != entry.entry_id: logger.error( { - 'entry_by_entry_id': entry_by_entry_id, - 'entry_by_entity_id': entry_by_entity_id, - 'entry': entry.dict(), + "entry_by_entry_id": entry_by_entry_id, + "entry_by_entity_id": entry_by_entity_id, + "entry": entry.dict(), } ) - raise RuntimeError(f'entry = {entry.dict()}') + raise RuntimeError(f"entry = {entry.dict()}") if entry_by_entity_id.entry_id != entry.entry_id: - logger.error({'entry_by_entry_id': entry_by_entry_id, - 'entry': entry.dict()}) - raise RuntimeError(f'entry = {entry.dict()}') + logger.error( + {"entry_by_entry_id": entry_by_entry_id, "entry": entry.dict()} + ) + raise RuntimeError(f"entry = {entry.dict()}") for key, value in runtime_entry_raw.items(): setattr(entry_by_entry_id, key, value) -# update_result = self._session.query( -# self.runtime_model -# ).filter_by( -# id=entry.id -# ).update( -# runtime_entry_raw, -# ) -# -# if update_result != 1: -# self._session.add( -# self.runtime_model( -# **runtime_entry_raw -# ) -# ) - # return self._session.add(runtime_entry) + # update_result = self._session.query( + # self.runtime_model + # ).filter_by( + # id=entry.id + # ).update( + # runtime_entry_raw, + # ) + # + # if update_result != 1: + # self._session.add( + # self.runtime_model( + # **runtime_entry_raw + # ) + # ) + # return self._session.add(runtime_entry) except db.exc.DBAPIError as exc: - logger.exception('db error') + logger.exception("db error") raise errors.RepositoryError("db failure") from exc def _update(self, entry: Entry): @@ -280,8 +282,7 @@ def _insert_history(self, entry: Entry): def entry_ids(self) -> List[str]: self._check_has_session() - query = self._session.query( - self.runtime_model).filter_by(discarded=False) + query = self._session.query(self.runtime_model).filter_by(discarded=False) return [row.entry_id for row in query.all()] def _by_entry_id( @@ -289,10 +290,14 @@ def _by_entry_id( entry_id: str, ) -> Optional[Entry]: self._check_has_session() - subq = sql.select( - self.history_model.entity_id, - sa.func.max(self.history_model.last_modified).label('maxdate') - ).group_by(self.history_model.entity_id).subquery('t2') + subq = ( + sql.select( + self.history_model.entity_id, + sa.func.max(self.history_model.last_modified).label("maxdate"), + ) + .group_by(self.history_model.entity_id) + .subquery("t2") + ) stmt = sql.select(self.history_model).join( subq, @@ -300,7 +305,7 @@ def _by_entry_id( self.history_model.entity_id == subq.c.entity_id, self.history_model.last_modified == subq.c.maxdate, self.history_model.entry_id == entry_id, - ) + ), ) stmt = stmt.order_by(self.history_model.last_modified.desc()) query = self._session.execute(stmt).scalars() @@ -363,10 +368,14 @@ def teardown(self): def all_entries(self) -> typing.Iterable[Entry]: self._check_has_session() - subq = sql.select( - self.history_model.entity_id, - sa.func.max(self.history_model.last_modified).label('maxdate') - ).group_by(self.history_model.entity_id).subquery('t2') + subq = ( + sql.select( + self.history_model.entity_id, + sa.func.max(self.history_model.last_modified).label("maxdate"), + ) + .group_by(self.history_model.entity_id) + .subquery("t2") + ) stmt = sql.select(self.history_model).join( subq, @@ -374,7 +383,7 @@ def all_entries(self) -> typing.Iterable[Entry]: self.history_model.entity_id == subq.c.entity_id, self.history_model.last_modified == subq.c.maxdate, self.history_model.discarded == False, - ) + ), ) query = self._session.execute(stmt).scalars() # query = self._session.query( @@ -383,8 +392,9 @@ def all_entries(self) -> typing.Iterable[Entry]: def get_total_entries(self) -> int: self._check_has_session() - query = self._session.query( - self.runtime_model.discarded).filter_by(discarded=False) + query = self._session.query(self.runtime_model.discarded).filter_by( + discarded=False + ) return query.count() def by_referenceable(self, filters: Optional[Dict] = None, **kwargs) -> List[Entry]: @@ -407,8 +417,7 @@ def by_referenceable(self, filters: Optional[Dict] = None, **kwargs) -> List[Ent if filter_key in self.resource_config[ "referenceable" ] and self.resource_config["fields"][filter_key].get("collection"): - logger.debug('collection field', extra={ - 'filter_key': filter_key}) + logger.debug("collection field", extra={"filter_key": filter_key}) # child_cls = self.runtime_model.child_tables[filter_key] # tmp[child_cls.__tablename__][filter_key] = filters[filter_key] # print(f"tmp.values() = {tmp.values()}") @@ -429,18 +438,15 @@ def by_referenceable(self, filters: Optional[Dict] = None, **kwargs) -> List[Ent if match: raise errors.NonExistingField(match.group(1)) from exc else: - raise errors.RepositoryError( - "Unknown invalid request") from exc + raise errors.RepositoryError("Unknown invalid request") from exc for child_filters in joined_filters: child_table_name = list(child_filters.keys())[0] - logger.debug('child table name', extra={ - 'table_name': child_table_name}) + logger.debug("child table name", extra={"table_name": child_table_name}) child_cls = self.runtime_model.child_tables[child_table_name] - child_query = self._session.query( - child_cls).filter_by(**child_filters) + child_query = self._session.query(child_cls).filter_by(**child_filters) for child_e in child_query: - logger.debug('child hit', extra={'child_hit': child_e}) + logger.debug("child hit", extra={"child_hit": child_e}) query = query.join(child_cls).filter_by(**child_filters) # result = query.filter_by(**kwargs).all() # # query = self._session.query(self.history_model) @@ -516,7 +522,7 @@ def _entry_to_history_dict( "message": entry.message, "op": entry.op, "discarded": entry.discarded, - 'repo_id': entry.repo_id, + "repo_id": entry.repo_id, } def _history_row_to_entry(self, row) -> Entry: @@ -551,8 +557,7 @@ def _entry_to_runtime_dict(self, history_id: int, entry: Entry) -> Dict: for elem in field_val: if field_name not in _entry: _entry[field_name] = [] - _entry[field_name].append( - child_table(**{field_name: elem})) + _entry[field_name].append(child_table(**{field_name: elem})) else: _entry[field_name] = field_val return _entry @@ -562,18 +567,16 @@ class SqlEntryUnitOfWork( SqlUnitOfWork, repositories.EntryUnitOfWork, ): - repository_type: str = 'sql_entries_base' + repository_type: str = "sql_entries_base" def __init__( self, session_factory: sessionmaker, event_bus: EventBus, **kwargs, - ): SqlUnitOfWork.__init__(self) - repositories.EntryUnitOfWork.__init__( - self, event_bus=event_bus, **kwargs) + repositories.EntryUnitOfWork.__init__(self, event_bus=event_bus, **kwargs) self.session_factory = session_factory self._entries = None self._session = None @@ -585,7 +588,7 @@ def _begin(self): self._entries = SqlEntryRepository.from_dict( name=self.table_name(), resource_config=self.config, - session=self._session + session=self._session, ) return self @@ -610,17 +613,18 @@ def collect_new_events(self) -> typing.Iterable: class SqlEntryUnitOfWorkV1(SqlEntryUnitOfWork): - repository_type: str = 'sql_entries_v1' + repository_type: str = "sql_entries_v1" class SqlEntryUnitOfWorkV2(SqlEntryUnitOfWork): - repository_type: str = 'sql_entries_v2' + repository_type: str = "sql_entries_v2" def table_name(self) -> str: u = ulid.from_uuid(self.entity_id) random_part = u.randomness().str return f"{self.name}_{random_part}" + # ===== Value objects ===== # class SqlEntryRepositorySettings(EntryRepositorySettings): # def __init__(self, *, table_name: str, config: Dict): @@ -638,7 +642,7 @@ def table_name(self) -> str: # runtime_table_name, history_model, settings.config # ) # return SqlEntryRepository(history_model, runtime_model, settings.config) -SqlEntryUowType = TypeVar('SqlEntryUowType', bound=SqlEntryUnitOfWork) +SqlEntryUowType = TypeVar("SqlEntryUowType", bound=SqlEntryUnitOfWork) class SqlEntryUowCreator(Generic[SqlEntryUowType]): @@ -679,18 +683,14 @@ def __call__( return self.cache[entity_id] -class SqlEntryUowV1Creator( - SqlEntryUowCreator[SqlEntryUnitOfWorkV1] -): +class SqlEntryUowV1Creator(SqlEntryUowCreator[SqlEntryUnitOfWorkV1]): repository_type: str = "sql_entries_v1" def _create_uow(self, **kwargs) -> SqlEntryUnitOfWorkV1: return SqlEntryUnitOfWorkV1(**kwargs) -class SqlEntryUowV2Creator( - SqlEntryUowCreator[SqlEntryUnitOfWorkV2] -): +class SqlEntryUowV2Creator(SqlEntryUowCreator[SqlEntryUnitOfWorkV2]): repository_type: str = "sql_entries_v2" def _create_uow(self, **kwargs) -> SqlEntryUnitOfWorkV2: From 386f5f60a62088914ac95fe1d24c2b329019f80f Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 28 Mar 2022 15:28:37 +0200 Subject: [PATCH 15/43] fix: add num_entities --- karp/cliapp/subapps/entry_repo_subapp.py | 2 ++ karp/foundation/entity.py | 7 +++--- karp/foundation/repository.py | 4 ++++ karp/lex/__init__.py | 11 ++++++++++ karp/lex/application/repositories/entries.py | 3 +++ karp/lex/application/use_cases/__init__.py | 2 +- .../use_cases/entry_repo_handlers.py | 18 +++++++++++++++ karp/lex/domain/commands/__init__.py | 2 +- .../domain/commands/entry_repo_commands.py | 6 +++++ .../repositories/sql_entries.py | 6 +++++ .../repositories/sql_entry_uows.py | 22 +++++++++++++++++++ .../repositories/sql_resources.py | 21 ++++++++++++++++++ .../application/repositories/indicies.py | 3 +++ karp/tests/foundation/unit/test_entity.py | 17 +------------- karp/tests/unit/lex/adapters.py | 11 +++++++++- karp/tests/unit/lex/factories.py | 9 ++++++++ .../unit/lex/test_entry_repo_handlers.py | 8 +++++-- karp/tests/unit/search/adapters.py | 3 +++ 18 files changed, 130 insertions(+), 25 deletions(-) diff --git a/karp/cliapp/subapps/entry_repo_subapp.py b/karp/cliapp/subapps/entry_repo_subapp.py index ce5cde19..f8b0b328 100644 --- a/karp/cliapp/subapps/entry_repo_subapp.py +++ b/karp/cliapp/subapps/entry_repo_subapp.py @@ -1,9 +1,11 @@ import json +from typing import Optional from tabulate import tabulate import typer from karp import lex +from karp.foundation.value_objects import UniqueId from karp.foundation.commands import CommandBus from karp.lex.application.queries import ListEntryRepos from karp.lex.domain.commands import CreateEntryRepository diff --git a/karp/foundation/entity.py b/karp/foundation/entity.py index f65728bb..2c7d5d48 100644 --- a/karp/foundation/entity.py +++ b/karp/foundation/entity.py @@ -131,10 +131,10 @@ def stamp(self, user, *, timestamp: float = None): self._last_modified_by = user self._last_modified = monotonic_utc_now() if timestamp is None else timestamp - def discard(self, *, user, last_modified: float, timestamp: float = None): + def discard(self, *, user, timestamp: float = None): self._check_not_discarded() - self._validate_last_modified(last_modified) - super.discard() + # self._validate_last_modified(last_modified) + super().discard() self._last_modified_by = user self._last_modified = self._ensure_timestamp(timestamp) @@ -236,4 +236,3 @@ def update( self._last_modified_by = user self._last_modified = self._ensure_timestamp(timestamp) self._increment_version() - diff --git a/karp/foundation/repository.py b/karp/foundation/repository.py index 37c05715..760db194 100644 --- a/karp/foundation/repository.py +++ b/karp/foundation/repository.py @@ -58,3 +58,7 @@ def _by_id( self, id_: Union[uuid.UUID, str], *, version: Optional[int] = None ) -> Optional[EntityType]: raise NotImplementedError() + + @abc.abstractmethod + def num_entities(self) -> int: + ... diff --git a/karp/lex/__init__.py b/karp/lex/__init__.py index 9eea9e6a..1a343401 100644 --- a/karp/lex/__init__.py +++ b/karp/lex/__init__.py @@ -10,6 +10,7 @@ from karp.lex.domain.commands import ( CreateEntryRepository, CreateResource, + DeleteEntryRepository, ) from karp.lex.domain import commands from karp.lex.domain.value_objects import EntrySchema @@ -19,6 +20,7 @@ CreatingEntryRepo, CreatingResource, DeletingEntry, + DeletingEntryRepository, DeletingResource, PublishingResource, UpdatingEntry, @@ -55,6 +57,15 @@ def create_entry_repository( entry_repo_uow=uow, ) + @injector.provider + def deleting_entry_repository( + self, + uow: EntryUowRepositoryUnitOfWork, + ) -> CommandHandler[DeleteEntryRepository]: + return DeletingEntryRepository( + entry_repo_uow=uow, + ) + @injector.provider def create_resource( self, diff --git a/karp/lex/application/repositories/entries.py b/karp/lex/application/repositories/entries.py index 03c1286a..24d4df93 100644 --- a/karp/lex/application/repositories/entries.py +++ b/karp/lex/application/repositories/entries.py @@ -185,3 +185,6 @@ def config(self) -> Dict: @property def message(self) -> str: return self._message + + def discard(self, *, user, timestamp: Optional[float] = None): + return entity.TimestampedEntity.discard(self, user=user, timestamp=timestamp) diff --git a/karp/lex/application/use_cases/__init__.py b/karp/lex/application/use_cases/__init__.py index 17907bda..279eb95c 100644 --- a/karp/lex/application/use_cases/__init__.py +++ b/karp/lex/application/use_cases/__init__.py @@ -4,7 +4,7 @@ DeletingEntry, UpdatingEntry, ) -from .entry_repo_handlers import CreatingEntryRepo +from .entry_repo_handlers import CreatingEntryRepo, DeletingEntryRepository from .resource_handlers import ( CreatingResource, DeletingResource, diff --git a/karp/lex/application/use_cases/entry_repo_handlers.py b/karp/lex/application/use_cases/entry_repo_handlers.py index f606c80d..34ef3f4c 100644 --- a/karp/lex/application/use_cases/entry_repo_handlers.py +++ b/karp/lex/application/use_cases/entry_repo_handlers.py @@ -1,4 +1,5 @@ import logging +from typing import Any from karp.foundation.commands import CommandHandler from karp.lex.domain import commands @@ -37,3 +38,20 @@ def execute( uow.repo.save(entry_repo) uow.commit() return entry_repo + + +class DeletingEntryRepository(CommandHandler[commands.DeleteEntryRepository]): + def __init__( + self, + entry_repo_uow: repositories.EntryUowRepositoryUnitOfWork, + **kwargs, + ): + self._entry_repo_uow = entry_repo_uow + + def execute(self, command: commands.DeleteEntryRepository) -> None: + with self._entry_repo_uow as uow: + entry_repo = uow.repo.get_by_id(command.entity_id) + entry_repo.discard( + user=command.user, timestamp=command.timestamp) + uow.repo.save(entry_repo) + uow.commit() diff --git a/karp/lex/domain/commands/__init__.py b/karp/lex/domain/commands/__init__.py index 8c0e5dcf..8cbc77ff 100644 --- a/karp/lex/domain/commands/__init__.py +++ b/karp/lex/domain/commands/__init__.py @@ -6,6 +6,6 @@ DeleteEntry, UpdateEntry, ) -from .entry_repo_commands import CreateEntryRepository +from .entry_repo_commands import CreateEntryRepository, DeleteEntryRepository from .resource_commands import ( CreateResource, DeleteResource, PublishResource, UpdateResource) diff --git a/karp/lex/domain/commands/entry_repo_commands.py b/karp/lex/domain/commands/entry_repo_commands.py index a203f8b6..51a75bce 100644 --- a/karp/lex/domain/commands/entry_repo_commands.py +++ b/karp/lex/domain/commands/entry_repo_commands.py @@ -33,3 +33,9 @@ def from_dict( user=user, message=message or 'Entry repository created' ) + + +class DeleteEntryRepository(Command): + entity_id: UniqueId + message: str + user: str diff --git a/karp/lex_infrastructure/repositories/sql_entries.py b/karp/lex_infrastructure/repositories/sql_entries.py index 1dbcb6a8..af6c5515 100644 --- a/karp/lex_infrastructure/repositories/sql_entries.py +++ b/karp/lex_infrastructure/repositories/sql_entries.py @@ -397,6 +397,12 @@ def get_total_entries(self) -> int: ) return query.count() + def num_entities(self) -> int: + self._check_has_session() + query = self._session.query( + self.runtime_model.discarded).filter_by(discarded=False) + return query.count() + def by_referenceable(self, filters: Optional[Dict] = None, **kwargs) -> List[Entry]: self._check_has_session() # query = self._session.query(self.runtime_model) diff --git a/karp/lex_infrastructure/repositories/sql_entry_uows.py b/karp/lex_infrastructure/repositories/sql_entry_uows.py index d607280c..53203161 100644 --- a/karp/lex_infrastructure/repositories/sql_entry_uows.py +++ b/karp/lex_infrastructure/repositories/sql_entry_uows.py @@ -1,6 +1,7 @@ import logging from typing import Optional +import sqlalchemy as sa from sqlalchemy import sql from sqlalchemy import orm as sa_orm @@ -53,6 +54,27 @@ def _by_id(self, id_: UniqueId, **kwargs) -> Optional[EntryUnitOfWork]: return self._row_to_entity(row) return None + def num_entities(self) -> int: + self._check_has_session() + subq = ( + self._session.query( + EntryUowModel.entity_id, + sa.func.max(EntryUowModel.last_modified).label("maxdate"), + ) + .group_by(EntryUowModel.entity_id) + .subquery("t2") + ) + query = self._session.query(EntryUowModel).join( + subq, + db.and_( + EntryUowModel.entity_id == subq.c.entity_id, + EntryUowModel.last_modified == subq.c.maxdate, + EntryUowModel.discarded == False, + ), + ) + + return query.count() + def _row_to_entity(self, row_proxy) -> EntryUnitOfWork: return self.entry_uow_factory.create( repository_type=row_proxy.type, diff --git a/karp/lex_infrastructure/repositories/sql_resources.py b/karp/lex_infrastructure/repositories/sql_resources.py index 75e829f7..e651a60b 100644 --- a/karp/lex_infrastructure/repositories/sql_resources.py +++ b/karp/lex_infrastructure/repositories/sql_resources.py @@ -178,6 +178,27 @@ def _get_all_resources(self) -> typing.List[entities.Resource]: if resource_dto is not None ] + def num_entities(self) -> int: + self._check_has_session() + subq = ( + self._session.query( + ResourceModel.entity_id, + sa.func.max(ResourceModel.last_modified).label("maxdate"), + ) + .group_by(ResourceModel.entity_id) + .subquery("t2") + ) + query = self._session.query(ResourceModel).join( + subq, + db.and_( + ResourceModel.entity_id == subq.c.entity_id, + ResourceModel.last_modified == subq.c.maxdate, + ResourceModel.discarded == False, + ), + ) + + return query.count() + def _resource_to_dict(self, resource: Resource) -> typing.Dict: return { "history_id": None, diff --git a/karp/search/application/repositories/indicies.py b/karp/search/application/repositories/indicies.py index 32cfefb9..814db1fd 100644 --- a/karp/search/application/repositories/indicies.py +++ b/karp/search/application/repositories/indicies.py @@ -59,6 +59,9 @@ def _save(self, _notused): def _by_id(self, id) -> None: return None + def num_entities(self) -> int: + raise NotImplementedError("num_entities is not used for indicies") + class IndexUnitOfWork( unit_of_work.UnitOfWork[Index] diff --git a/karp/tests/foundation/unit/test_entity.py b/karp/tests/foundation/unit/test_entity.py index 2b6fc3b4..9c9074a0 100644 --- a/karp/tests/foundation/unit/test_entity.py +++ b/karp/tests/foundation/unit/test_entity.py @@ -9,11 +9,6 @@ class TestVersionedEntity: - def test_discard_w_wrong_version_raises_consistency_error(self): - entity = VersionedEntity("id_v1", 1) - - with pytest.raises(ConsistencyError): - entity.discard(version=2) def test_updated_w_wrong_version_raises_consistency_error(self): entity = VersionedEntity("id_v1", 1) @@ -21,7 +16,6 @@ def test_updated_w_wrong_version_raises_consistency_error(self): with pytest.raises(ConsistencyError): entity.update(version=2) - def test_update_increments_version(self): entity = VersionedEntity("id_v1", 1) @@ -31,13 +25,6 @@ def test_update_increments_version(self): class TestTimestampedEntity: - def test_discard_w_wrong_last_modified_raises_consistency_error(self): - entity = TimestampedEntity("id_v1", 1) - with pytest.raises(ConsistencyError): - entity.discard( - last_modified=2, - user="Test", - ) def test_update_w_wrong_last_modified_raises_consistency_error(self): entity = TimestampedEntity("id_v1", 1) @@ -88,7 +75,6 @@ def test_discard_discards(self): assert entity.last_modified_by != previous_last_modified_by assert entity.version == (previous_version + 1) - def test_discard_w_wrong_version_raises_consistency_error(self): entity = TimestampedVersionedEntity("id_v1", version=1) @@ -127,7 +113,7 @@ def test_update_w_wrong_last_modified_raises_consistency_error(self): version=1, last_modified=2, user="Test" - ) + ) def test_update_updates(self): entity = TimestampedVersionedEntity("id_v1", version=1) @@ -146,4 +132,3 @@ def test_update_updates(self): assert entity.last_modified_by != previous_last_modified_by assert entity.last_modified_by == "Test" assert entity.version == (previous_version + 1) - diff --git a/karp/tests/unit/lex/adapters.py b/karp/tests/unit/lex/adapters.py index d097d9c5..b921eb7f 100644 --- a/karp/tests/unit/lex/adapters.py +++ b/karp/tests/unit/lex/adapters.py @@ -56,6 +56,9 @@ def _get_all_resources(self) -> typing.Iterable[lex_entities.Resource]: def resource_ids(self) -> typing.Iterable[str]: return (res.resource_id for res in self.resources) + def num_entities(self) -> int: + return len(self.resources) + class InMemoryReadResourceRepository(ReadOnlyResourceRepository): def __init__(self, resources: Dict): @@ -143,6 +146,9 @@ def from_dict(cls, _): def all_entries(self) -> typing.Iterable[lex_entities.Entry]: yield from self.entries + def num_entities(self) -> int: + return len(self.entries) + class InMemoryEntryUnitOfWork( InMemoryUnitOfWork, lex_repositories.EntryUnitOfWork @@ -246,7 +252,7 @@ def create_entry_uow2( class InMemoryEntryUowRepository(lex_repositories.EntryUowRepository): def __init__(self) -> None: super().__init__() - self._storage = {} + self._storage: dict[UniqueId, dict] = {} def _save(self, entry_repo): self._storage[entry_repo.id] = entry_repo @@ -257,6 +263,9 @@ def _by_id(self, id_, *, version: Optional[int] = None): def __len__(self): return len(self._storage) + def num_entities(self) -> int: + return len(self._storage) + class InMemoryEntryUowRepositoryUnitOfWork(InMemoryUnitOfWork, lex_repositories.EntryUowRepositoryUnitOfWork): def __init__( diff --git a/karp/tests/unit/lex/factories.py b/karp/tests/unit/lex/factories.py index 14e45844..2c393944 100644 --- a/karp/tests/unit/lex/factories.py +++ b/karp/tests/unit/lex/factories.py @@ -125,6 +125,15 @@ class Meta: user = factory.Faker('email') +class DeleteEntryRepositoryFactory(factory.Factory): + class Meta: + model = commands.DeleteEntryRepository + + entity_id = factory.LazyFunction(make_unique_id) + message = 'entry repository deleted' + user = factory.Faker('email') + + class MachineNameFactory(factory.Factory): class Meta: model = MachineName diff --git a/karp/tests/unit/lex/test_entry_repo_handlers.py b/karp/tests/unit/lex/test_entry_repo_handlers.py index 1c9671e5..14f6f582 100644 --- a/karp/tests/unit/lex/test_entry_repo_handlers.py +++ b/karp/tests/unit/lex/test_entry_repo_handlers.py @@ -17,7 +17,7 @@ def test_create_entry_repository( entry_uow_repo_uow = lex_ctx.container.get( EntryUowRepositoryUnitOfWork) assert entry_uow_repo_uow.was_committed - assert len(entry_uow_repo_uow.repo) == 1 + assert entry_uow_repo_uow.repo.num_entities() == 1 class TestDeleteEntryRepository: @@ -30,4 +30,8 @@ def test_delete_entry_repository_succeeds( entry_uow_repo_uow = lex_ctx.container.get( EntryUowRepositoryUnitOfWork) - assert entry_uow_repo_uow.was_committed + assert entry_uow_repo_uow.repo.num_entities() == 1 + + cmd = factories.DeleteEntryRepositoryFactory(entity_id=cmd.entity_id) + lex_ctx.command_bus.dispatch(cmd) + assert entry_uow_repo_uow.repo.num_entities() == 0 diff --git a/karp/tests/unit/search/adapters.py b/karp/tests/unit/search/adapters.py index c804c9ca..e4cac9d1 100644 --- a/karp/tests/unit/search/adapters.py +++ b/karp/tests/unit/search/adapters.py @@ -6,6 +6,7 @@ from karp.foundation.commands import CommandBus from karp.foundation.events import EventBus from karp.foundation.time import utc_now +from karp.lex.application.repositories import entries from karp.search.application.repositories import IndexUnitOfWork, Index, IndexEntry from karp.tests.foundation.adapters import InMemoryUnitOfWork @@ -64,6 +65,8 @@ def delete_entry( # # def statistics(self, resource_id: str, field: str): # return {} + def num_entities(self) -> int: + return sum(len(entries) for entries in self.indicies.values()) class InMemoryIndexUnitOfWork( From 4ddf9441de9010960cfe8f7a8161f942e0368f32 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 29 Mar 2022 07:48:36 +0200 Subject: [PATCH 16/43] test: fix num_entities in InMemoryRepos --- karp/tests/unit/lex/adapters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/karp/tests/unit/lex/adapters.py b/karp/tests/unit/lex/adapters.py index b921eb7f..fc6307ae 100644 --- a/karp/tests/unit/lex/adapters.py +++ b/karp/tests/unit/lex/adapters.py @@ -41,7 +41,7 @@ def _save(self, resource): def _by_id(self, id_, *, version=None): return self.resources.get(id_) - def _by_resource_id(self, resource_id, *, version=None): + def _by_resource_id(self, resource_id): return next((res for res in self.resources.values() if res.resource_id == resource_id), None) def __len__(self): @@ -57,7 +57,7 @@ def resource_ids(self) -> typing.Iterable[str]: return (res.resource_id for res in self.resources) def num_entities(self) -> int: - return len(self.resources) + return sum( not res.discarded for res in self.resources.values() ) class InMemoryReadResourceRepository(ReadOnlyResourceRepository): @@ -147,7 +147,7 @@ def all_entries(self) -> typing.Iterable[lex_entities.Entry]: yield from self.entries def num_entities(self) -> int: - return len(self.entries) + return sum( not e.discarded for e in self.entries.values() ) class InMemoryEntryUnitOfWork( @@ -264,7 +264,7 @@ def __len__(self): return len(self._storage) def num_entities(self) -> int: - return len(self._storage) + return sum(not er.discarded for er in self._storage.values()) class InMemoryEntryUowRepositoryUnitOfWork(InMemoryUnitOfWork, lex_repositories.EntryUowRepositoryUnitOfWork): From 4b4096abc61520c6a888b8a1385c5518f7b33a97 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 29 Mar 2022 06:08:47 +0000 Subject: [PATCH 17/43] =?UTF-8?q?=F0=9F=93=9D=20Update=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ae210ac..26b75a96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Latest Changes +* Add support for discarding entry-repositories. PR [#195](https://github.com/spraakbanken/karp-backend/pull/195) by [@kod-kristoff](https://github.com/kod-kristoff). * Avoid name clashes in SqlEntryRepository. PR [#194](https://github.com/spraakbanken/karp-backend/pull/194) by [@kod-kristoff](https://github.com/kod-kristoff). * Fix correct fetching from repos. PR [#193](https://github.com/spraakbanken/karp-backend/pull/193) by [@kod-kristoff](https://github.com/kod-kristoff). ## 6.0.17 From 55723a0c2e400edf40bb7ad318968e034b70efd5 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 29 Mar 2022 09:38:57 +0200 Subject: [PATCH 18/43] style: use black again --- karp/tests/e2e/test_cliapp_resources.py | 29 +++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/karp/tests/e2e/test_cliapp_resources.py b/karp/tests/e2e/test_cliapp_resources.py index ea62642b..e5dd6a35 100644 --- a/karp/tests/e2e/test_cliapp_resources.py +++ b/karp/tests/e2e/test_cliapp_resources.py @@ -9,7 +9,7 @@ class TestCliResourceLifetime: def test_help(self, runner: CliRunner, cliapp: Typer): - result = runner.invoke(cliapp, ['resource', '--help']) + result = runner.invoke(cliapp, ["resource", "--help"]) assert result.exit_code == 0 def test_create_and_publish_repo( @@ -19,24 +19,29 @@ def test_create_and_publish_repo( app_context: AppContext, ): result = runner.invoke( - cliapp, ['entry-repo', 'create', 'karp/tests/data/config/lexlex.json']) - print(f'{result.stdout=}') + cliapp, ["entry-repo", "create", "karp/tests/data/config/lexlex.json"] + ) + print(f"{result.stdout=}") assert result.exit_code == 0 - entry_repo_repo = app_context.container.get( - lex.ReadOnlyEntryRepoRepositry) + entry_repo_repo = app_context.container.get(lex.ReadOnlyEntryRepoRepositry) - entry_repo = entry_repo_repo.get_by_name('lexlex') + entry_repo = entry_repo_repo.get_by_name("lexlex") assert entry_repo is not None result = runner.invoke( - cliapp, ['resource', 'create', - 'karp/tests/data/config/lexlex.json', '--entry-repo-id', str(entry_repo.entity_id)] + cliapp, + [ + "resource", + "create", + "karp/tests/data/config/lexlex.json", + "--entry-repo-id", + str(entry_repo.entity_id), + ], ) - print(f'{result.stdout=}') + print(f"{result.stdout=}") assert result.exit_code == 0 - resource_repo = app_context.container.get( - lex.ReadOnlyResourceRepository) + resource_repo = app_context.container.get(lex.ReadOnlyResourceRepository) - assert resource_repo.get_by_resource_id('lexlex') is not None + assert resource_repo.get_by_resource_id("lexlex") is not None From 3de75d6e20791899e9f566a22ce91c0b4ddbcecf Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 29 Mar 2022 09:39:15 +0200 Subject: [PATCH 19/43] build: add black --- poetry.lock | 73 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index cb74d365..7ae8e5b5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -148,6 +148,28 @@ test = ["coverage (>=4.5.4)", "fixtures (>=3.0.0)", "flake8 (>=4.0.0)", "stestr toml = ["toml"] yaml = ["pyyaml"] +[[package]] +name = "black" +version = "22.3.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "bump2version" version = "1.0.1" @@ -836,6 +858,14 @@ python-versions = "*" [package.extras] dev = ["pyre-check", "flake8", "pytest", "pytest-cov", "mypy"] +[[package]] +name = "pathspec" +version = "0.9.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + [[package]] name = "pbr" version = "5.8.1" @@ -1338,6 +1368,14 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + [[package]] name = "tqdm" version = "4.63.0" @@ -1479,7 +1517,7 @@ sqlite = ["aiosqlite"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "3458915d2cbe2136d46597608bf65ee09e8f04ba1a2c3f5f9322ec07a9397ab5" +content-hash = "5738bedfceed5fb7327b041f6ff2cc72914a10ff076fe47185af52a34035612b" [metadata.files] aiomysql = [ @@ -1530,6 +1568,31 @@ bandit = [ {file = "bandit-1.7.4-py3-none-any.whl", hash = "sha256:412d3f259dab4077d0e7f0c11f50f650cc7d10db905d98f6520a95a18049658a"}, {file = "bandit-1.7.4.tar.gz", hash = "sha256:2d63a8c573417bae338962d4b9b06fbc6080f74ecd955a092849e1e65c717bd2"}, ] +black = [ + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, +] bump2version = [ {file = "bump2version-1.0.1-py2.py3-none-any.whl", hash = "sha256:37f927ea17cde7ae2d7baf832f8e80ce3777624554a653006c9144f8017fe410"}, {file = "bump2version-1.0.1.tar.gz", hash = "sha256:762cb2bfad61f4ec8e2bdf452c7c267416f8c70dd9ecb1653fd0bbb01fa936e6"}, @@ -2074,6 +2137,10 @@ paradigmextract = [ {file = "paradigmextract-0.1.1-py3-none-any.whl", hash = "sha256:f001ca9f147905d5a9abce158c6923005fcc5af108134858b74cfa322dd130f4"}, {file = "paradigmextract-0.1.1.tar.gz", hash = "sha256:63764fee55adf48abb68f9956a22ecd4a1257952b99b1592bcf3ae064ed7cc0c"}, ] +pathspec = [ + {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, + {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, +] pbr = [ {file = "pbr-5.8.1-py2.py3-none-any.whl", hash = "sha256:27108648368782d07bbf1cb468ad2e2eeef29086affd14087a6d04b7de8af4ec"}, {file = "pbr-5.8.1.tar.gz", hash = "sha256:66bc5a34912f408bb3925bf21231cb6f59206267b7f63f3503ef865c1a292e25"}, @@ -2465,6 +2532,10 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] tqdm = [ {file = "tqdm-4.63.0-py2.py3-none-any.whl", hash = "sha256:e643e071046f17139dea55b880dc9b33822ce21613b4a4f5ea57f202833dbc29"}, {file = "tqdm-4.63.0.tar.gz", hash = "sha256:1d9835ede8e394bb8c9dcbffbca02d717217113adc679236873eeaac5bc0b3cd"}, diff --git a/pyproject.toml b/pyproject.toml index 06d07292..48b6f386 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,6 +106,7 @@ vulture = "^2.3" flake8-bandit = "^2.1.2" pylint = "^2.12.2" flake8-pylint = "^0.1.3" +black = "^22.3.0" [tool.flakehell] exclude = ["README.rst", "README.md"] From 31b2bbcc4f16333aad01cd519337988548c799c4 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 29 Mar 2022 09:43:56 +0200 Subject: [PATCH 20/43] fix: return result from command_handler --- karp/foundation/commands.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/karp/foundation/commands.py b/karp/foundation/commands.py index 9112a2d1..ff4bff3f 100644 --- a/karp/foundation/commands.py +++ b/karp/foundation/commands.py @@ -5,7 +5,7 @@ import injector -CommandType = TypeVar('CommandType') +CommandType = TypeVar("CommandType") logger = logging.getLogger(__name__) @@ -22,20 +22,18 @@ class Command: class CommandBus(abc.ABC): @abc.abstractmethod - def dispatch(self, command: Command) -> None: + def dispatch(self, command: Command) -> Any: raise NotImplementedError class InjectorCommandBus(CommandBus): - def __init__(self, injector: injector.Injector) -> None: - self._injector = injector + def __init__(self, container: injector.Injector) -> None: + self._container = container - def dispatch(self, command: Command) -> None: - logger.info('Handling command: %s', command) + def dispatch(self, command: Command) -> Any: + logger.info("Handling command: %s", command) cmd_cls = type(command) - cmd_handler = self._injector.get( - CommandHandler[cmd_cls]) # type: ignore + cmd_handler = self._container.get(CommandHandler[cmd_cls]) # type: ignore - logger.info('Handling command %s with handler %s', - command, cmd_handler) - cmd_handler.execute(command) + logger.info("Handling command %s with handler %s", command, cmd_handler) + return cmd_handler.execute(command) From ba9184309071a3513907a088d081b0af665f6cbc Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Tue, 29 Mar 2022 14:10:50 +0200 Subject: [PATCH 21/43] fix: add set entry repo id --- karp/cliapp/subapps/resource_subapp.py | 73 +++++------ karp/lex/__init__.py | 46 ++++--- karp/lex/application/use_cases/__init__.py | 1 + .../use_cases/resource_handlers.py | 122 +++++++++++------- karp/lex/domain/commands/__init__.py | 7 +- karp/lex/domain/commands/resource_commands.py | 9 +- karp/lex/domain/entities/resource.py | 55 +++++--- karp/tests/e2e/test_cliapp_resources.py | 35 +++++ 8 files changed, 229 insertions(+), 119 deletions(-) diff --git a/karp/cliapp/subapps/resource_subapp.py b/karp/cliapp/subapps/resource_subapp.py index a5008699..49b76170 100644 --- a/karp/cliapp/subapps/resource_subapp.py +++ b/karp/cliapp/subapps/resource_subapp.py @@ -28,14 +28,14 @@ subapp = typer.Typer() -T = TypeVar('T') +T = TypeVar("T") def choose_from(choices: List[T], choice_fmt: Callable[[T], str]) -> T: for i, choice in enumerate(choices): - typer.echo(f'{i}) {choice_fmt(choice)}') + typer.echo(f"{i}) {choice_fmt(choice)}") while True: - number = typer.prompt(f'Choose from above with (0-{len(choices)-1}):') + number = typer.prompt(f"Choose from above with (0-{len(choices)-1}):") return choices[int(number)] @@ -45,8 +45,7 @@ def choose_from(choices: List[T], choice_fmt: Callable[[T], str]) -> T: def create( ctx: typer.Context, config: Path, - entry_repo_id: Optional[str] = typer.Option( - None, help='id for entry-repo'), + entry_repo_id: Optional[str] = typer.Option(None, help="id for entry-repo"), ): bus = inject_from_ctx(CommandBus, ctx) if config.is_file(): @@ -55,7 +54,8 @@ def create( query = inject_from_ctx(ListEntryRepos, ctx) entry_repos = list(query.query()) entry_repo = choose_from( - entry_repos, lambda x: f'{x.name} {x.repository_type}') + entry_repos, lambda x: f"{x.name} {x.repository_type}" + ) entry_repo_uuid = entry_repo.entity_id else: entry_repo_uuid = unique_id.UniqueId(entry_repo_id) @@ -68,7 +68,7 @@ def create( typer.echo(f"Created resource '{cmd.resource_id}' ({cmd.entity_id})") elif config.is_dir(): - typer.Abort('not supported yetls') + typer.Abort("not supported yetls") # new_resources = resources.create_new_resource_from_dir(config) # for resource_id in new_resources: @@ -78,9 +78,26 @@ def create( @subapp.command() @cli_error_handler @cli_timer -def update( - ctx: typer.Context, - config: Path): +def set_entry_repo( + ctx: typer.Context, + resource_id: str, + entry_repo_id: str = typer.Argument(..., help="id for entry-repo"), + user: Optional[str] = typer.Option(None), +): + bus = inject_from_ctx(CommandBus, ctx) + entry_repo_uuid = unique_id.UniqueId(entry_repo_id) + cmd = lex_commands.SetEntryRepoId( + resource_id=resource_id, + entry_repo_id=entry_repo_uuid, + user=user or "local admin", + ) + bus.dispatch(cmd) + + +@subapp.command() +@cli_error_handler +@cli_timer +def update(ctx: typer.Context, config: Path): bus = inject_from_ctx(CommandBus, ctx) if config.is_file(): with open(config) as fp: @@ -95,16 +112,13 @@ def update( if version is None: typer.echo(f"Nothing to do for resource '{resource_id}'") else: - typer.echo( - f"Updated version {version} of resource '{resource_id}'") + typer.echo(f"Updated version {version} of resource '{resource_id}'") @subapp.command() @cli_error_handler @cli_timer -def publish( - ctx: typer.Context, - resource_id: str): +def publish(ctx: typer.Context, resource_id: str): bus = inject_from_ctx(CommandBus, ctx) try: cmd = lex_commands.PublishResource( @@ -122,9 +136,7 @@ def publish( @subapp.command() @cli_error_handler @cli_timer -def reindex( - ctx: typer.Context, - resource_id: str): +def reindex(ctx: typer.Context, resource_id: str): bus = inject_from_ctx(CommandBus, ctx) cmd = search_commands.ReindexResource(resource_id=resource_id) bus.dispatch(cmd) @@ -180,8 +192,7 @@ def reindex( @cli_timer def list_resources( ctx: typer.Context, - show_published: Optional[bool] = typer.Option( - True, "--show-published/--show-all") + show_published: Optional[bool] = typer.Option(True, "--show-published/--show-all"), ): if show_published: query = inject_from_ctx(GetPublishedResources, ctx) @@ -201,11 +212,7 @@ def list_resources( @subapp.command() @cli_error_handler @cli_timer -def show( - ctx: typer.Context, - resource_id: str, - version: Optional[int] = None -): +def show(ctx: typer.Context, resource_id: str, version: Optional[int] = None): repo = inject_from_ctx(lex.ReadOnlyResourceRepository, ctx) resource = repo.get_by_resource_id(resource_id, version=version) # if version: @@ -221,14 +228,7 @@ def show( ) raise typer.Exit(3) - typer.echo( - tabulate( - ( - (key, value) - for key, value in resource.dict().items() - ) - ) - ) + typer.echo(tabulate(((key, value) for key, value in resource.dict().items()))) @subapp.command() @@ -241,6 +241,8 @@ def set_permissions( level: str, ): bus = inject_from_ctx(CommandBus, ctx) + + # # TODO use level # permissions = {"write": True, "read": True} # resourcemgr.set_permissions(resource_id, version, permissions) @@ -267,11 +269,10 @@ def delete( cmd = lex_commands.DeleteResource( resource_id=resource_id, user=user or "local admin", - message=message or "resource deleted" + message=message or "resource deleted", ) resource = bus.dispatch(cmd) - typer.echo( - f"Deleted resource '{resource.resource_id}' ({resource.entity_id})") + typer.echo(f"Deleted resource '{resource.resource_id}' ({resource.entity_id})") def init_app(app): diff --git a/karp/lex/__init__.py b/karp/lex/__init__.py index 1a343401..3c3068c7 100644 --- a/karp/lex/__init__.py +++ b/karp/lex/__init__.py @@ -11,8 +11,10 @@ CreateEntryRepository, CreateResource, DeleteEntryRepository, + SetEntryRepoId, ) from karp.lex.domain import commands +from karp.lex.domain.commands.resource_commands import SetEntryRepoId from karp.lex.domain.value_objects import EntrySchema from karp.lex.application.use_cases import ( AddingEntries, @@ -23,6 +25,7 @@ DeletingEntryRepository, DeletingResource, PublishingResource, + SettingEntryRepoId, UpdatingEntry, UpdatingResource, ) @@ -45,7 +48,9 @@ class Lex(injector.Module): @injector.provider - def entry_uow_factory(self, container: injector.Injector) -> EntryRepositoryUnitOfWorkFactory: + def entry_uow_factory( + self, container: injector.Injector + ) -> EntryRepositoryUnitOfWorkFactory: return InjectorEntryUnitOfWorkRepoFactory(container) @injector.provider @@ -84,6 +89,17 @@ def deleting_resource( ) -> CommandHandler[commands.DeleteResource]: return DeletingResource(resource_uow) + @injector.provider + def setting_entry_repo_id( + self, + entry_repo_uow: EntryUowRepositoryUnitOfWork, + resource_uow: ResourceUnitOfWork, + ) -> CommandHandler[SetEntryRepoId]: + return SettingEntryRepoId( + entry_repo_uow=entry_repo_uow, + resource_uow=resource_uow, + ) + @injector.provider def update_resource( self, @@ -102,42 +118,30 @@ def publish_resource( def add_entry( self, resource_uow: ResourceUnitOfWork, - entry_repo_uow: EntryUowRepositoryUnitOfWork + entry_repo_uow: EntryUowRepositoryUnitOfWork, ) -> CommandHandler[commands.AddEntry]: - return AddingEntry( - resource_uow=resource_uow, - entry_repo_uow=entry_repo_uow - ) + return AddingEntry(resource_uow=resource_uow, entry_repo_uow=entry_repo_uow) @injector.provider def add_entries( self, resource_uow: ResourceUnitOfWork, - entry_repo_uow: EntryUowRepositoryUnitOfWork + entry_repo_uow: EntryUowRepositoryUnitOfWork, ) -> CommandHandler[commands.AddEntries]: - return AddingEntries( - resource_uow=resource_uow, - entry_repo_uow=entry_repo_uow - ) + return AddingEntries(resource_uow=resource_uow, entry_repo_uow=entry_repo_uow) @injector.provider def update_entry( self, resource_uow: ResourceUnitOfWork, - entry_repo_uow: EntryUowRepositoryUnitOfWork + entry_repo_uow: EntryUowRepositoryUnitOfWork, ) -> CommandHandler[commands.UpdateEntry]: - return UpdatingEntry( - resource_uow=resource_uow, - entry_repo_uow=entry_repo_uow - ) + return UpdatingEntry(resource_uow=resource_uow, entry_repo_uow=entry_repo_uow) @injector.provider def delete_entry( self, resource_uow: ResourceUnitOfWork, - entry_repo_uow: EntryUowRepositoryUnitOfWork + entry_repo_uow: EntryUowRepositoryUnitOfWork, ) -> CommandHandler[commands.DeleteEntry]: - return DeletingEntry( - resource_uow=resource_uow, - entry_repo_uow=entry_repo_uow - ) + return DeletingEntry(resource_uow=resource_uow, entry_repo_uow=entry_repo_uow) diff --git a/karp/lex/application/use_cases/__init__.py b/karp/lex/application/use_cases/__init__.py index 279eb95c..b5f68928 100644 --- a/karp/lex/application/use_cases/__init__.py +++ b/karp/lex/application/use_cases/__init__.py @@ -9,5 +9,6 @@ CreatingResource, DeletingResource, PublishingResource, + SettingEntryRepoId, UpdatingResource, ) diff --git a/karp/lex/application/use_cases/resource_handlers.py b/karp/lex/application/use_cases/resource_handlers.py index 0f57265f..85dda9ae 100644 --- a/karp/lex/application/use_cases/resource_handlers.py +++ b/karp/lex/application/use_cases/resource_handlers.py @@ -6,12 +6,10 @@ import logging -from karp import errors as karp_errors -from karp.lex.domain import errors, events, entities +from karp.lex.domain import errors, entities from karp.lex.domain.entities import Resource from karp.foundation import events as foundation_events from karp.foundation.commands import CommandHandler -from karp.foundation import messagebus from karp.lex.application.queries import ResourceDto from karp.lex.application import repositories as lex_repositories from karp.lex.domain import commands @@ -34,15 +32,13 @@ def get_field_translations(resource_id: str) -> Optional[Dict]: class BasingResource: - def __init__( - self, - resource_uow: repositories.ResourceUnitOfWork - ) -> None: + def __init__(self, resource_uow: repositories.ResourceUnitOfWork) -> None: self.resource_uow = resource_uow def collect_new_events(self) -> typing.Iterable[foundation_events.Event]: yield from self.resource_uow.collect_new_events() + # def get_resource(resource_id: str, version: Optional[int] = None) -> Resource: # if not version: # resource_def = database.get_active_resource_definition(resource_id) @@ -157,43 +153,46 @@ def create_resource_from_path(config: Path) -> List[Resource]: # def update_resource_from_file(config_file: BinaryIO) -> Tuple[str, int]: # return update_resource(config_file) + class CreatingResource(CommandHandler[commands.CreateResource], BasingResource): def __init__( self, resource_uow: repositories.ResourceUnitOfWork, - entry_repo_uow: lex_repositories.EntryUowRepositoryUnitOfWork + entry_repo_uow: lex_repositories.EntryUowRepositoryUnitOfWork, ) -> None: super().__init__(resource_uow=resource_uow) self.entry_repo_uow = entry_repo_uow - def execute(self, cmd: commands.CreateResource) -> ResourceDto: + def execute(self, command: commands.CreateResource) -> ResourceDto: with self.entry_repo_uow as uow: - entry_repo_exists = uow.repo.get_by_id_optional(cmd.entry_repo_id) + entry_repo_exists = uow.repo.get_by_id_optional(command.entry_repo_id) if not entry_repo_exists: raise errors.NoSuchEntryRepository( - f"Entry repository '{cmd.entry_repo_id}' not found") + f"Entry repository '{command.entry_repo_id}' not found" + ) with self.resource_uow as uow: existing_resource = uow.repo.get_by_resource_id_optional( - cmd.resource_id) + command.resource_id + ) if ( existing_resource and not existing_resource.discarded - and existing_resource.entity_id != cmd.entity_id + and existing_resource.entity_id != command.entity_id ): raise errors.IntegrityError( - f"Resource with resource_id='{cmd.resource_id}' already exists." + f"Resource with resource_id='{command.resource_id}' already exists." ) resource = entities.create_resource( - entity_id=cmd.entity_id, - resource_id=cmd.resource_id, - config=cmd.config, - message=cmd.message, - entry_repo_id=cmd.entry_repo_id, - created_at=cmd.timestamp, - created_by=cmd.user, - name=cmd.name, + entity_id=command.entity_id, + resource_id=command.resource_id, + config=command.config, + message=command.message, + entry_repo_id=command.entry_repo_id, + created_at=command.timestamp, + created_by=command.user, + name=command.name, ) uow.repo.save(resource) @@ -204,6 +203,36 @@ def collect_new_events(self) -> typing.Iterable[foundation_events.Event]: yield from self.resource_uow.collect_new_events() +class SettingEntryRepoId(CommandHandler[commands.SetEntryRepoId], BasingResource): + def __init__( + self, + resource_uow: repositories.ResourceUnitOfWork, + entry_repo_uow: lex_repositories.EntryUowRepositoryUnitOfWork, + ) -> None: + super().__init__(resource_uow=resource_uow) + self.entry_repo_uow = entry_repo_uow + + def execute(self, command: commands.SetEntryRepoId) -> None: + with self.entry_repo_uow as uow: + entry_repo_exists = uow.repo.get_by_id_optional(command.entry_repo_id) + if not entry_repo_exists: + raise errors.NoSuchEntryRepository( + f"Entry repository '{command.entry_repo_id}' not found" + ) + + with self.resource_uow as uow: + resource = uow.repo.get_by_resource_id(command.resource_id) + + resource.set_entry_repo_id( + entry_repo_id=command.entry_repo_id, + user=command.user, + timestamp=command.timestamp, + ) + + uow.repo.save(resource) + uow.commit() + + def setup_existing_resources(evt): with ctx.resource_uow: for resource_id in ctx.resource_uow.repo.resource_ids(): @@ -289,25 +318,26 @@ def create_new_resource(config_file: IO, config_dir=None) -> Resource: # return config + class UpdatingResource(CommandHandler[commands.UpdateResource], BasingResource): def __init__(self, resource_uow: repositories.ResourceUnitOfWork) -> None: super().__init__(resource_uow=resource_uow) - def execute(self, cmd: commands.UpdateResource): + def execute(self, command: commands.UpdateResource): with self.resource_uow as uow: - resource = uow.repo.by_resource_id(cmd.resource_id) + resource = uow.repo.by_resource_id(command.resource_id) found_changes = False - if resource.name != cmd.name: - resource.name = cmd.name + if resource.name != command.name: + resource.name = command.name found_changes = True - if resource.config != cmd.config: - resource.config = cmd.config + if resource.config != command.config: + resource.config = command.config found_changes = True if found_changes: resource.stamp( - user=cmd.user, - message=cmd.message, - timestamp=cmd.timestamp, + user=command.user, + message=command.message, + timestamp=command.timestamp, ) uow.repo.save(resource) uow.commit() @@ -401,6 +431,7 @@ def collect_new_events(self) -> typing.Iterable[foundation_events.Event]: # return resource_id, resource_def.version + class PublishingResource(CommandHandler[commands.PublishResource], BasingResource): def __init__( self, @@ -409,15 +440,15 @@ def __init__( ) -> None: super().__init__(resource_uow=resource_uow) - def execute(self, cmd: commands.PublishResource): - logger.info('publishing resource', extra={ - 'resource_id': cmd.resource_id}) + def execute(self, command: commands.PublishResource): + logger.info("publishing resource", extra={"resource_id": command.resource_id}) with self.resource_uow as uow: - resource = uow.repo.by_resource_id(cmd.resource_id) + resource = uow.repo.by_resource_id(command.resource_id) if not resource: - raise errors.ResourceNotFound(cmd.resource_id) - resource.publish(user=cmd.user, message=cmd.message, - timestamp=cmd.timestamp) + raise errors.ResourceNotFound(command.resource_id) + resource.publish( + user=command.user, message=command.message, timestamp=command.timestamp + ) uow.repo.save(resource) uow.commit() @@ -445,6 +476,7 @@ def collect_new_events(self) -> typing.Iterable[foundation_events.Event]: # db.session.commit() # remove_from_caches(resource_id) + class DeletingResource(CommandHandler[commands.DeleteResource], BasingResource): def __init__( self, @@ -453,15 +485,15 @@ def __init__( ) -> None: super().__init__(resource_uow=resource_uow) - def execute(self, cmd: commands.DeleteResource): - logger.info('deleting resource', extra={ - 'resource_id': cmd.resource_id}) + def execute(self, command: commands.DeleteResource): + logger.info("deleting resource", extra={"resource_id": command.resource_id}) with self.resource_uow as uow: - resource = uow.repo.by_resource_id(cmd.resource_id) + resource = uow.repo.by_resource_id(command.resource_id) if not resource: - raise errors.ResourceNotFound(cmd.resource_id) - resource.discard(user=cmd.user, message=cmd.message, - timestamp=cmd.timestamp) + raise errors.ResourceNotFound(command.resource_id) + resource.discard( + user=command.user, message=command.message, timestamp=command.timestamp + ) uow.repo.save(resource) uow.commit() diff --git a/karp/lex/domain/commands/__init__.py b/karp/lex/domain/commands/__init__.py index 8cbc77ff..12d6231c 100644 --- a/karp/lex/domain/commands/__init__.py +++ b/karp/lex/domain/commands/__init__.py @@ -8,4 +8,9 @@ ) from .entry_repo_commands import CreateEntryRepository, DeleteEntryRepository from .resource_commands import ( - CreateResource, DeleteResource, PublishResource, UpdateResource) + CreateResource, + DeleteResource, + PublishResource, + SetEntryRepoId, + UpdateResource, +) diff --git a/karp/lex/domain/commands/resource_commands.py b/karp/lex/domain/commands/resource_commands.py index 599b5269..70e2d2f0 100644 --- a/karp/lex/domain/commands/resource_commands.py +++ b/karp/lex/domain/commands/resource_commands.py @@ -11,7 +11,8 @@ class CreateResource(Command): entity_id: unique_id.UniqueId = pydantic.Field( - default_factory=unique_id.make_unique_id) + default_factory=unique_id.make_unique_id + ) resource_id: str name: str config: typing.Dict @@ -65,3 +66,9 @@ class DeleteResource(Command): resource_id: str message: str user: str + + +class SetEntryRepoId(Command): + resource_id: str + entry_repo_id: unique_id.UniqueId + user: str diff --git a/karp/lex/domain/entities/resource.py b/karp/lex/domain/entities/resource.py index 7f668e04..7ab1ed98 100644 --- a/karp/lex/domain/entities/resource.py +++ b/karp/lex/domain/entities/resource.py @@ -226,6 +226,32 @@ def publish( ) ) + def set_entry_repo_id( + self, + *, + entry_repo_id: unique_id.UniqueId, + user: str, + timestamp: Optional[float] = None, + ): + self._extracted_from_publish_9( + timestamp, user, "entry repo id updated", "entry repo id updated" + ) + self._increment_version() + self._entry_repo_id = entry_repo_id + self.queue_event( + events.ResourceUpdated( + entity_id=self.entity_id, + resource_id=self.resource_id, + entry_repo_id=self.entry_repository_id, + timestamp=self.last_modified, + user=self.last_modified_by, + version=self.version, + name=self.name, + config=self.config, + message=self.message, + ) + ) + def _extracted_from_publish_9(self, timestamp, user, message, arg3): self._check_not_discarded() self._last_modified = timestamp or utc_now() @@ -287,22 +313,21 @@ def entry_json_schema(self) -> Dict: def id_getter(self) -> Callable[[Dict], str]: return create_field_getter(self.config["id"], str) - def dict(self) -> Dict: return { - 'entity_id': self.entity_id, - 'resource_id': self.resource_id, - 'name': self.name, - 'version': self.version, - 'last_modified': self.last_modified, - 'last_modified_by': self.last_modified_by, - 'op': self.op, - 'message': self.message, - 'entry_repository_id': self.entry_repository_id, - 'is_published': self.is_published, - 'discarded': self.discarded, - 'resource_type': self.resource_type, - 'config': self.config, + "entity_id": self.entity_id, + "resource_id": self.resource_id, + "name": self.name, + "version": self.version, + "last_modified": self.last_modified, + "last_modified_by": self.last_modified_by, + "op": self.op, + "message": self.message, + "entry_repository_id": self.entry_repository_id, + "is_published": self.is_published, + "discarded": self.discarded, + "resource_type": self.resource_type, + "config": self.config, } def create_entry_from_dict( @@ -396,7 +421,7 @@ def create_resource( op=ResourceOp.ADDED, version=1, last_modified=created_at or time.utc_now(), - last_modified_by=user or created_by or 'unknown', + last_modified_by=user or created_by or "unknown", ) resource.queue_event( events.ResourceCreated( diff --git a/karp/tests/e2e/test_cliapp_resources.py b/karp/tests/e2e/test_cliapp_resources.py index e5dd6a35..35d89c4c 100644 --- a/karp/tests/e2e/test_cliapp_resources.py +++ b/karp/tests/e2e/test_cliapp_resources.py @@ -45,3 +45,38 @@ def test_create_and_publish_repo( resource_repo = app_context.container.get(lex.ReadOnlyResourceRepository) assert resource_repo.get_by_resource_id("lexlex") is not None + + def test_update_entry_repo( + self, + runner: CliRunner, + cliapp: Typer, + app_context: AppContext, + ): + result = runner.invoke( + cliapp, ["entry-repo", "create", "karp/tests/data/config/lexlex.json"] + ) + print(f"{result.stdout=}") + assert result.exit_code == 0 + + entry_repo_repo = app_context.container.get(lex.ReadOnlyEntryRepoRepositry) + + entry_repo = entry_repo_repo.get_by_name("lexlex") + assert entry_repo is not None + + result = runner.invoke( + cliapp, + [ + "resource", + "create", + "karp/tests/data/config/lexlex.json", + "--entry-repo-id", + str(entry_repo.entity_id), + ], + ) + print(f"{result.stdout=}") + assert result.exit_code == 0 + + resource_repo = app_context.container.get(lex.ReadOnlyResourceRepository) + + resource_lexlex = resource_repo.get_by_resource_id("lexlex") + assert resource_lexlex is not None From edea0287ab7169ecf6245e5dcb2812deaf2276b9 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Wed, 30 Mar 2022 11:05:37 +0200 Subject: [PATCH 22/43] build: remove old deploy --- deploy/requirements.txt | 116 ---------------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 deploy/requirements.txt diff --git a/deploy/requirements.txt b/deploy/requirements.txt deleted file mode 100644 index 029ffd12..00000000 --- a/deploy/requirements.txt +++ /dev/null @@ -1,116 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile --output-file=deploy/requirements.txt setup.py -# -alembic==1.4.1 - # via karp-tng-backend (setup.py) -asgiref==3.4.1 - # via uvicorn -attrs==19.3.0 - # via sb-json-tools -cffi==1.14.0 - # via cryptography -click==8.0.1 - # via - # karp-tng-backend (setup.py) - # sb-json-tools - # typer - # uvicorn -cryptography==3.4.8 - # via karp-tng-backend (setup.py) -dependency-injector==4.36.0 - # via karp-tng-backend (setup.py) -elasticsearch==6.4.0 - # via - # elasticsearch-dsl - # karp-tng-backend (setup.py) -elasticsearch-dsl==6.4.0 - # via karp-tng-backend (setup.py) -environs==9.3.3 - # via karp-tng-backend (setup.py) -fastapi==0.68.1 - # via karp-tng-backend (setup.py) -fastjsonschema==2.14.3 - # via - # karp-tng-backend (setup.py) - # sb-json-tools -gunicorn==20.0.4 - # via karp-tng-backend (setup.py) -h11==0.12.0 - # via uvicorn -ijson==2.6.1 - # via - # json-streams - # sb-json-tools -json-streams==0.10.1 - # via karp-tng-backend (setup.py) -mako==1.1.2 - # via alembic -markupsafe==1.1.1 - # via mako -marshmallow==3.13.0 - # via environs -paradigmextract==0.1.1 - # via karp-tng-backend (setup.py) -pycparser==2.20 - # via cffi -pydantic==1.8.2 - # via - # fastapi - # karp-tng-backend (setup.py) -pyjwt==1.7.1 - # via karp-tng-backend (setup.py) -python-dateutil==2.8.1 - # via - # alembic - # elasticsearch-dsl -python-dotenv==0.19.0 - # via environs -python-editor==1.0.4 - # via alembic -sb-json-tools==0.5.2 - # via karp-tng-backend (setup.py) -simplejson==3.17.0 - # via - # json-streams - # sb-json-tools -six==1.14.0 - # via - # dependency-injector - # elasticsearch-dsl - # python-dateutil - # sqlalchemy-json - # sqlalchemy-utils -sqlalchemy==1.3.13 - # via - # alembic - # karp-tng-backend (setup.py) - # sqlalchemy-json - # sqlalchemy-utils -sqlalchemy-json==0.4.0 - # via karp-tng-backend (setup.py) -sqlalchemy-utils==0.37.8 - # via karp-tng-backend (setup.py) -starlette==0.14.2 - # via fastapi -tabulate==0.8.9 - # via karp-tng-backend (setup.py) -tenacity==8.0.1 - # via karp-tng-backend (setup.py) -tqdm==4.62.2 - # via karp-tng-backend (setup.py) -typer==0.4.0 - # via karp-tng-backend (setup.py) -typing-extensions==3.10.0.2 - # via pydantic -urllib3==1.26.6 - # via - # elasticsearch - # karp-tng-backend (setup.py) -uvicorn==0.15.0 - # via karp-tng-backend (setup.py) - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From a1afef8e74665f3918734f0ffcb130ce500ccfe4 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 09:39:26 +0200 Subject: [PATCH 23/43] stuff --- karp/cliapp/subapps/resource_subapp.py | 28 ++--- karp/foundation/entity.py | 61 ++++------- karp/foundation/repository.py | 9 +- karp/lex/application/queries/entries.py | 16 +-- karp/lex/application/repositories/entries.py | 24 ++-- .../queries/generic_entries.py | 97 +++++++++-------- .../transformers/entry_transformer.py | 2 +- karp/search/application/use_cases.py | 103 ++++++++---------- .../transformers/generic_entry_transformer.py | 90 ++++++++------- karp/tests/e2e/test_cliapp_resources.py | 8 +- karp/tests/unit/lex/adapters.py | 67 ++++++++---- karp/tests/unit/test_json_schema.py | 67 +++++++++++- 12 files changed, 322 insertions(+), 250 deletions(-) diff --git a/karp/cliapp/subapps/resource_subapp.py b/karp/cliapp/subapps/resource_subapp.py index 49b76170..499c7a87 100644 --- a/karp/cliapp/subapps/resource_subapp.py +++ b/karp/cliapp/subapps/resource_subapp.py @@ -99,20 +99,20 @@ def set_entry_repo( @cli_timer def update(ctx: typer.Context, config: Path): bus = inject_from_ctx(CommandBus, ctx) - if config.is_file(): - with open(config) as fp: - new_resource = resources.update_resource_from_file(fp) - new_resources = [new_resource] - elif config.is_dir(): - new_resources = resources.update_resource_from_dir(config) - else: - typer.echo("Must give either --config or --config-dir") - raise typer.Exit(3) # Usage error - for (resource_id, version) in new_resources: - if version is None: - typer.echo(f"Nothing to do for resource '{resource_id}'") - else: - typer.echo(f"Updated version {version} of resource '{resource_id}'") + # if config.is_file(): + # with open(config) as fp: + # new_resource = resources.update_resource_from_file(fp) + # new_resources = [new_resource] + # elif config.is_dir(): + # new_resources = resources.update_resource_from_dir(config) + # else: + # typer.echo("Must give either --config or --config-dir") + # raise typer.Exit(3) # Usage error + # for (resource_id, version) in new_resources: + # if version is None: + # typer.echo(f"Nothing to do for resource '{resource_id}'") + # else: + # typer.echo(f"Updated version {version} of resource '{resource_id}'") @subapp.command() diff --git a/karp/foundation/entity.py b/karp/foundation/entity.py index 2c7d5d48..ecd1c5e5 100644 --- a/karp/foundation/entity.py +++ b/karp/foundation/entity.py @@ -56,7 +56,6 @@ def _validate_event_applicability(self, event): class VersionedEntity(Entity): - def __init__(self, entity_id, version: int, discarded: bool = False): super().__init__(entity_id, discarded=discarded) self._version = version @@ -89,7 +88,6 @@ def _validate_version(self, version: int) -> None: class TimestampedEntity(Entity): - def __init__( self, entity_id, @@ -108,30 +106,18 @@ def last_modified(self): """The time this entity was last modified.""" return self._last_modified - @last_modified.setter - @deprecated(version='6.0.7', reason='use update') - def last_modified(self, timestamp: float): - self._check_not_discarded() - self._last_modified = timestamp - @property def last_modified_by(self): """The time this entity was last modified.""" return self._last_modified_by - @last_modified_by.setter - @deprecated(version='6.0.7', reason='use update') - def last_modified_by(self, user): - self._check_not_discarded() - self._last_modified_by = user - - @deprecated(version='6.0.7', reason='use update') + @deprecated(version="6.0.7", reason="use update") def stamp(self, user, *, timestamp: float = None): self._check_not_discarded() self._last_modified_by = user self._last_modified = monotonic_utc_now() if timestamp is None else timestamp - def discard(self, *, user, timestamp: float = None): + def discard(self, *, user, timestamp: Optional[float] = None): self._check_not_discarded() # self._validate_last_modified(last_modified) super().discard() @@ -156,7 +142,6 @@ def _validate_last_modified(self, last_modified: float): class TimestampedVersionedEntity(VersionedEntity, TimestampedEntity): - def __init__( self, entity_id, @@ -175,27 +160,27 @@ def __init__( last_modified_by=last_modified_by, ) -# @property -# def last_modified(self): -# """The time this entity was last modified.""" -# return self._last_modified -# -# @last_modified.setter -# def last_modified(self, timestamp: float): -# self._check_not_discarded() -# self._last_modified = timestamp -# -# @property -# def last_modified_by(self): -# """The time this entity was last modified.""" -# return self._last_modified_by -# -# @last_modified_by.setter -# def last_modified_by(self, user: str): -# self._check_not_discarded() -# self._last_modified_by = user - - @deprecated(version='6.0.7', reason='use update') + # @property + # def last_modified(self): + # """The time this entity was last modified.""" + # return self._last_modified + # + # @last_modified.setter + # def last_modified(self, timestamp: float): + # self._check_not_discarded() + # self._last_modified = timestamp + # + # @property + # def last_modified_by(self): + # """The time this entity was last modified.""" + # return self._last_modified_by + # + # @last_modified_by.setter + # def last_modified_by(self, user: str): + # self._check_not_discarded() + # self._last_modified_by = user + + @deprecated(version="6.0.7", reason="use update") def stamp(self, user, *, timestamp: float = None, increment_version: bool = True): self._check_not_discarded() diff --git a/karp/foundation/repository.py b/karp/foundation/repository.py index 760db194..afb3e363 100644 --- a/karp/foundation/repository.py +++ b/karp/foundation/repository.py @@ -29,7 +29,7 @@ def _check_id_has_correct_type(self, id_) -> None: ) def by_id( - self, id_: unique_id.typing_UniqueId, *, version: Optional[int] = None + self, id_: unique_id.typing_UniqueId, *, version: Optional[int] = None, **kwargs ) -> EntityType: self._check_id_has_correct_type(id_) entity = self._by_id(id_, version=version) @@ -42,10 +42,7 @@ def by_id( get_by_id = by_id def get_by_id_optional( - self, - id_: unique_id.typing_UniqueId, - *, - version: Optional[int] = None, + self, id_: unique_id.typing_UniqueId, *, version: Optional[int] = None, **kwargs ) -> Optional[EntityType]: self._check_id_has_correct_type(id_) entity = self._by_id(id_, version=version) @@ -55,7 +52,7 @@ def get_by_id_optional( @abc.abstractmethod def _by_id( - self, id_: Union[uuid.UUID, str], *, version: Optional[int] = None + self, id_: Union[uuid.UUID, str], *, version: Optional[int] = None, **kwargs ) -> Optional[EntityType]: raise NotImplementedError() diff --git a/karp/lex/application/queries/entries.py b/karp/lex/application/queries/entries.py index 2b2a19d6..f688b46d 100644 --- a/karp/lex/application/queries/entries.py +++ b/karp/lex/application/queries/entries.py @@ -6,7 +6,10 @@ from karp import errors as karp_errors from karp.foundation.value_objects import unique_id -from karp.lex.application.repositories import ResourceUnitOfWork, EntryUowRepositoryUnitOfWork +from karp.lex.application.repositories import ( + ResourceUnitOfWork, + EntryUowRepositoryUnitOfWork, +) from karp.lex.domain.entities.entry import EntryOp @@ -61,7 +64,7 @@ class HistoryDto(pydantic.BaseModel): class GetEntryDiff(abc.ABC): @abc.abstractmethod - def query(self, req: EntryDiffRequest) -> EntryDiffDto: + def query(self, request: EntryDiffRequest) -> EntryDiffDto: pass @@ -83,10 +86,7 @@ class GetHistoryDto(pydantic.BaseModel): class GetHistory(abc.ABC): @abc.abstractmethod - def query( - self, - req: EntryHistoryRequest - ) -> GetHistoryDto: + def query(self, request: EntryHistoryRequest) -> GetHistoryDto: pass @@ -128,7 +128,9 @@ def get_total(self, resource_id: str) -> int: pass @abc.abstractmethod - def get_by_referenceable(self, resource_id: str, filters) -> typing.Iterable[EntryDto]: + def get_by_referenceable( + self, resource_id: str, filters + ) -> typing.Iterable[EntryDto]: pass @abc.abstractmethod diff --git a/karp/lex/application/repositories/entries.py b/karp/lex/application/repositories/entries.py index 24d4df93..7c4ba672 100644 --- a/karp/lex/application/repositories/entries.py +++ b/karp/lex/application/repositories/entries.py @@ -34,6 +34,7 @@ def by_id( after_date: Optional[float] = None, before_date: Optional[float] = None, oldest_first: bool = False, + **kwargs, ) -> entities.Entry: entry = self._by_id( id_, @@ -59,13 +60,13 @@ def _by_id( ) -> typing.Optional[entities.Entry]: raise NotImplementedError() - # @abc.abstractmethod - def move(self, entry: entities.Entry, *, old_entry_id: str): - raise NotImplementedError() + # # @abc.abstractmethod + # def move(self, entry: entities.Entry, *, old_entry_id: str): + # raise NotImplementedError() - # @abc.abstractmethod - def delete(self, entry: entities.Entry): - raise NotImplementedError() + # # @abc.abstractmethod + # def delete(self, entry: entities.Entry): + # raise NotImplementedError() # @abc.abstractmethod def entry_ids(self) -> List[str]: @@ -115,9 +116,11 @@ def teardown(self): """Use for testing purpose.""" return - # @abc.abstractmethod - # def by_referenceable(self, filters: Optional[Dict] = None, **kwargs) -> List[Entry]: - # raise NotImplementedError() + @abc.abstractmethod + def by_referenceable( + self, filters: Optional[Dict] = None, **kwargs + ) -> list[entities.Entry]: + raise NotImplementedError() # @abc.abstractmethod def get_history( @@ -159,8 +162,7 @@ def __init__( **kwargs, ): unit_of_work.UnitOfWork.__init__(self, event_bus) - entity.TimestampedEntity.__init__( - self, *args, **kwargs) + entity.TimestampedEntity.__init__(self, *args, **kwargs) self._name = name self._connection_str = connection_str self._config = config diff --git a/karp/lex_infrastructure/queries/generic_entries.py b/karp/lex_infrastructure/queries/generic_entries.py index b06fae38..dd69d15d 100644 --- a/karp/lex_infrastructure/queries/generic_entries.py +++ b/karp/lex_infrastructure/queries/generic_entries.py @@ -7,7 +7,17 @@ from karp.lex import GetHistoryDto, HistoryDto from karp.lex.domain.entities import Entry -from karp.lex.application.queries import EntryViews, EntryDto, EntryDiffDto, GetEntryDiff, GetEntryHistory, GetHistory, EntryHistoryRequest, EntryDiffRequest, GetEntryRepositoryId +from karp.lex.application.queries import ( + EntryViews, + EntryDto, + EntryDiffDto, + GetEntryDiff, + GetEntryHistory, + GetHistory, + EntryHistoryRequest, + EntryDiffRequest, + GetEntryRepositoryId, +) from karp.foundation.value_objects import unique_id from karp.lex.application.repositories import EntryUowRepositoryUnitOfWork @@ -30,15 +40,10 @@ def get_by_id(self, resource_id: str, entity_id: unique_id.UniqueId) -> EntryDto with self.entry_repo_uow as uw: entry_uow = uw.repo.get_by_id(entry_repo_id) with entry_uow as uw: - return self._entry_to_entry_dto( - uw.repo.by_id(entity_id), - resource_id - ) + return self._entry_to_entry_dto(uw.repo.by_id(entity_id), resource_id) def get_by_id_optional( - self, - resource_id: str, - entity_id: unique_id.UniqueId + self, resource_id: str, entity_id: unique_id.UniqueId ) -> typing.Optional[EntryDto]: entry_repo_id = self.get_entry_repo_id.query(resource_id) with self.entry_repo_uow as uw: @@ -54,15 +59,10 @@ def get_by_entry_id(self, resource_id: str, entry_id: str) -> EntryDto: with self.entry_repo_uow as uw: entry_uow = uw.repo.get_by_id(entry_repo_id) with entry_uow as uw: - return self._entry_to_entry_dto( - uw.repo.by_entry_id(entry_id), - resource_id - ) + return self._entry_to_entry_dto(uw.repo.by_entry_id(entry_id), resource_id) def get_by_entry_id_optional( - self, - resource_id: str, - entry_id: str + self, resource_id: str, entry_id: str ) -> typing.Optional[EntryDto]: entry_repo_id = self.get_entry_repo_id.query(resource_id) with self.entry_repo_uow as uw: @@ -134,7 +134,9 @@ def query( version: typing.Optional[int], ) -> EntryDto: entry_repo_id = self.get_entry_repo_id(resource_id) - with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id(entry_repo_id) as uw: + with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id( + entry_repo_id + ) as uw: result = uw.repo.by_entry_id(entry_id, version=version) return EntryDto( @@ -151,21 +153,22 @@ def query( class GenericGetHistory(GenericEntryQuery, GetHistory): def query( self, - history_request: EntryHistoryRequest, + request: EntryHistoryRequest, ) -> GetHistoryDto: - logger.info('querying history', extra={ - 'history_request': history_request}) - entry_repo_id = self.get_entry_repo_id(history_request.resource_id) - with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id(entry_repo_id) as uw: + logger.info("querying history", extra={"request": request}) + entry_repo_id = self.get_entry_repo_id(request.resource_id) + with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id( + entry_repo_id + ) as uw: paged_query, total = uw.repo.get_history( - entry_id=history_request.entry_id, - user_id=history_request.user_id, - from_date=history_request.from_date, - to_date=history_request.to_date, - from_version=history_request.from_version, - to_version=history_request.to_version, - offset=history_request.current_page * history_request.page_size, - limit=history_request.page_size, + entry_id=request.entry_id, + user_id=request.user_id, + from_date=request.from_date, + to_date=request.to_date, + from_version=request.from_version, + to_version=request.to_version, + offset=request.current_page * request.page_size, + limit=request.page_size, ) result = [] previous_body = {} @@ -180,7 +183,7 @@ def query( # else: # previous_body = {} history_diff = jsondiff.compare(previous_body, history_entry.body) - logger.info('diff', extra={'diff': history_diff}) + logger.info("diff", extra={"diff": history_diff}) result.append( HistoryDto( timestamp=history_entry.last_modified, @@ -200,40 +203,38 @@ def query( class GenericGetEntryDiff(GenericEntryQuery, GetEntryDiff): def query( self, - diff_request: EntryDiffRequest, + request: EntryDiffRequest, ) -> EntryDiffDto: - entry_repo_id = self.get_entry_repo_id(diff_request.resource_id) - with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id(entry_repo_id) as uw: - db_entry = uw.repo.by_entry_id(diff_request.entry_id) + entry_repo_id = self.get_entry_repo_id(request.resource_id) + with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id( + entry_repo_id + ) as uw: + db_entry = uw.repo.by_entry_id(request.entry_id) # src = resource_obj.model.query.filter_by(entry_id=entry_id).first() # # query = resource_obj.history_model.query.filter_by(entry_id=src.id) # timestamp_field = resource_obj.history_model.timestamp # - if diff_request.from_version: - obj1 = uw.repo.by_id( - db_entry.id, version=diff_request.from_version) - elif diff_request.from_date is not None: - obj1 = uw.repo.by_id( - db_entry.id, after_date=diff_request.from_date) + if request.from_version: + obj1 = uw.repo.by_id(db_entry.id, version=request.from_version) + elif request.from_date is not None: + obj1 = uw.repo.by_id(db_entry.id, after_date=request.from_date) else: obj1 = uw.repo.by_id(db_entry.id, oldest_first=True) obj1_body = obj1.body if obj1 else None - if diff_request.to_version: - obj2 = uw.repo.by_id( - db_entry.id, version=diff_request.to_version) + if request.to_version: + obj2 = uw.repo.by_id(db_entry.id, version=request.to_version) obj2_body = obj2.body - elif diff_request.to_date is not None: - obj2 = uw.repo.by_id( - db_entry.id, before_date=diff_request.to_date) + elif request.to_date is not None: + obj2 = uw.repo.by_id(db_entry.id, before_date=request.to_date) obj2_body = obj2.body - elif diff_request.entry is not None: + elif request.entry is not None: obj2 = None - obj2_body = diff_request.entry + obj2_body = request.entry else: obj2 = db_entry obj2_body = db_entry.body diff --git a/karp/search/application/transformers/entry_transformer.py b/karp/search/application/transformers/entry_transformer.py index a532d692..810eec53 100644 --- a/karp/search/application/transformers/entry_transformer.py +++ b/karp/search/application/transformers/entry_transformer.py @@ -9,6 +9,6 @@ class EntryTransformer(abc.ABC): def transform( self, resource_id: str, - entry: lex.EntryDto, + src_entry: lex.EntryDto, ) -> IndexEntry: pass diff --git a/karp/search/application/use_cases.py b/karp/search/application/use_cases.py index 2fa0c381..136e5ec7 100644 --- a/karp/search/application/use_cases.py +++ b/karp/search/application/use_cases.py @@ -1,6 +1,7 @@ import collections import json import logging + # from karp.infrastructure.unit_of_work import unit_of_work import sys import typing @@ -16,16 +17,20 @@ from karp.search.application.queries import ResourceViews from karp.search.application.repositories import IndexUnitOfWork from karp.search.application.transformers import EntryTransformer, PreProcessor + # from .search_service import SearchServiceModule # import karp.resourcemgr as resourcemgr # import karp.resourcemgr.entryread as entryread # from karp.resourcemgr.resource import Resource from karp import errors as karp_errors from karp.lex.domain import events, entities + # , errors, events, search_service, model from karp.search.domain import commands + # from karp.search.domain.search_service import SearchService, IndexEntry from karp.lex.domain.entities.entry import Entry, create_entry + # from karp.domain.models.resource import Resource # from karp.domain.repository import ResourceRepository # from karp.services import context, network_handlers @@ -95,9 +100,7 @@ # ): -class ReindexingResource( - foundation_commands.CommandHandler[commands.ReindexResource] -): +class ReindexingResource(foundation_commands.CommandHandler[commands.ReindexResource]): def __init__( self, index_uow: IndexUnitOfWork, @@ -109,19 +112,15 @@ def __init__( self.resource_views = resource_views self.pre_processor = pre_processor - def execute( - self, - cmd: commands.ReindexResource - ) -> None: - logger.debug("Reindexing resource '%s'", cmd.resource_id) + def execute(self, command: commands.ReindexResource) -> None: + logger.debug("Reindexing resource '%s'", command.resource_id) with self.index_uow as uw: uw.repo.create_index( - cmd.resource_id, - self.resource_views.get_resource_config(cmd.resource_id), + command.resource_id, + self.resource_views.get_resource_config(command.resource_id), ) uw.repo.add_entries( - cmd.resource_id, - self.pre_processor.process(cmd.resource_id) + command.resource_id, self.pre_processor.process(command.resource_id) ) uw.commit() @@ -131,12 +130,12 @@ def research_service( ): print("creating search_service ...") search_service_name = search_serviceer.create_search_service( - resource.resource_id, resource.config) + resource.resource_id, resource.config + ) if not search_entries: print("preprocessing entries ...") - search_entries = pre_process_resource( - resource, resource_repo, search_serviceer) + search_entries = pre_process_resource(resource, resource_repo, search_serviceer) print(f"adding entries to '{search_service_name}' ...") # add_entries( # resource_repo, @@ -148,21 +147,21 @@ def research_service( # ) search_serviceer.add_entries(search_service_name, search_entries) print("publishing ...") - search_serviceer.publish_search_service( - resource.resource_id, search_service_name) + search_serviceer.publish_search_service(resource.resource_id, search_service_name) # def publish_search_service(resource_id: str, version: Optional[int] = None) -> None: -class ResourcePublishedHandler(foundation_events.EventHandler[lex_events.ResourcePublished]): - def __init__( - self, - index_uow: IndexUnitOfWork - ): +class ResourcePublishedHandler( + foundation_events.EventHandler[lex_events.ResourcePublished] +): + def __init__(self, index_uow: IndexUnitOfWork): self.index_uow = index_uow def __call__( self, evt: events.ResourcePublished, + *args, + **kwargs, ) -> None: # research_service(evt, ctx) with self.index_uow as uw: @@ -172,11 +171,10 @@ def __call__( # resourcemgr.publish_resource(resource_id, version) -class CreateSearchServiceHandler(foundation_events.EventHandler[lex_events.ResourceCreated]): - def __init__( - self, - index_uow: IndexUnitOfWork - ): +class CreateSearchServiceHandler( + foundation_events.EventHandler[lex_events.ResourceCreated] +): + def __init__(self, index_uow: IndexUnitOfWork): self.index_uow = index_uow def collect_new_events(self) -> Iterable[foundation_events.Event]: @@ -189,10 +187,7 @@ def __call__(self, event: events.ResourceCreated, *args, **kwargs): class DeletingIndex(foundation_events.EventHandler[lex_events.ResourceDiscarded]): - def __init__( - self, - index_uow: IndexUnitOfWork - ): + def __init__(self, index_uow: IndexUnitOfWork): self.index_uow = index_uow def collect_new_events(self) -> Iterable[foundation_events.Event]: @@ -201,6 +196,7 @@ def collect_new_events(self) -> Iterable[foundation_events.Event]: def __call__(self, event: events.ResourceDiscarded, *args, **kwargs): pass + # def add_entries( # resource_id: str, # entries: List[Tuple[str, EntryMetadata, Dict]], @@ -225,6 +221,8 @@ def __init__( def __call__( self, evt: events.EntryAdded, + *args, + **kwargs, ): with self.index_uow as uw: @@ -241,17 +239,13 @@ def __call__( version=1, ) uw.repo.add_entries( - resource_id, - [self.entry_transformer.transform(resource_id, entry)] + resource_id, [self.entry_transformer.transform(resource_id, entry)] ) - self.entry_transformer.update_references( - resource_id, [evt.entry_id]) + self.entry_transformer.update_references(resource_id, [evt.entry_id]) uw.commit() -class EntryUpdatedHandler( - foundation_events.EventHandler[lex_events.EntryUpdated] -): +class EntryUpdatedHandler(foundation_events.EventHandler[lex_events.EntryUpdated]): def __init__( self, index_uow: IndexUnitOfWork, @@ -265,6 +259,8 @@ def __init__( def __call__( self, evt: events.EntryUpdated, + *args, + **kwargs, ): with self.index_uow as uw: @@ -281,11 +277,9 @@ def __call__( version=evt.version, ) uw.repo.add_entries( - resource_id, - [self.entry_transformer.transform(resource_id, entry)] + resource_id, [self.entry_transformer.transform(resource_id, entry)] ) - self.entry_transformer.update_references( - resource_id, [evt.entry_id]) + self.entry_transformer.update_references(resource_id, [evt.entry_id]) uw.commit() # add_entries(evt.resource_id, [entry], ctx) # ctx.index_uow.commit() @@ -317,8 +311,10 @@ def add_entries( raise errors.ResourceNotFound(resource_id) ctx.index_uow.repo.add_entries( search_service_name, - [transform_to_search_service_entry(resource, entry, ctx) - for entry in entries], + [ + transform_to_search_service_entry(resource, entry, ctx) + for entry in entries + ], ) if update_refs: _update_references(resource, entries, ctx) @@ -326,9 +322,7 @@ def add_entries( ctx.index_uow.commit() -class EntryDeletedHandler( - foundation_events.EventHandler[lex_events.EntryDeleted] -): +class EntryDeletedHandler(foundation_events.EventHandler[lex_events.EntryDeleted]): def __init__( self, index_uow: IndexUnitOfWork, @@ -339,18 +333,9 @@ def __init__( self.entry_transformer = entry_transformer self.resource_views = resource_views - def __call__( - self, - evt: events.EntryDeleted - ): + def __call__(self, evt: events.EntryDeleted): with self.index_uow as uw: for resource_id in self.resource_views.get_resource_ids(evt.repo_id): - uw.repo.delete_entry( - resource_id, - entry_id=evt.entry_id - ) - self.entry_transformer.update_references( - resource_id, - [evt.entry_id] - ) + uw.repo.delete_entry(resource_id, entry_id=evt.entry_id) + self.entry_transformer.update_references(resource_id, [evt.entry_id]) uw.commit() diff --git a/karp/search_infrastructure/transformers/generic_entry_transformer.py b/karp/search_infrastructure/transformers/generic_entry_transformer.py index 4f3b7a11..01f1a4cc 100644 --- a/karp/search_infrastructure/transformers/generic_entry_transformer.py +++ b/karp/search_infrastructure/transformers/generic_entry_transformer.py @@ -13,7 +13,10 @@ EntryViews, EntryDto, ) -from karp.lex.application.repositories import ResourceUnitOfWork, EntryUowRepositoryUnitOfWork +from karp.lex.application.repositories import ( + ResourceUnitOfWork, + EntryUowRepositoryUnitOfWork, +) from karp.search.application.transformers import EntryTransformer from karp.search.application.repositories import IndexUnitOfWork, IndexEntry @@ -42,12 +45,15 @@ def transform(self, resource_id: str, src_entry: EntryDto) -> IndexEntry: TODO somehow get the needed entries in bulk after transforming some entries and insert them into TODO the transformed entries afterward. Very tricky. """ - logger.debug('transforming entry', - extra={'entry_id': src_entry.entry_id, 'resource_id': resource_id}) + logger.debug( + "transforming entry", + extra={"entry_id": src_entry.entry_id, "resource_id": resource_id}, + ) index_entry = self.index_uow.repo.create_empty_object() index_entry.id = src_entry.entry_id self.index_uow.repo.assign_field( - index_entry, "_entry_version", src_entry.version) + index_entry, "_entry_version", src_entry.version + ) self.index_uow.repo.assign_field( index_entry, "_last_modified", src_entry.last_modified ) @@ -81,11 +87,12 @@ def _transform_to_index_entry( if field_conf.get("virtual"): logger.debug("found virtual field") res = self._evaluate_function( - field_conf["function"], _src_entry, resource) - logger.debug(f"res = {res}") + field_conf["function"], _src_entry, resource + ) if res: self.index_uow.repo.assign_field( - _index_entry, "v_" + field_name, res) + _index_entry, "v_" + field_name, res + ) elif field_conf.get("collection"): field_content = self.index_uow.repo.create_empty_list() if field_name in _src_entry: @@ -99,10 +106,12 @@ def _transform_to_index_entry( field_conf["fields"].items(), ) self.index_uow.repo.add_to_list_field( - field_content, subfield_content.entry) + field_content, subfield_content.entry + ) else: self.index_uow.repo.add_to_list_field( - field_content, subfield) + field_content, subfield + ) elif field_conf["type"] == "object": field_content = self.index_uow.repo.create_empty_object() if field_name in _src_entry: @@ -118,15 +127,18 @@ def _transform_to_index_entry( if field_content: self.index_uow.repo.assign_field( - _index_entry, field_name, field_content) + _index_entry, field_name, field_content + ) # Handle ref if field_conf.get("ref") and field_name in _src_entry: res = self._resolve_ref( - resource, _src_entry, field_conf["ref"], field_name) + resource, _src_entry, field_conf["ref"], field_name + ) if res: self.index_uow.repo.assign_field( - _index_entry, f"v_{field_name}", res) + _index_entry, f"v_{field_name}", res + ) def _resolve_ref( self, @@ -136,10 +148,9 @@ def _resolve_ref( field_name: str, ): res = None - if 'resource_id' in ref_conf: + if "resource_id" in ref_conf: ref_resource = self.resource_repo.get_by_resource_id( - ref_conf["resource_id"], - version=ref_conf.get("resource_version") + ref_conf["resource_id"], version=ref_conf.get("resource_version") ) else: ref_resource = resource @@ -147,26 +158,28 @@ def _resolve_ref( if not ref_resource: return res - if ref_conf['field'].get('collection'): + if ref_conf["field"].get("collection"): res = self.index_uow.repo.create_empty_list() for ref_id in src_entry[field_name]: ref_entry_body = self.entry_views.get_by_entry_id_optional( - ref_resource.resource_id, str(ref_id)) + ref_resource.resource_id, str(ref_id) + ) if ref_entry_body: # ref_entry = { # field_name: ref_entry_body.entry # } ref_entry = ref_entry_body.entry - if ref_conf['field']['type'] == 'object': + if ref_conf["field"]["type"] == "object": ref_index_entry = self.index_uow.repo.create_empty_object() - for ref_field_name, _ref_field_conf in ref_conf['field']['fields'].items(): + for ref_field_name, _ref_field_conf in ref_conf["field"][ + "fields" + ].items(): self.index_uow.repo.assign_field( ref_index_entry, ref_field_name, ref_entry[ref_field_name], ) - self.index_uow.repo.add_to_list_field( - res, ref_index_entry) + self.index_uow.repo.add_to_list_field(res, ref_index_entry) # ref_objs = [] # if ref_resource: # for ref_id in _src_entry[field_name]: @@ -202,12 +215,12 @@ def _resolve_ref( # for elem in ref_id: ref = self.entry_views.get_by_entry_id_optional( - ref_resource.resource_id, str(src_entry[field_name])) + ref_resource.resource_id, str(src_entry[field_name]) + ) if ref: ref_entry = {field_name: ref.entry} ref_index_entry = self.index_uow.repo.create_empty_object() - list_of_sub_fields = ( - (field_name, ref_conf['field']),) + list_of_sub_fields = ((field_name, ref_conf["field"]),) self._transform_to_index_entry( ref_resource, ref_entry, @@ -246,18 +259,21 @@ def _evaluate_function( src_resource: ResourceDto, ): logger.debug( - "indexing._evaluate_function", extra={'src_resource': src_resource.resource_id}) - logger.debug('indexing._evaluate_function', - extra={'src_entry': src_entry}) + "indexing._evaluate_function", + extra={"src_resource": src_resource.resource_id}, + ) + logger.debug("indexing._evaluate_function", extra={"src_entry": src_entry}) if "multi_ref" in function_conf: function_conf = function_conf["multi_ref"] target_field = function_conf["field"] if "resource_id" in function_conf: logger.debug( - "indexing._evaluate_function: trying to find '%s'", function_conf['resource_id'] + "indexing._evaluate_function: trying to find '%s'", + function_conf["resource_id"], ) target_resource = self.resource_repo.get_by_resource_id( - function_conf["resource_id"], version=function_conf["resource_version"] + function_conf["resource_id"], + version=function_conf["resource_version"], ) if target_resource is None: logger.warning( @@ -268,7 +284,8 @@ def _evaluate_function( else: target_resource = src_resource logger.debug( - "indexing._evaluate_function target_resource='%s'", target_resource.resource_id + "indexing._evaluate_function target_resource='%s'", + target_resource.resource_id, ) if "test" in function_conf: operator, args = list(function_conf["test"].items())[0] @@ -283,7 +300,8 @@ def _evaluate_function( # target_resource, filters # ) target_entries = self.entry_views.get_by_referenceable( - target_resource.resource_id, filters) + target_resource.resource_id, filters + ) else: raise NotImplementedError() else: @@ -301,8 +319,7 @@ def _evaluate_function( index_entry, list_of_sub_fields, ) - self.index_uow.repo.add_to_list_field( - res, index_entry.entry["tmp"]) + self.index_uow.repo.add_to_list_field(res, index_entry.entry["tmp"]) elif "plugin" in function_conf: plugin_id = function_conf["plugin"] import karp.pluginmanager as plugins @@ -324,13 +341,10 @@ def update_references( ) -> None: add = collections.defaultdict(list) for src_entry_id in entry_ids: - refs = self.get_referenced_entries.query( - resource_id, src_entry_id - ) + refs = self.get_referenced_entries.query(resource_id, src_entry_id) for field_ref in refs: ref_resource = self.resource_repo.get_by_resource_id( - field_ref.resource_id, - version=(field_ref.resource_version) + field_ref.resource_id, version=(field_ref.resource_version) ) if ref_resource: ref_index_entry = self.transform( diff --git a/karp/tests/e2e/test_cliapp_resources.py b/karp/tests/e2e/test_cliapp_resources.py index 35d89c4c..4af1e31a 100644 --- a/karp/tests/e2e/test_cliapp_resources.py +++ b/karp/tests/e2e/test_cliapp_resources.py @@ -46,6 +46,7 @@ def test_create_and_publish_repo( assert resource_repo.get_by_resource_id("lexlex") is not None + @pytest.mark.xfail(reason="not ready") def test_update_entry_repo( self, runner: CliRunner, @@ -67,9 +68,8 @@ def test_update_entry_repo( cliapp, [ "resource", - "create", - "karp/tests/data/config/lexlex.json", - "--entry-repo-id", + "set-entry-repo", + "lexlex", str(entry_repo.entity_id), ], ) @@ -79,4 +79,4 @@ def test_update_entry_repo( resource_repo = app_context.container.get(lex.ReadOnlyResourceRepository) resource_lexlex = resource_repo.get_by_resource_id("lexlex") - assert resource_lexlex is not None + assert resource_lexlex.entry_repository_id == entry_repo.entity_id diff --git a/karp/tests/unit/lex/adapters.py b/karp/tests/unit/lex/adapters.py index fc6307ae..1b9aed6b 100644 --- a/karp/tests/unit/lex/adapters.py +++ b/karp/tests/unit/lex/adapters.py @@ -33,6 +33,7 @@ def check_status(self): def _save(self, resource): self.resources[resource.id] = resource + # def _update(self, resource): # r = self._by_id(resource.id) # self.resources.discard(r) @@ -42,7 +43,10 @@ def _by_id(self, id_, *, version=None): return self.resources.get(id_) def _by_resource_id(self, resource_id): - return next((res for res in self.resources.values() if res.resource_id == resource_id), None) + return next( + (res for res in self.resources.values() if res.resource_id == resource_id), + None, + ) def __len__(self): return len(self.resources) @@ -57,21 +61,24 @@ def resource_ids(self) -> typing.Iterable[str]: return (res.resource_id for res in self.resources) def num_entities(self) -> int: - return sum( not res.discarded for res in self.resources.values() ) + return sum(not res.discarded for res in self.resources.values()) class InMemoryReadResourceRepository(ReadOnlyResourceRepository): def __init__(self, resources: Dict): self.resources = resources - def get_by_resource_id(self, resource_id: str, version=None) -> Optional[ResourceDto]: + def get_by_resource_id( + self, resource_id: str, version=None + ) -> Optional[ResourceDto]: return next( ( self._row_to_dto(res) for res in self.resources.values() if res.resource_id == resource_id ), - None) + None, + ) def _row_to_dto(self, res) -> ResourceDto: return ResourceDto( @@ -90,9 +97,7 @@ def _row_to_dto(self, res) -> ResourceDto: def get_published_resources(self) -> Iterable[ResourceDto]: return ( - self._row_to_dto(res) - for res in self.resources.values() - if res.is_published + self._row_to_dto(res) for res in self.resources.values() if res.is_published ) @@ -147,17 +152,35 @@ def all_entries(self) -> typing.Iterable[lex_entities.Entry]: yield from self.entries def num_entities(self) -> int: - return sum( not e.discarded for e in self.entries.values() ) + return sum(not e.discarded for e in self.entries) + def by_referenceable( + self, filters: Optional[Dict] = None, **kwargs + ) -> list[lex_entities.Entry]: + return [] -class InMemoryEntryUnitOfWork( - InMemoryUnitOfWork, lex_repositories.EntryUnitOfWork -): - def __init__(self, entity_id, name: str, config: typing.Dict, connection_str: typing.Optional[str], message: str, user: str, event_bus: EventBus): + +class InMemoryEntryUnitOfWork(InMemoryUnitOfWork, lex_repositories.EntryUnitOfWork): + def __init__( + self, + entity_id, + name: str, + config: typing.Dict, + connection_str: typing.Optional[str], + message: str, + user: str, + event_bus: EventBus, + ): InMemoryUnitOfWork.__init__(self) lex_repositories.EntryUnitOfWork.__init__( self, - entity_id=entity_id, name=name, config=config, connection_str=connection_str, message=message, event_bus=event_bus) + entity_id=entity_id, + name=name, + config=config, + connection_str=connection_str, + message=message, + event_bus=event_bus, + ) self._entries = InMemoryEntryRepository() # self.id = entity_id # self.name = name @@ -168,13 +191,12 @@ def repo(self) -> lex_repositories.EntryRepository: return self._entries -class InMemoryEntryUnitOfWork2( - InMemoryUnitOfWork, lex_repositories.EntryUnitOfWork -): +class InMemoryEntryUnitOfWork2(InMemoryUnitOfWork, lex_repositories.EntryUnitOfWork): def __init__(self, entity_id, name: str, config: typing.Dict): InMemoryUnitOfWork.__init__(self) lex_repositories.EntryUnitOfWork.__init__( - self, entity_id=entity_id, name=name, config=config) + self, entity_id=entity_id, name=name, config=config + ) self._entries = InMemoryEntryRepository() # self.id = entity_id # self.name = name @@ -185,7 +207,9 @@ def repo(self) -> lex_repositories.EntryRepository: return self._entries -class InMemoryResourceUnitOfWork(InMemoryUnitOfWork, lex_repositories.ResourceUnitOfWork): +class InMemoryResourceUnitOfWork( + InMemoryUnitOfWork, lex_repositories.ResourceUnitOfWork +): def __init__(self, event_bus: EventBus): InMemoryUnitOfWork.__init__(self) lex_repositories.ResourceUnitOfWork.__init__(self, event_bus=event_bus) @@ -267,7 +291,9 @@ def num_entities(self) -> int: return sum(not er.discarded for er in self._storage.values()) -class InMemoryEntryUowRepositoryUnitOfWork(InMemoryUnitOfWork, lex_repositories.EntryUowRepositoryUnitOfWork): +class InMemoryEntryUowRepositoryUnitOfWork( + InMemoryUnitOfWork, lex_repositories.EntryUowRepositoryUnitOfWork +): def __init__( self, event_bus: EventBus, @@ -304,7 +330,7 @@ def entry_uow_repo_uow( # return InMemoryEntryRepositoryUnitOfWorkFactory() @injector.multiprovider def entry_uow_creator_map( - self + self, ) -> Dict[str, lex_repositories.EntryUnitOfWorkCreator]: return {"fake": InMemoryEntryUnitOfWorkCreator} @@ -323,6 +349,7 @@ def resource_repo( resources=resource_uow.repo.resources, ) + # def bootstrap_test_app( # resource_uow: lex_repositories.ResourceUnitOfWork = None, # entry_uows: lex_repositories.EntriesUnitOfWork = None, diff --git a/karp/tests/unit/test_json_schema.py b/karp/tests/unit/test_json_schema.py index 36569a8a..c86db027 100644 --- a/karp/tests/unit/test_json_schema.py +++ b/karp/tests/unit/test_json_schema.py @@ -35,16 +35,75 @@ }""" +@pytest.fixture() +def problem_config() -> dict: + return { + "resource_id": "konstruktikon", + "resource_name": "Konstruktikon", + "fields": { + "cat": {"type": "string"}, + "illustration": {"type": "string"}, + "cee": {"type": "string", "collection": True}, + "coll": {"type": "string", "collection": True}, + "createdBy": {"type": "string"}, + "definition": {"type": "string"}, + "examples": {"type": "string", "collection": True}, + "entryStatus": {"type": "string"}, + "intConstElem": { + "type": "object", + "fields": { + "role": {"type": "string"}, + "name": {"type": "string"}, + "cat": {"type": "string"}, + "lu": {"type": "string"}, + "gfunc": {"type": "string"}, + "msd": {"type": "string"}, + "aux": {"type": "string"}, + }, + "collection": True, + }, + "extConstElem": { + "type": "object", + "fields": { + "role": {"type": "string"}, + "name": {"type": "string"}, + "cat": {"type": "string"}, + "lu": {"type": "string"}, + "gfunc": {"type": "string"}, + "msd": {"type": "string"}, + "aux": {"type": "string"}, + }, + "collection": True, + }, + "inheritance": {"type": "string", "collection": True}, + "type": {"type": "string", "collection": True}, + "structure": {"type": "string", "collection": True}, + "constructionID": {"type": "string", "required": True}, + "BCxnID": {"type": "string"}, + "evokes": {"type": "string", "collection": True}, + "comment": {"type": "string"}, + "reference": {"type": "string"}, + "internal_comment": {"type": "string"}, + }, + "sort": "constructionID", + "protected": {"read": False}, + "id": "constructionID", + } + + +def test_error(problem_config: dict): + json_schema = create_entry_json_schema(problem_config["fields"]) + _entry_schema = EntrySchema(json_schema) + + def test_create_json_schema(json_schema_config): json_schema = create_entry_json_schema(json_schema_config["fields"]) assert json_schema["type"] == "object" class TestCreateJsonSchema: - @pytest.mark.parametrize( - 'field_type', ['long_string'] - ) + @pytest.mark.parametrize("field_type", ["long_string"]) def test_create_with_type(self, field_type: str): - resource_config = {'field_name': {'type': field_type}} + resource_config = {"field_name": {"type": field_type}} json_schema = create_entry_json_schema(resource_config) _entry_schema = EntrySchema(json_schema) From 8b930b3f8ffefd4dedeb3cb4502008ddd72a38c6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 1 Apr 2022 07:46:08 +0000 Subject: [PATCH 24/43] =?UTF-8?q?=F0=9F=93=9D=20Update=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b75a96..7a0dd6d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Latest Changes +* Return result from commandbus. PR [#196](https://github.com/spraakbanken/karp-backend/pull/196) by [@kod-kristoff](https://github.com/kod-kristoff). * Add support for discarding entry-repositories. PR [#195](https://github.com/spraakbanken/karp-backend/pull/195) by [@kod-kristoff](https://github.com/kod-kristoff). * Avoid name clashes in SqlEntryRepository. PR [#194](https://github.com/spraakbanken/karp-backend/pull/194) by [@kod-kristoff](https://github.com/kod-kristoff). * Fix correct fetching from repos. PR [#193](https://github.com/spraakbanken/karp-backend/pull/193) by [@kod-kristoff](https://github.com/kod-kristoff). From 5512db6283bced0d638898d7bd9a8fef7e9394bc Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 09:47:23 +0200 Subject: [PATCH 25/43] build: prepare release --- CHANGELOG.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a0dd6d7..5a35f588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Latest Changes -* Return result from commandbus. PR [#196](https://github.com/spraakbanken/karp-backend/pull/196) by [@kod-kristoff](https://github.com/kod-kristoff). -* Add support for discarding entry-repositories. PR [#195](https://github.com/spraakbanken/karp-backend/pull/195) by [@kod-kristoff](https://github.com/kod-kristoff). -* Avoid name clashes in SqlEntryRepository. PR [#194](https://github.com/spraakbanken/karp-backend/pull/194) by [@kod-kristoff](https://github.com/kod-kristoff). -* Fix correct fetching from repos. PR [#193](https://github.com/spraakbanken/karp-backend/pull/193) by [@kod-kristoff](https://github.com/kod-kristoff). +## 6.0.20 + +### Changed + +- Return result from commandbus. PR [#196](https://github.com/spraakbanken/karp-backend/pull/196) by [@kod-kristoff](https://github.com/kod-kristoff). +- Add support for discarding entry-repositories. PR [#195](https://github.com/spraakbanken/karp-backend/pull/195) by [@kod-kristoff](https://github.com/kod-kristoff). +- Avoid name clashes in SqlEntryRepository. PR [#194](https://github.com/spraakbanken/karp-backend/pull/194) by [@kod-kristoff](https://github.com/kod-kristoff). +- Fix correct fetching from repos. PR [#193](https://github.com/spraakbanken/karp-backend/pull/193) by [@kod-kristoff](https://github.com/kod-kristoff). + ## 6.0.17 ### Changed From ac3cf603536215430cdd44c7ab8e38f7530b5199 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 09:47:34 +0200 Subject: [PATCH 26/43] =?UTF-8?q?Bump=20version:=206.0.19=20=E2=86=92=206.?= =?UTF-8?q?0.20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- karp/__init__.py | 2 +- karp/data_files/karp_api_spec.yaml | 2 +- karp/main/config.py | 2 +- pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 42a9244f..69c5de7b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 6.0.19 +current_version = 6.0.20 commit = True tag = True sign_tag = True diff --git a/karp/__init__.py b/karp/__init__.py index c95449fa..d7ed6e02 100644 --- a/karp/__init__.py +++ b/karp/__init__.py @@ -7,7 +7,7 @@ # import werkzeug.exceptions -__version__ = "6.0.19" +__version__ = "6.0.20" # TODO handle settings correctly diff --git a/karp/data_files/karp_api_spec.yaml b/karp/data_files/karp_api_spec.yaml index c23b9378..6f299ca7 100644 --- a/karp/data_files/karp_api_spec.yaml +++ b/karp/data_files/karp_api_spec.yaml @@ -1,7 +1,7 @@ openapi: 3.0.2 info: title: Karp API - version: 6.0.19 + version: 6.0.20 description: | Karp TNG diff --git a/karp/main/config.py b/karp/main/config.py index 1583b1c4..dd5f89c2 100644 --- a/karp/main/config.py +++ b/karp/main/config.py @@ -6,7 +6,7 @@ from starlette.datastructures import Secret PROJECT_NAME = 'Karp' -VERSION = '6.0.19' +VERSION = '6.0.20' API_PREFIX = '/' # SECRET_KEY = config("SECRET_KEY", cast=Secret, default="CHANGEME") diff --git a/pyproject.toml b/pyproject.toml index 48b6f386..01686d87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karp-backend" -version = "6.0.19" +version = "6.0.20" license = "MIT" description = "Karp backend" readme = "README.md" From a07eb9a3e1a98f1e6f94c6cbbe2880c7539bfbd6 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 10:48:57 +0200 Subject: [PATCH 27/43] refactor: add logging --- .../transformers/generic_entry_transformer.py | 4 ++++ .../transformers/generic_pre_processor.py | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/karp/search_infrastructure/transformers/generic_entry_transformer.py b/karp/search_infrastructure/transformers/generic_entry_transformer.py index 01f1a4cc..321072a8 100644 --- a/karp/search_infrastructure/transformers/generic_entry_transformer.py +++ b/karp/search_infrastructure/transformers/generic_entry_transformer.py @@ -71,6 +71,9 @@ def transform(self, resource_id: str, src_entry: EntryDto) -> IndexEntry: index_entry, resource.config["fields"].items(), ) + logger.debug( + "transformed entry", extra={"entry": src_entry, "index_entry": index_entry} + ) return index_entry def _transform_to_index_entry( @@ -82,6 +85,7 @@ def _transform_to_index_entry( _index_entry: IndexEntry, fields, ): + logger.debug("transforming [part of] entry", extra={"_src_entry": _src_entry}) for field_name, field_conf in fields: field_content = None if field_conf.get("virtual"): diff --git a/karp/search_infrastructure/transformers/generic_pre_processor.py b/karp/search_infrastructure/transformers/generic_pre_processor.py index ca204559..aff178e6 100644 --- a/karp/search_infrastructure/transformers/generic_pre_processor.py +++ b/karp/search_infrastructure/transformers/generic_pre_processor.py @@ -1,3 +1,4 @@ +import logging import typing from karp.lex.application.queries import EntryViews @@ -5,6 +6,9 @@ from karp.search.application.transformers import PreProcessor, EntryTransformer +logger = logging.getLogger(__name__) + + class GenericPreProcessor(PreProcessor): def __init__( self, @@ -19,5 +23,12 @@ def process( self, resource_id: str, ) -> typing.Iterable[IndexEntry]: + logger.debug( + "processing entries for resource", extra={"resource_id": resource_id} + ) + for entry in self.entry_views.all_entries(resource_id): + logger.debug( + "processing entry", extra={"entry": entry, "resource_id": resource_id} + ) yield self.entry_transformer.transform(resource_id, entry) From 51e349643247d948dd78c91afc95c3153511056e Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 11:18:15 +0200 Subject: [PATCH 28/43] fix: handle long_string --- .../transformers/generic_entry_transformer.py | 8 +- .../test_generic_entry_transformer.py | 123 +++++++++--------- 2 files changed, 69 insertions(+), 62 deletions(-) diff --git a/karp/search_infrastructure/transformers/generic_entry_transformer.py b/karp/search_infrastructure/transformers/generic_entry_transformer.py index 321072a8..f92d2a6a 100644 --- a/karp/search_infrastructure/transformers/generic_entry_transformer.py +++ b/karp/search_infrastructure/transformers/generic_entry_transformer.py @@ -125,7 +125,13 @@ def _transform_to_index_entry( field_content, field_conf["fields"].items(), ) - elif field_conf["type"] in ("integer", "string", "number", "boolean"): + elif field_conf["type"] in ( + "integer", + "string", + "number", + "boolean", + "long_string", + ): if field_name in _src_entry: field_content = _src_entry[field_name] diff --git a/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py b/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py index 39d271fd..7217331f 100644 --- a/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py +++ b/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py @@ -11,9 +11,11 @@ class TestGenericEntryTransformer: @pytest.mark.parametrize( - 'field_name, field_config, field_value', [ - ('single', {'type': 'boolean'}, True), - ]) + "field_name, field_config, field_value", + [ + ("single", {"type": "boolean"}, True), + ], + ) def test_transform_to_index_entry( self, field_name: str, @@ -21,56 +23,54 @@ def test_transform_to_index_entry( field_value: Any, search_unit_ctx, ): - resource_id = 'transform_res' + resource_id = "transform_res" create_entry_repo = lex_factories.CreateEntryRepositoryFactory() search_unit_ctx.command_bus.dispatch(create_entry_repo) create_resource = lex_factories.CreateResourceFactory( entry_repo_id=create_entry_repo.entity_id, resource_id=resource_id, config={ - 'fields': { - 'id': {'type': 'string'}, + "fields": { + "id": {"type": "string"}, field_name: field_config, }, - 'id': 'id', - - } + "id": "id", + }, ) search_unit_ctx.command_bus.dispatch(create_resource) transformer = search_unit_ctx.container.get(EntryTransformer) - entry_id = 'entry..1' + entry_id = "entry..1" src_entry = EntryDto( entry_id=entry_id, resource=resource_id, version=1, - entry={ - 'id': entry_id, - field_name: field_value - }, + entry={"id": entry_id, field_name: field_value}, last_modified=1234567, - last_modified_by='alice@example.com', + last_modified_by="alice@example.com", ) index_entry = transformer.transform(resource_id, src_entry) assert index_entry.id == entry_id - assert index_entry.entry['_entry_version'] == 1 - assert index_entry.entry['id'] == entry_id + assert index_entry.entry["_entry_version"] == 1 + assert index_entry.entry["id"] == entry_id assert index_entry.entry[field_name] == field_value @pytest.mark.parametrize( - 'field_name, field_config, field_value', [ - ('single', {'type': 'boolean'}, True), - ( - 'single', - { - 'type': 'object', - 'fields': { - 'sub': {'type': 'string'}, - } - }, - {'sub': 'test'}, - ), - ]) + "field_name, field_config, field_value", + [ + ("single", {"type": "boolean"}, True), + ( + "single", + { + "type": "object", + "fields": { + "sub": {"type": "string"}, + }, + }, + {"sub": "test"}, + ), + ], + ) def test_transform_to_index_entry_collection( self, field_name: str, @@ -78,47 +78,49 @@ def test_transform_to_index_entry_collection( field_value: Any, search_unit_ctx, ): - resource_id = 'transform_res' + resource_id = "transform_res" create_entry_repo = lex_factories.CreateEntryRepositoryFactory() search_unit_ctx.command_bus.dispatch(create_entry_repo) - field_config['collection'] = True + field_config["collection"] = True create_resource = lex_factories.CreateResourceFactory( entry_repo_id=create_entry_repo.entity_id, resource_id=resource_id, config={ - 'fields': { - 'id': {'type': 'string'}, + "fields": { + "id": {"type": "string"}, field_name: field_config, }, - 'id': 'id', - - } + "id": "id", + }, ) search_unit_ctx.command_bus.dispatch(create_resource) transformer = search_unit_ctx.container.get(EntryTransformer) - entry_id = 'entry..1' + entry_id = "entry..1" src_entry = EntryDto( entry_id=entry_id, resource=resource_id, version=1, entry={ - 'id': entry_id, + "id": entry_id, field_name: [field_value], }, last_modified=1234567, - last_modified_by='alice@example.com', + last_modified_by="alice@example.com", ) index_entry = transformer.transform(resource_id, src_entry) assert index_entry.id == entry_id - assert index_entry.entry['_entry_version'] == 1 - assert index_entry.entry['id'] == entry_id + assert index_entry.entry["_entry_version"] == 1 + assert index_entry.entry["id"] == entry_id assert index_entry.entry[field_name][0] == field_value @pytest.mark.parametrize( - 'field_name, field_config, field_value', [ - ('single', {'type': 'boolean'}, True), - ]) + "field_name, field_config, field_value", + [ + ("single", {"type": "boolean"}, True), + ("single", {"type": "long_string"}, "very long string"), + ], + ) def test_transform_to_index_entry_object( self, field_name: str, @@ -126,45 +128,44 @@ def test_transform_to_index_entry_object( field_value: Any, search_unit_ctx, ): - resource_id = 'transform_res' + resource_id = "transform_res" create_entry_repo = lex_factories.CreateEntryRepositoryFactory() search_unit_ctx.command_bus.dispatch(create_entry_repo) create_resource = lex_factories.CreateResourceFactory( entry_repo_id=create_entry_repo.entity_id, resource_id=resource_id, config={ - 'fields': { - 'id': {'type': 'string'}, - 'obj': { - 'type': 'object', - 'fields': { + "fields": { + "id": {"type": "string"}, + "obj": { + "type": "object", + "fields": { field_name: field_config, }, }, }, - 'id': 'id', - - } + "id": "id", + }, ) search_unit_ctx.command_bus.dispatch(create_resource) transformer = search_unit_ctx.container.get(EntryTransformer) - entry_id = 'entry..1' + entry_id = "entry..1" src_entry = EntryDto( entry_id=entry_id, resource=resource_id, version=1, entry={ - 'id': entry_id, - 'obj': { + "id": entry_id, + "obj": { field_name: field_value, }, }, last_modified=1234567, - last_modified_by='alice@example.com', + last_modified_by="alice@example.com", ) index_entry = transformer.transform(resource_id, src_entry) assert index_entry.id == entry_id - assert index_entry.entry['_entry_version'] == 1 - assert index_entry.entry['id'] == entry_id - assert index_entry.entry['obj'][field_name] == field_value + assert index_entry.entry["_entry_version"] == 1 + assert index_entry.entry["id"] == entry_id + assert index_entry.entry["obj"][field_name] == field_value From 8ffdc980194c88fa5761c44a5bdf4e9084104365 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 1 Apr 2022 09:27:18 +0000 Subject: [PATCH 29/43] =?UTF-8?q?=F0=9F=93=9D=20Update=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a35f588..d6b1c89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Latest Changes +* fix: handle long_string in transforming. PR [#197](https://github.com/spraakbanken/karp-backend/pull/197) by [@kod-kristoff](https://github.com/kod-kristoff). ## 6.0.20 ### Changed From 88236ca0f823c5240fff79daa2173727c17a5fad Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 12:09:58 +0200 Subject: [PATCH 30/43] test: expand tests --- .../transformers/generic_entry_transformer.py | 2 +- .../test_generic_entry_transformer.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/karp/search_infrastructure/transformers/generic_entry_transformer.py b/karp/search_infrastructure/transformers/generic_entry_transformer.py index f92d2a6a..ea8a3d6d 100644 --- a/karp/search_infrastructure/transformers/generic_entry_transformer.py +++ b/karp/search_infrastructure/transformers/generic_entry_transformer.py @@ -85,7 +85,7 @@ def _transform_to_index_entry( _index_entry: IndexEntry, fields, ): - logger.debug("transforming [part of] entry", extra={"_src_entry": _src_entry}) + logger.debug("transforming [part of] entry", extra={"src_entry": _src_entry}) for field_name, field_conf in fields: field_content = None if field_conf.get("virtual"): diff --git a/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py b/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py index 7217331f..5942a48b 100644 --- a/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py +++ b/karp/tests/unit/search_infrastructure/test_generic_entry_transformer.py @@ -59,6 +59,10 @@ def test_transform_to_index_entry( "field_name, field_config, field_value", [ ("single", {"type": "boolean"}, True), + ("single", {"type": "string"}, "plain string"), + ("single", {"type": "number"}, 3.14), + ("single", {"type": "integer"}, 3), + ("single", {"type": "long_string"}, "very long string"), ( "single", { @@ -118,7 +122,15 @@ def test_transform_to_index_entry_collection( "field_name, field_config, field_value", [ ("single", {"type": "boolean"}, True), + ("single", {"type": "string"}, "plain string"), + ("single", {"type": "number"}, 3.14), + ("single", {"type": "integer"}, 3), ("single", {"type": "long_string"}, "very long string"), + ( + "single", + {"type": "object", "fields": {"sub": {"type": "string"}}}, + {"sub": "test"}, + ), ], ) def test_transform_to_index_entry_object( From 05bd17c9d75fb266d752ad94fe8d436b5beb3524 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Fri, 1 Apr 2022 12:17:46 +0200 Subject: [PATCH 31/43] fix: make karp-cli query output json --- karp/cliapp/subapps/query_subapp.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/karp/cliapp/subapps/query_subapp.py b/karp/cliapp/subapps/query_subapp.py index bdb95659..b034c1de 100644 --- a/karp/cliapp/subapps/query_subapp.py +++ b/karp/cliapp/subapps/query_subapp.py @@ -1,3 +1,7 @@ +from pathlib import Path +from typing import Optional + +import json_streams import typer from karp import search @@ -9,14 +13,20 @@ @subapp.command() def resource( - resource_id: str, ctx: typer.Context, + resource_id: str, + output: Optional[Path] = typer.Option( + None, help="Path to write to. Defaults to stdout." + ), ): - typer.echo('query') search_service = inject_from_ctx(search.SearchService, ctx) query_request = search.QueryRequest(resource_ids=[resource_id]) - typer.echo(search_service.query(query_request)) + json_streams.dump_to_file( + search_service.query(query_request), + output, + use_stdout_as_default=True, + ) def init_app(app: typer.Typer) -> None: - app.add_typer(subapp, name='query') + app.add_typer(subapp, name="query") From c0dc725ddbdda753d58a68ec40034b47e29f77b8 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 4 Apr 2022 12:05:28 +0200 Subject: [PATCH 32/43] fix: add export stub --- karp/cliapp/subapps/entries_subapp.py | 19 +++++++++++++++++++ karp/lex/__init__.py | 1 + 2 files changed, 20 insertions(+) diff --git a/karp/cliapp/subapps/entries_subapp.py b/karp/cliapp/subapps/entries_subapp.py index 8caf91d3..3a11a047 100644 --- a/karp/cliapp/subapps/entries_subapp.py +++ b/karp/cliapp/subapps/entries_subapp.py @@ -8,6 +8,7 @@ from tqdm import tqdm from karp.foundation.commands import CommandBus +from karp import lex from karp.lex.domain import commands # from karp.lex.domain.errors import ResourceAlreadyPublished @@ -53,6 +54,24 @@ def update_entries(resource_id: str, data: Path): resource_id, json_streams.load_from_file(data) ) +@subapp.command("export") +@cli_error_handler +@cli_timer +def export_entries( + ctx: typer.Context, + resource_id: str, + output: Optional[Path] = typer.Option(None, "--output", "-o"), +): + entry_views = inject_from_ctx(lex.EntryViews, ctx=ctx) + json_streams.dump_to_file( + tqdm( + entry_views.all_entries(resource_id), + desc="Exporting", + unit=" entries", + ), + output, + use_stdout_as_default=None, + ) def init_app(app): app.add_typer(subapp, name="entries") diff --git a/karp/lex/__init__.py b/karp/lex/__init__.py index 3c3068c7..e6dda496 100644 --- a/karp/lex/__init__.py +++ b/karp/lex/__init__.py @@ -31,6 +31,7 @@ ) from karp.lex.application.queries import ( EntryDto, + EntryViews, GetEntryDiff, GetEntryHistory, GetHistory, From e8b34e28de0054a33a1ac67946404c7d59f244d8 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Wed, 6 Apr 2022 12:59:41 +0200 Subject: [PATCH 33/43] feat: add chunked import --- karp/cliapp/subapps/entries_subapp.py | 38 ++-- karp/lex/__init__.py | 22 +++ karp/lex/application/use_cases/__init__.py | 6 + .../application/use_cases/entry_handlers.py | 173 ++++++++++++------ karp/lex/domain/commands/__init__.py | 5 + karp/lex/domain/commands/entry_commands.py | 7 +- 6 files changed, 177 insertions(+), 74 deletions(-) diff --git a/karp/cliapp/subapps/entries_subapp.py b/karp/cliapp/subapps/entries_subapp.py index 3a11a047..626b7aa2 100644 --- a/karp/cliapp/subapps/entries_subapp.py +++ b/karp/cliapp/subapps/entries_subapp.py @@ -10,6 +10,7 @@ from karp.foundation.commands import CommandBus from karp import lex from karp.lex.domain import commands + # from karp.lex.domain.errors import ResourceAlreadyPublished from karp.cliapp.utility import cli_error_handler, cli_timer @@ -25,23 +26,34 @@ @cli_error_handler @cli_timer def import_resource( + ctx: typer.Context, resource_id: str, # version: Optional[int], data: Path, - ctx: typer.Context, + chunked: bool = False, + chunk_size: int = 1000, + user: Optional[str] = typer.Option(None), + message: Optional[str] = typer.Option(None), ): bus = inject_from_ctx(CommandBus, ctx) - cmd = commands.AddEntries( - resource_id=resource_id, - # entries=json_streams.load_from_file(data), - entries=tqdm( - json_streams.load_from_file(data), - desc="Importing", - unit=" entries" - ), - user="local admin", - message="imported through cli", - ) + user = user or "local admin" + message = message or "imported through cli" + entries = tqdm(json_streams.load_from_file(data), desc="Importing", unit=" entries") + if chunked: + cmd = lex.AddEntriesInChunks( + resource_id=resource_id, + chunk_size=chunk_size, + entries=entries, + user=user, + message=message, + ) + else: + cmd = lex.AddEntries( + resource_id=resource_id, + entries=entries, + user=user, + message=message, + ) bus.dispatch(cmd) typer.echo(f"Successfully imported entries to {resource_id}") @@ -54,6 +66,7 @@ def update_entries(resource_id: str, data: Path): resource_id, json_streams.load_from_file(data) ) + @subapp.command("export") @cli_error_handler @cli_timer @@ -73,5 +86,6 @@ def export_entries( use_stdout_as_default=None, ) + def init_app(app): app.add_typer(subapp, name="entries") diff --git a/karp/lex/__init__.py b/karp/lex/__init__.py index e6dda496..7f846e0c 100644 --- a/karp/lex/__init__.py +++ b/karp/lex/__init__.py @@ -8,6 +8,9 @@ ResourceUnitOfWork, ) from karp.lex.domain.commands import ( + AddEntries, + AddEntriesInChunks, + AddEntry, CreateEntryRepository, CreateResource, DeleteEntryRepository, @@ -18,6 +21,7 @@ from karp.lex.domain.value_objects import EntrySchema from karp.lex.application.use_cases import ( AddingEntries, + AddingEntriesInChunks, AddingEntry, CreatingEntryRepo, CreatingResource, @@ -47,6 +51,14 @@ ) +__all__ = [ + # commands + "AddEntriesInChunks", + # use cases + "AddingEntriesInChunks", +] + + class Lex(injector.Module): @injector.provider def entry_uow_factory( @@ -131,6 +143,16 @@ def add_entries( ) -> CommandHandler[commands.AddEntries]: return AddingEntries(resource_uow=resource_uow, entry_repo_uow=entry_repo_uow) + @injector.provider + def adding_entries_in_chunks( + self, + resource_uow: ResourceUnitOfWork, + entry_repo_uow: EntryUowRepositoryUnitOfWork, + ) -> CommandHandler[AddEntriesInChunks]: + return AddingEntriesInChunks( + resource_uow=resource_uow, entry_repo_uow=entry_repo_uow + ) + @injector.provider def update_entry( self, diff --git a/karp/lex/application/use_cases/__init__.py b/karp/lex/application/use_cases/__init__.py index b5f68928..04917f87 100644 --- a/karp/lex/application/use_cases/__init__.py +++ b/karp/lex/application/use_cases/__init__.py @@ -1,6 +1,7 @@ from .entry_handlers import ( AddingEntry, AddingEntries, + AddingEntriesInChunks, DeletingEntry, UpdatingEntry, ) @@ -12,3 +13,8 @@ SettingEntryRepoId, UpdatingResource, ) + + +__all__ = [ + "AddingEntriesInChunks", +] diff --git a/karp/lex/application/use_cases/entry_handlers.py b/karp/lex/application/use_cases/entry_handlers.py index 71d58882..b21cd47c 100644 --- a/karp/lex/application/use_cases/entry_handlers.py +++ b/karp/lex/application/use_cases/entry_handlers.py @@ -1,12 +1,8 @@ -import collections -import json import logging import typing -from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Generic, List, Optional, Tuple -import fastjsonschema # pyre-ignore import json_streams from sb_json_tools import jsondiff import logging @@ -17,8 +13,13 @@ from karp.lex.domain.entities.entry import Entry from karp.lex.domain.entities.resource import Resource from karp.foundation.value_objects import unique_id -from karp.errors import (ClientErrorCodes, EntryIdMismatch, EntryNotFoundError, - KarpError, UpdateConflict) +from karp.errors import ( + ClientErrorCodes, + EntryIdMismatch, + EntryNotFoundError, + KarpError, + UpdateConflict, +) from karp.foundation import events as foundation_events from karp.foundation import messagebus from karp.lex.domain import commands @@ -78,7 +79,7 @@ class BasingEntry: def __init__( self, entry_repo_uow: repositories.EntryUowRepositoryUnitOfWork, - resource_uow: repositories.ResourceUnitOfWork + resource_uow: repositories.ResourceUnitOfWork, ) -> None: super().__init__() self.entry_repo_uow = entry_repo_uow @@ -94,18 +95,15 @@ def get_entry_uow(self, entry_repo_id: unique_id.UniqueId) -> EntryUnitOfWork: class AddingEntry(BasingEntry, CommandHandler[commands.AddEntry]): - - def execute(self, cmd: commands.AddEntry): + def execute(self, command: commands.AddEntry): with self.resource_uow: - resource = self.resource_uow.repo.by_resource_id( - cmd.resource_id) + resource = self.resource_uow.repo.by_resource_id(command.resource_id) try: - entry_id = resource.id_getter()(cmd.entry) + entry_id = resource.id_getter()(command.entry) except KeyError as err: raise errors.MissingIdField( - resource_id=cmd.resource_id, - entry=cmd.entry + resource_id=command.resource_id, entry=command.entry ) from err entry_schema = EntrySchema(resource.entry_json_schema) @@ -114,19 +112,19 @@ def execute(self, cmd: commands.AddEntry): if ( existing_entry and not existing_entry.discarded - and existing_entry.entity_id != cmd.entity_id + and existing_entry.entity_id != command.entity_id ): raise errors.IntegrityError( f"An entry with entry_id '{entry_id}' already exists." ) - entry_schema.validate_entry(cmd.entry) + entry_schema.validate_entry(command.entry) entry = resource.create_entry_from_dict( - cmd.entry, - user=cmd.user, - message=cmd.message, - entity_id=cmd.entity_id + command.entry, + user=command.user, + message=command.message, + entity_id=command.entity_id, ) uw.entries.save(entry) uw.commit() @@ -141,28 +139,24 @@ def execute(self, cmd: commands.AddEntry): class UpdatingEntry(BasingEntry, CommandHandler[commands.UpdateEntry]): - def execute(self, cmd: commands.UpdateEntry): + def execute(self, command: commands.UpdateEntry): with self.resource_uow: - resource = self.resource_uow.repo.by_resource_id( - cmd.resource_id - ) + resource = self.resource_uow.repo.by_resource_id(command.resource_id) entry_schema = EntrySchema(resource.entry_json_schema) - entry_schema.validate_entry(cmd.entry) + entry_schema.validate_entry(command.entry) with self.get_entry_uow(resource.entry_repository_id) as uw: try: - current_db_entry = uw.repo.by_entry_id( - cmd.entry_id - ) + current_db_entry = uw.repo.by_entry_id(command.entry_id) except errors.EntryNotFound as err: raise errors.EntryNotFound( - cmd.resource_id, - cmd.entry_id, + command.resource_id, + command.entry_id, entity_id=None, ) from err - diff = jsondiff.compare(current_db_entry.body, cmd.entry) + diff = jsondiff.compare(current_db_entry.body, command.entry) if not diff: return current_db_entry @@ -173,22 +167,29 @@ def execute(self, cmd: commands.UpdateEntry): # .order_by(resource.history_model.version.desc()) # .first() # ) - if not cmd.force and current_db_entry.version != cmd.version: + if not command.force and current_db_entry.version != command.version: logger.info( - 'version conflict', current_version=current_db_entry.version, version=cmd.version) + "version conflict", + current_version=current_db_entry.version, + version=command.version, + ) raise errors.UpdateConflict(diff) id_getter = resource.id_getter() - new_entry_id = id_getter(cmd.entry) + new_entry_id = id_getter(command.entry) - current_db_entry.body = cmd.entry + current_db_entry.body = command.entry current_db_entry.stamp( - cmd.user, message=cmd.message, timestamp=cmd.timestamp) - if new_entry_id != cmd.entry_id: - logger.info('updating entry_id', - entry_id=cmd.entry_id, new_entry_id=new_entry_id) + command.user, message=command.message, timestamp=command.timestamp + ) + if new_entry_id != command.entry_id: + logger.info( + "updating entry_id", + entry_id=command.entry_id, + new_entry_id=new_entry_id, + ) current_db_entry.entry_id = new_entry_id - # uw.repo.move(current_db_entry, old_entry_id=cmd.entry_id) + # uw.repo.move(current_db_entry, old_entry_id=command.entry_id) uw.repo.save(current_db_entry) else: uw.repo.save(current_db_entry) @@ -212,11 +213,8 @@ def add_entries_from_file( ) -class AddingEntries( - BasingEntry, - CommandHandler[commands.AddEntries] -): - def execute(self, cmd: commands.AddEntries): +class AddingEntries(BasingEntry, CommandHandler[commands.AddEntries]): + def execute(self, command: commands.AddEntries): """ Add entries to DB and INDEX (if present and resource is active). @@ -234,29 +232,29 @@ def execute(self, cmd: commands.AddEntries): List of the id's of the created entries. """ - if not isinstance(cmd.resource_id, str): + if not isinstance(command.resource_id, str): raise ValueError( - f"'resource_id' must be of type 'str', were '{type(cmd.resource_id)}'" + f"'resource_id' must be of type 'str', were '{type(command.resource_id)}'" ) with self.resource_uow: - resource = self.resource_uow.resources.by_resource_id( - cmd.resource_id) + resource = self.resource_uow.resources.by_resource_id(command.resource_id) if not resource: - raise errors.ResourceNotFound(cmd.resource_id) + raise errors.ResourceNotFound(command.resource_id) entry_schema = EntrySchema(resource.entry_json_schema) created_db_entries = [] with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id( - resource.entry_repository_id) as uw: - for entry_raw in cmd.entries: + resource.entry_repository_id + ) as uw: + for entry_raw in command.entries: entry_schema.validate_entry(entry_raw) entry = resource.create_entry_from_dict( entry_raw, - user=cmd.user, - message=cmd.message, + user=command.user, + message=command.message, entity_id=unique_id.make_unique_id(), ) uw.entries.save(entry) @@ -266,6 +264,60 @@ def execute(self, cmd: commands.AddEntries): return created_db_entries +class AddingEntriesInChunks(BasingEntry, CommandHandler[commands.AddEntriesInChunks]): + def execute(self, command: commands.AddEntriesInChunks): + """ + Add entries to DB and INDEX (if present and resource is active). + + Raises + ------ + RuntimeError + If the resource.entry_json_schema fails to compile. + KarpError + - If an entry fails to be validated against the json schema. + - If the DB interaction fails. + + Returns + ------- + List + List of the id's of the created entries. + """ + + if not isinstance(command.resource_id, str): + raise ValueError( + f"'resource_id' must be of type 'str', were '{type(command.resource_id)}'" + ) + with self.resource_uow: + resource = self.resource_uow.resources.by_resource_id(command.resource_id) + + if not resource: + raise errors.ResourceNotFound(command.resource_id) + + entry_schema = EntrySchema(resource.entry_json_schema) + + created_db_entries = [] + with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id( + resource.entry_repository_id + ) as uw: + for i, entry_raw in enumerate(command.entries): + entry_schema.validate_entry(entry_raw) + + entry = resource.create_entry_from_dict( + entry_raw, + user=command.user, + message=command.message, + entity_id=unique_id.make_unique_id(), + ) + uw.entries.save(entry) + created_db_entries.append(entry) + + if i % command.chunk_size == 0: + uw.commit() + uw.commit() + + return created_db_entries + + # def add_entries( # resource_id: str, # entries: List[Dict], @@ -382,20 +434,19 @@ def execute(self, cmd: commands.AddEntries): class DeletingEntry(BasingEntry, CommandHandler[commands.DeleteEntry]): - - def execute(self, cmd: commands.DeleteEntry): + def execute(self, command: commands.DeleteEntry): with self.resource_uow: - resource = self.resource_uow.repo.by_resource_id(cmd.resource_id) + resource = self.resource_uow.repo.by_resource_id(command.resource_id) with self.entry_repo_uow, self.entry_repo_uow.repo.get_by_id( resource.entry_repository_id ) as uw: - entry = uw.repo.by_entry_id(cmd.entry_id) + entry = uw.repo.by_entry_id(command.entry_id) entry.discard( - user=cmd.user, - message=cmd.message, - timestamp=cmd.timestamp, + user=command.user, + message=command.message, + timestamp=command.timestamp, ) uw.repo.save(entry) uw.commit() diff --git a/karp/lex/domain/commands/__init__.py b/karp/lex/domain/commands/__init__.py index 12d6231c..719047b9 100644 --- a/karp/lex/domain/commands/__init__.py +++ b/karp/lex/domain/commands/__init__.py @@ -2,6 +2,7 @@ from .entry_commands import ( AddEntries, + AddEntriesInChunks, AddEntry, DeleteEntry, UpdateEntry, @@ -14,3 +15,7 @@ SetEntryRepoId, UpdateResource, ) + +__all__ = [ + "AddEntriesInChunks", +] diff --git a/karp/lex/domain/commands/entry_commands.py b/karp/lex/domain/commands/entry_commands.py index 780ed92a..05e0038d 100644 --- a/karp/lex/domain/commands/entry_commands.py +++ b/karp/lex/domain/commands/entry_commands.py @@ -11,7 +11,8 @@ class AddEntry(Command): entity_id: unique_id.UniqueId = pydantic.Field( - default_factory=unique_id.make_unique_id) + default_factory=unique_id.make_unique_id + ) resource_id: str entry: typing.Dict user: str @@ -25,6 +26,10 @@ class AddEntries(Command): message: str +class AddEntriesInChunks(AddEntries): + chunk_size: int + + class DeleteEntry(Command): resource_id: str entry_id: str From 077237ded859db0ec2cd063d17294455c874e764 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Wed, 6 Apr 2022 13:11:55 +0200 Subject: [PATCH 34/43] ci: remove caching of poetry --- .github/workflows/build-publish.yml | 48 ++++++++++++++--------------- .github/workflows/build.yml | 30 +++++++++--------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-publish.yml b/.github/workflows/build-publish.yml index 620af06d..1a7acc46 100644 --- a/.github/workflows/build-publish.yml +++ b/.github/workflows/build-publish.yml @@ -19,11 +19,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Load cached $HOME/.local - uses: actions/cache@v2.1.6 - with: - path: ~/.local - key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} + # - name: Load cached $HOME/.local + # uses: actions/cache@v2.1.6 + # with: + # path: ~/.local + # key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} - name: Install Poetry uses: snok/install-poetry@v1.3 @@ -32,15 +32,15 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2 - with: - path: .venv - key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/build-publish.yml') }} + # - name: Load cached venv + # id: cached-poetry-dependencies + # uses: actions/cache@v2 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/build-publish.yml') }} - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: make install-dev - name: Setup up environment @@ -86,11 +86,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Load cached $HOME/.local - uses: actions/cache@v2.1.6 - with: - path: ~/.local - key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} + # - name: Load cached $HOME/.local + # uses: actions/cache@v2.1.6 + # with: + # path: ~/.local + # key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} - name: Install Poetry uses: snok/install-poetry@v1.3 @@ -99,15 +99,15 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2 - with: - path: .venv - key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/build-publish.yml') }} + # - name: Load cached venv + # id: cached-poetry-dependencies + # uses: actions/cache@v2 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}-${{ hashFiles('.github/workflows/build-publish.yml') }} - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: make install-dev - name: Setup up environment diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 187a7cb8..a46bd6ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,11 +43,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Load cached $HOME/.local - uses: actions/cache@v2.1.6 - with: - path: ~/.local - key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} + # - name: Load cached $HOME/.local + # uses: actions/cache@v2.1.6 + # with: + # path: ~/.local + # key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} - name: Install Poetry uses: snok/install-poetry@v1.3 @@ -99,11 +99,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Load cached $HOME/.local - uses: actions/cache@v2.1.6 - with: - path: ~/.local - key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} + # - name: Load cached $HOME/.local + # uses: actions/cache@v2.1.6 + # with: + # path: ~/.local + # key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} - name: Install Poetry uses: snok/install-poetry@v1.3 @@ -186,11 +186,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Load cached $HOME/.local - uses: actions/cache@v2.1.6 - with: - path: ~/.local - key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} + # - name: Load cached $HOME/.local + # uses: actions/cache@v2.1.6 + # with: + # path: ~/.local + # key: dotlocal-${{ runner.os }}-${{ matrix.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('.github/workflows/build.yml') }} - name: Install Poetry uses: snok/install-poetry@v1.3 From f24d6a1af13c002e2f4485b5ca3897fc2ce52a24 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 6 Apr 2022 11:26:41 +0000 Subject: [PATCH 35/43] =?UTF-8?q?=F0=9F=93=9D=20Update=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b1c89f..781fd336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Latest Changes +* Fix cli export and add chunked cli import. PR [#198](https://github.com/spraakbanken/karp-backend/pull/198) by [@kod-kristoff](https://github.com/kod-kristoff). * fix: handle long_string in transforming. PR [#197](https://github.com/spraakbanken/karp-backend/pull/197) by [@kod-kristoff](https://github.com/kod-kristoff). ## 6.0.20 From 1b3156efb27ee12680d5037b195407e614162246 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Wed, 6 Apr 2022 13:48:41 +0200 Subject: [PATCH 36/43] refactor: remove unused import --- karp/cliapp/subapps/entries_subapp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/karp/cliapp/subapps/entries_subapp.py b/karp/cliapp/subapps/entries_subapp.py index 626b7aa2..8dcd375d 100644 --- a/karp/cliapp/subapps/entries_subapp.py +++ b/karp/cliapp/subapps/entries_subapp.py @@ -9,7 +9,6 @@ from karp.foundation.commands import CommandBus from karp import lex -from karp.lex.domain import commands # from karp.lex.domain.errors import ResourceAlreadyPublished From f00a13c5ee29f504b7bf8a935418abf586901585 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Wed, 6 Apr 2022 13:48:58 +0200 Subject: [PATCH 37/43] fix: print --- karp/cliapp/subapps/resource_subapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karp/cliapp/subapps/resource_subapp.py b/karp/cliapp/subapps/resource_subapp.py index 499c7a87..e079cec0 100644 --- a/karp/cliapp/subapps/resource_subapp.py +++ b/karp/cliapp/subapps/resource_subapp.py @@ -272,7 +272,7 @@ def delete( message=message or "resource deleted", ) resource = bus.dispatch(cmd) - typer.echo(f"Deleted resource '{resource.resource_id}' ({resource.entity_id})") + typer.echo(f"Deleted resource '{resource_id}' ({resource})") def init_app(app): From 9697a87a9716fab89f7bbe209368b4340a2ee79d Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Wed, 6 Apr 2022 13:49:18 +0200 Subject: [PATCH 38/43] fix: display entity_id also --- karp/foundation/entity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/karp/foundation/entity.py b/karp/foundation/entity.py index ecd1c5e5..a7483d76 100644 --- a/karp/foundation/entity.py +++ b/karp/foundation/entity.py @@ -46,7 +46,9 @@ def discard(self) -> None: def _check_not_discarded(self): if self._discarded: - raise self.DiscardedEntityError(f"Attempt to use {self!r}") + raise self.DiscardedEntityError( + f"Attempt to use {self!r}, entity_id = {self.entity_id}" + ) def _validate_event_applicability(self, event): if event.entity_id != self.id: From f04667e2af5c4572a982cacef8d8b95133fbf767 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 7 Apr 2022 10:49:11 +0200 Subject: [PATCH 39/43] fix: update --- karp/lex/domain/entities/resource.py | 2 +- karp/webapp/routes/resources_api.py | 83 ++++++++++++++-------------- karp/webapp/schemas.py | 7 ++- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/karp/lex/domain/entities/resource.py b/karp/lex/domain/entities/resource.py index 7ab1ed98..71040344 100644 --- a/karp/lex/domain/entities/resource.py +++ b/karp/lex/domain/entities/resource.py @@ -210,7 +210,7 @@ def publish( timestamp: float = None, ): self._extracted_from_publish_9(timestamp, user, message, "Published") - self._version += 1 + self._increment_version() self.is_published = True self.queue_event( events.ResourcePublished( diff --git a/karp/webapp/routes/resources_api.py b/karp/webapp/routes/resources_api.py index 51cdccb4..1b1f9178 100644 --- a/karp/webapp/routes/resources_api.py +++ b/karp/webapp/routes/resources_api.py @@ -8,7 +8,7 @@ from karp import auth, lex from karp.auth.application.queries import GetResourcePermissions, ResourcePermissionDto -from karp.webapp.schemas import ResourceCreate, ResourcePublic +from karp.webapp.schemas import ResourceCreate, ResourcePublic, ResourceProtected from karp.webapp import dependencies as deps, schemas from karp.webapp.dependencies.fastapi_injector import inject_from_req @@ -26,40 +26,37 @@ logger = logging.getLogger(__name__) -@router.get( - '/permissions', - response_model=list[ResourcePermissionDto]) +@router.get("/permissions", response_model=list[ResourcePermissionDto]) def list_resource_permissions( query: GetResourcePermissions = Depends(deps.get_resource_permissions), ): return query.query() -@router.get('/') +@router.get( + "/", + response_model=list[ResourceProtected], +) def get_all_resources( - get_resources: lex.GetResources = Depends( - deps.inject_from_req(lex.GetResources)), + get_resources: lex.GetResources = Depends(deps.inject_from_req(lex.GetResources)), ) -> list[lex.ResourceDto]: return get_resources.query() -@router.post( - '/', - response_model=ResourceDto, - status_code=status.HTTP_201_CREATED -) +@router.post("/", response_model=ResourceDto, status_code=status.HTTP_201_CREATED) def create_new_resource( new_resource: ResourceCreate = Body(...), user: auth.User = Security(deps.get_user, scopes=["admin"]), auth_service: auth.AuthService = Depends(deps.get_auth_service), - creating_resource_uc: CreatingResource = Depends( - deps.get_lex_uc(CreatingResource)), + creating_resource_uc: CreatingResource = Depends(deps.get_lex_uc(CreatingResource)), creating_entry_repo_uc: CreatingEntryRepo = Depends( - deps.get_lex_uc(CreatingEntryRepo)), + deps.get_lex_uc(CreatingEntryRepo) + ), ) -> ResourceDto: - logger.info('creating new resource', - extra={'user': user.identifier, - 'resource': new_resource}) + logger.info( + "creating new resource", + extra={"user": user.identifier, "resource": new_resource}, + ) if not auth_service.authorize( auth.PermissionLevel.admin, user, [new_resource.resource_id] ): @@ -72,11 +69,11 @@ def create_new_resource( if new_resource.entry_repo_id is None: entry_repo = creating_entry_repo_uc.execute( commands.CreateEntryRepository( - repository_type='default', + repository_type="default", name=new_resource.resource_id, config=new_resource.config, user=user.identifier, - message=new_resource.message or 'Entry repository created', + message=new_resource.message or "Entry repository created", ) ) new_resource.entry_repo_id = entry_repo.entity_id @@ -85,19 +82,20 @@ def create_new_resource( **new_resource.dict(), ) resource = creating_resource_uc.execute(create_resource) - logger.info('resource created', extra={'resource': resource}) + logger.info("resource created", extra={"resource": resource}) return resource except Exception as err: - logger.exception('error occured', extra={'user': user.identifier, - 'resource': new_resource}) + logger.exception( + "error occured", extra={"user": user.identifier, "resource": new_resource} + ) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f'{err=}', + detail=f"{err=}", ) -@ router.post( - '/{resource_id}/publish', +@router.post( + "/{resource_id}/publish", # response_model=ResourceDto, status_code=status.HTTP_200_OK, ) @@ -107,18 +105,19 @@ def publishing_resource( user: auth.User = Security(deps.get_user, scopes=["admin"]), auth_service: auth.AuthService = Depends(deps.get_auth_service), publishing_resource_uc: lex.PublishingResource = Depends( - deps.get_lex_uc(lex.PublishingResource)), + deps.get_lex_uc(lex.PublishingResource) + ), ): - if not auth_service.authorize( - auth.PermissionLevel.admin, user, [resource_id] - ): + if not auth_service.authorize(auth.PermissionLevel.admin, user, [resource_id]): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Not enough permissions", headers={"WWW-Authenticate": 'Bearer scope="lexica:admin"'}, ) - logger.info('publishing resource', - extra={'resource_id': resource_id, 'user': user.identifier}) + logger.info( + "publishing resource", + extra={"resource_id": resource_id, "user": user.identifier}, + ) try: resource_publish.resource_id = resource_id publish_resource = commands.PublishResource( @@ -126,26 +125,26 @@ def publishing_resource( **resource_publish.dict(), ) publishing_resource_uc.execute(publish_resource) - logger.info("resource published", extra={ - 'resource_id': resource_id}) + logger.info("resource published", extra={"resource_id": resource_id}) return except Exception as err: - logger.exception('error occured when publishing', - extra={'resource_id': resource_id, 'user': user.identifier}) + logger.exception( + "error occured when publishing", + extra={"resource_id": resource_id, "user": user.identifier}, + ) raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=f'{err=}', - ) + detail=f"{err=}", + ) from err -@ router.get( - '/{resource_id}', +@router.get( + "/{resource_id}", response_model=ResourcePublic, ) def get_resource_by_resource_id( resource_id: str, - resource_repo: ReadOnlyResourceRepository = Depends( - deps.get_resources_read_repo), + resource_repo: ReadOnlyResourceRepository = Depends(deps.get_resources_read_repo), ) -> ResourcePublic: resource = resource_repo.get_by_resource_id(resource_id) if not resource: diff --git a/karp/webapp/schemas.py b/karp/webapp/schemas.py index e05b6af4..ec1b1d2a 100644 --- a/karp/webapp/schemas.py +++ b/karp/webapp/schemas.py @@ -72,7 +72,9 @@ class ResourceBase(BaseModel): name: str config: typing.Dict message: Optional[str] = None - entry_repo_id: Optional[unique_id.UniqueId] + entry_repository_id: Optional[unique_id.UniqueId] + is_published: bool + version: int class ResourceCreate(ResourceBase): @@ -81,6 +83,9 @@ class ResourceCreate(ResourceBase): class ResourcePublic(EntityIdMixin, ResourceBase): last_modified: float + + +class ResourceProtected(ResourcePublic): last_modified_by: str From 230042fa6d291bc04ecf9e78488e8352adc4bca7 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 7 Apr 2022 10:54:49 +0200 Subject: [PATCH 40/43] fix: add is_published --- karp/webapp/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karp/webapp/schemas.py b/karp/webapp/schemas.py index ec1b1d2a..3716d340 100644 --- a/karp/webapp/schemas.py +++ b/karp/webapp/schemas.py @@ -73,8 +73,8 @@ class ResourceBase(BaseModel): config: typing.Dict message: Optional[str] = None entry_repository_id: Optional[unique_id.UniqueId] - is_published: bool - version: int + is_published: Optional[bool] + version: Optional[int] class ResourceCreate(ResourceBase): From 8da74051e12ffb782461e5611bd3010862ccbdaf Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Sat, 9 Apr 2022 14:00:40 +0200 Subject: [PATCH 41/43] stub: add resource_config --- karp/lex/domain/value_objects/__init__.py | 1 + karp/lex/domain/value_objects/resource_config.py | 5 +++++ .../unit/lex/value_objects/test_resource_config.py | 10 ++++++++++ 3 files changed, 16 insertions(+) create mode 100644 karp/lex/domain/value_objects/resource_config.py create mode 100644 karp/tests/unit/lex/value_objects/test_resource_config.py diff --git a/karp/lex/domain/value_objects/__init__.py b/karp/lex/domain/value_objects/__init__.py index 467b2f34..49d8e210 100644 --- a/karp/lex/domain/value_objects/__init__.py +++ b/karp/lex/domain/value_objects/__init__.py @@ -1,2 +1,3 @@ from karp.foundation.value_objects.unique_id import UniqueId, make_unique_id from karp.lex.domain.value_objects.entry_schema import EntrySchema +from karp.lex.domain.value_objects.resource_config import ResourceConfig diff --git a/karp/lex/domain/value_objects/resource_config.py b/karp/lex/domain/value_objects/resource_config.py new file mode 100644 index 00000000..4cfaddf4 --- /dev/null +++ b/karp/lex/domain/value_objects/resource_config.py @@ -0,0 +1,5 @@ +import pydantic + + +class ResourceConfig(pydantic.BaseModel): + fields: dict diff --git a/karp/tests/unit/lex/value_objects/test_resource_config.py b/karp/tests/unit/lex/value_objects/test_resource_config.py new file mode 100644 index 00000000..ba75383d --- /dev/null +++ b/karp/tests/unit/lex/value_objects/test_resource_config.py @@ -0,0 +1,10 @@ +import pytest +import pydantic + +from karp.lex.domain.value_objects import ResourceConfig + + +class TestResourceConfig: + def test_invalid_input_raises(self): + with pytest.raises(pydantic.ValidationError): + ResourceConfig(fields=3) From 394f23b9bb613215aab4a67c79e7a51edff09529 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Mon, 11 Apr 2022 06:46:27 +0200 Subject: [PATCH 42/43] fix: temporary disable check on sort --- karp/webapp/routes/query_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karp/webapp/routes/query_api.py b/karp/webapp/routes/query_api.py index 357b6e4e..253a0d1e 100644 --- a/karp/webapp/routes/query_api.py +++ b/karp/webapp/routes/query_api.py @@ -69,7 +69,7 @@ def query_split( sort: str = Query( None, description="The `field` to sort by. If missing, default order for each resource will be used.", - regex=r"^[a-zA-Z0-9_\-]+\|(asc|desc)", + # regex=r"^[a-zA-Z0-9_\-]+\|(asc|desc)", ), lexicon_stats: bool = Query( True, description="Show the hit count per lexicon"), @@ -151,7 +151,7 @@ def query( sort: List[str] = Query( [], description="The `field` to sort by. If missing, default order for each resource will be used.", - regex=r"^[a-zA-Z0-9_\-]+\|(asc|desc)", + # regex=r"^[a-zA-Z0-9_\-]+\|(asc|desc)", ), lexicon_stats: bool = Query( True, description="Show the hit count per lexicon"), From c06654a2b8c43305579a4fd6e04f9157ff5815a7 Mon Sep 17 00:00:00 2001 From: Kristoffer Andersson Date: Thu, 21 Apr 2022 12:52:51 +0200 Subject: [PATCH 43/43] fix: add defaults to schema --- .../use_cases/entry_repo_handlers.py | 14 ++-- karp/tests/e2e/test_resources_api.py | 78 +++++++++---------- karp/webapp/routes/resources_api.py | 8 +- karp/webapp/schemas.py | 6 +- 4 files changed, 47 insertions(+), 59 deletions(-) diff --git a/karp/lex/application/use_cases/entry_repo_handlers.py b/karp/lex/application/use_cases/entry_repo_handlers.py index 34ef3f4c..8edaec57 100644 --- a/karp/lex/application/use_cases/entry_repo_handlers.py +++ b/karp/lex/application/use_cases/entry_repo_handlers.py @@ -1,7 +1,7 @@ import logging -from typing import Any from karp.foundation.commands import CommandHandler +from karp.lex.application.repositories.entries import EntryUnitOfWork from karp.lex.domain import commands from karp.lex.application import repositories @@ -17,10 +17,7 @@ def __init__( ): self._entry_repo_uow = entry_repo_uow - def execute( - self, - command: commands.CreateEntryRepository - ) -> None: + def execute(self, command: commands.CreateEntryRepository) -> EntryUnitOfWork: entry_repo = self._entry_repo_uow.factory.create( repository_type=command.repository_type, entity_id=command.entity_id, @@ -32,9 +29,9 @@ def execute( message=command.message, ) - logger.debug('Created entry repo', extra={'entry_repo': entry_repo}) + logger.debug("Created entry repo", extra={"entry_repo": entry_repo}) with self._entry_repo_uow as uow: - logger.debug('Saving...') + logger.debug("Saving...") uow.repo.save(entry_repo) uow.commit() return entry_repo @@ -51,7 +48,6 @@ def __init__( def execute(self, command: commands.DeleteEntryRepository) -> None: with self._entry_repo_uow as uow: entry_repo = uow.repo.get_by_id(command.entity_id) - entry_repo.discard( - user=command.user, timestamp=command.timestamp) + entry_repo.discard(user=command.user, timestamp=command.timestamp) uow.repo.save(entry_repo) uow.commit() diff --git a/karp/tests/e2e/test_resources_api.py b/karp/tests/e2e/test_resources_api.py index 387f544e..516ba5b5 100644 --- a/karp/tests/e2e/test_resources_api.py +++ b/karp/tests/e2e/test_resources_api.py @@ -9,39 +9,37 @@ @pytest.fixture def new_resource() -> ResourceCreate: return ResourceCreate( - resource_id='test_resource', - name='Test resource', - message='test', + resource_id="test_resource", + name="Test resource", + message="test", config={ - 'fields': { - 'foo': {'type': 'string'} - }, - 'id': 'foo', + "fields": {"foo": {"type": "string"}}, + "id": "foo", }, ) class TestResourcesRoutes: def test_get_resource_exist(self, fa_data_client): - response = fa_data_client.get('/resources/places') + response = fa_data_client.get("/resources/places") assert response.status_code != status.HTTP_404_NOT_FOUND - def test_get_resources_exist(self, fa_data_client): - response = fa_data_client.get('/resources') + def test_get_resources_exist(self, fa_client): + response = fa_client.get("/resources") assert response.status_code != status.HTTP_404_NOT_FOUND - def test_post_resources_exist(self, fa_data_client): - response = fa_data_client.post("/resources") + def test_post_resources_exist(self, fa_client): + response = fa_client.post("/resources") assert response.status_code != status.HTTP_404_NOT_FOUND - def test_get_resource_permissionss_exist(self, fa_data_client): - response = fa_data_client.get('/resources/permissions') + def test_get_resource_permissionss_exist(self, fa_client): + response = fa_client.get("/resources/permissions") assert response.status_code != status.HTTP_404_NOT_FOUND class TestGetResourcePermissions: def test_get_resources(self, fa_data_client): - response = fa_data_client.get('/resources/permissions') + response = fa_data_client.get("/resources/permissions") assert response.status_code == status.HTTP_200_OK @@ -49,68 +47,64 @@ def test_get_resources(self, fa_data_client): assert len(response_data) == 2 assert response_data[0] == { - 'resource_id': 'places', - 'protected': None, + "resource_id": "places", + "protected": None, } assert response_data[1] == { - 'resource_id': 'municipalities', - 'protected': 'READ', + "resource_id": "municipalities", + "protected": "READ", } class TestCreateResource: - def test_missing_auth_header_returns_403(self, fa_data_client): - response = fa_data_client.post("/resources/", json={}) + def test_missing_auth_header_returns_403(self, fa_client): + response = fa_client.post("/resources/", json={}) assert response.status_code == status.HTTP_403_FORBIDDEN - def test_invalid_data_returns_422( - self, - fa_data_client, - admin_token: auth.AccessToken - ): - response = fa_data_client.post( + def test_invalid_data_returns_422(self, fa_client, admin_token: auth.AccessToken): + response = fa_client.post( "/resources/", json={}, headers=admin_token.as_header(), ) - print(f'{response.json()=}') + print(f"{response.json()=}") assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY def test_valid_input_creates_resource( - self, - fa_data_client, - new_resource: ResourceCreate, - admin_token: auth.AccessToken + self, fa_client, new_resource: ResourceCreate, admin_token: auth.AccessToken ): - response = fa_data_client.post( - '/resources/', + response = fa_client.post( + "/resources/", json=new_resource.dict(), headers=admin_token.as_header(), ) - print(f'{response.json()=}') + print(f"{response.json()=}") assert response.status_code == status.HTTP_201_CREATED created_resource = ResourceCreate( **response.json(), ) - assert created_resource == new_resource + expected_resource = new_resource + expected_resource.version = 1 + expected_resource.is_published = False + assert created_resource == expected_resource class TestGetResource: - def test_get_resource_by_resource_id(self, fa_data_client): - response = fa_data_client.get('/resources/test_resource') - print(f'{response.json()=}') + def test_get_resource_by_resource_id(self, fa_client): + response = fa_client.get("/resources/test_resource") + print(f"{response.json()=}") assert response.status_code == status.HTTP_200_OK resource = ResourcePublic(**response.json()) - assert resource.resource_id == 'test_resource' + assert resource.resource_id == "test_resource" class TestGetResources: def test_get_resources(self, fa_data_client): - response = fa_data_client.get('/resources/') + response = fa_data_client.get("/resources/") response_data = response.json() - print(f'{response_data=}') + print(f"{response_data=}") assert response.status_code == status.HTTP_200_OK assert len(response_data) >= 3 diff --git a/karp/webapp/routes/resources_api.py b/karp/webapp/routes/resources_api.py index 1b1f9178..3748fa8b 100644 --- a/karp/webapp/routes/resources_api.py +++ b/karp/webapp/routes/resources_api.py @@ -1,21 +1,19 @@ import logging -from typing import Dict, List +import typing from fastapi import APIRouter, Body, Depends, HTTPException, Security, status import logging -from karp.foundation.commands import CommandBus from karp import auth, lex from karp.auth.application.queries import GetResourcePermissions, ResourcePermissionDto +from karp.foundation.commands import CommandBus from karp.webapp.schemas import ResourceCreate, ResourcePublic, ResourceProtected from karp.webapp import dependencies as deps, schemas -from karp.webapp.dependencies.fastapi_injector import inject_from_req from karp.lex import ( CreatingEntryRepo, CreatingResource, - ResourceUnitOfWork, ResourceDto, ) from karp.lex.domain import commands @@ -39,7 +37,7 @@ def list_resource_permissions( ) def get_all_resources( get_resources: lex.GetResources = Depends(deps.inject_from_req(lex.GetResources)), -) -> list[lex.ResourceDto]: +) -> typing.Iterable[lex.ResourceDto]: return get_resources.query() diff --git a/karp/webapp/schemas.py b/karp/webapp/schemas.py index 3716d340..ea248240 100644 --- a/karp/webapp/schemas.py +++ b/karp/webapp/schemas.py @@ -72,9 +72,9 @@ class ResourceBase(BaseModel): name: str config: typing.Dict message: Optional[str] = None - entry_repository_id: Optional[unique_id.UniqueId] - is_published: Optional[bool] - version: Optional[int] + entry_repo_id: Optional[unique_id.UniqueId] = None + is_published: Optional[bool] = None + version: Optional[int] = None class ResourceCreate(ResourceBase):