From 5530d127f0c9b86cdf29a3a1e76f9032ffb8c3f4 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Fri, 29 Mar 2024 00:32:22 +1100 Subject: [PATCH 1/3] add: support for entities and entity_lists --- pyodk/_endpoints/entities.py | 194 +++++++++++++++++++++++++++ pyodk/_endpoints/entity_lists.py | 79 +++++++++++ pyodk/_utils/validators.py | 16 +++ pyodk/client.py | 8 ++ tests/endpoints/test_entities.py | 48 +++++++ tests/endpoints/test_entity_lists.py | 25 ++++ tests/resources/entities_data.py | 41 ++++++ tests/resources/entity_lists_data.py | 14 ++ 8 files changed, 425 insertions(+) create mode 100644 pyodk/_endpoints/entities.py create mode 100644 pyodk/_endpoints/entity_lists.py create mode 100644 tests/endpoints/test_entities.py create mode 100644 tests/endpoints/test_entity_lists.py create mode 100644 tests/resources/entities_data.py create mode 100644 tests/resources/entity_lists_data.py diff --git a/pyodk/_endpoints/entities.py b/pyodk/_endpoints/entities.py new file mode 100644 index 0000000..c06a90a --- /dev/null +++ b/pyodk/_endpoints/entities.py @@ -0,0 +1,194 @@ +import logging +from datetime import datetime +from uuid import uuid4 + +from pyodk._endpoints import bases +from pyodk._utils import validators as pv +from pyodk._utils.session import Session +from pyodk.errors import PyODKError + +log = logging.getLogger(__name__) + + +class CurrentVersion(bases.Model): + label: str + current: bool + creatorId: int + userAgent: str + version: int + baseVersion: int | None = None + conflictingProperties: list[str] | None = None + + +class Entity(bases.Model): + uuid: str + creatorId: int + createdAt: datetime + currentVersion: CurrentVersion + updatedAt: datetime | None = None + deletedAt: datetime | None = None + + +class URLs(bases.Model): + class Config: + frozen = True + + _entity_name: str = "projects/{project_id}/datasets/{el_name}" + list: str = f"{_entity_name}/entities" + post: str = f"{_entity_name}/entities" + get_table: str = f"{_entity_name}.svc/Entities" + + +class EntityService(bases.Service): + """ + Entity-related functionality is accessed through `client.entities`. For example: + + ```python + from pyodk.client import Client + + client = Client() + data = client.entities.list() + ``` + + An EntityList is a list of Entities, e.g. `list[Entity]`. + """ + + __slots__ = ("urls", "session", "default_project_id", "default_entity_list_name") + + def __init__( + self, + session: Session, + default_project_id: int | None = None, + default_entity_list_name: str | None = None, + urls: URLs = None, + ): + self.urls: URLs = urls if urls is not None else URLs() + self.session: Session = session + self.default_project_id: int | None = default_project_id + self.default_entity_list_name: str | None = default_entity_list_name + + def list( + self, entity_list_name: str | None = None, project_id: int | None = None + ) -> list[Entity]: + """ + Read all Entity metadata. + + :param entity_list_name: The name of the Entity List (Dataset) being referenced. + :param project_id: The id of the project the Entity belongs to. + + :return: A list of the object representation of all Entity metadata. + """ + try: + pid = pv.validate_project_id(project_id, self.default_project_id) + eln = pv.validate_entity_list_name( + entity_list_name, self.default_entity_list_name + ) + except PyODKError as err: + log.error(err, exc_info=True) + raise + + response = self.session.response_or_error( + method="GET", + url=self.session.urlformat(self.urls.list, project_id=pid, el_name=eln), + logger=log, + ) + data = response.json() + return [Entity(**r) for r in data] + + def create( + self, + label: str, + data: dict, + entity_list_name: str | None = None, + project_id: int | None = None, + uuid: str | None = None, + ) -> Entity: + """ + Create an Entity. + + :param label: Label of the Entity. + :param data: Data to store for the Entity. + :param entity_list_name: The name of the Entity List (Dataset) being referenced. + :param project_id: The id of the project this form belongs to. + :param uuid: An optional unique identifier for the Entity. If not provided then + a uuid will be generated and sent by the client. + """ + try: + pid = pv.validate_project_id(project_id, self.default_project_id) + eln = pv.validate_entity_list_name( + entity_list_name, self.default_entity_list_name + ) + req_data = { + "uuid": pv.validate_str(uuid, str(uuid4()), key="uuid"), + "label": pv.validate_str(label, key="label"), + "data": pv.validate_dict(data, key="data"), + } + except PyODKError as err: + log.error(err, exc_info=True) + raise + + response = self.session.response_or_error( + method="POST", + url=self.session.urlformat(self.urls.post, project_id=pid, el_name=eln), + logger=log, + data=req_data, + ) + data = response.json() + return Entity(**data) + + def get_table( + self, + entity_list_name: str | None = None, + project_id: int | None = None, + skip: int | None = None, + top: int | None = None, + count: bool | None = None, + filter: str | None = None, + select: str | None = None, + ) -> dict: + """ + Read Entity List data. + + :param entity_list_name: The name of the Entity List (Dataset) being referenced. + :param project_id: The id of the project this form belongs to. + :param skip: The first n rows will be omitted from the results. + :param top: Only up to n rows will be returned in the results. + :param count: If True, an @odata.count property will be added to the result to + indicate the total number of rows, ignoring the above paging parameters. + :param filter: Filter responses to those matching the query. Only certain fields + are available to reference. The operators lt, le, eq, neq, ge, gt, not, and, + and or are supported, and the built-in functions now, year, month, day, hour, + minute, second. + :param select: If provided, will return only the selected fields. + + :return: A dictionary representation of the OData JSON document. + """ + try: + pid = pv.validate_project_id(project_id, self.default_project_id) + eln = pv.validate_entity_list_name( + entity_list_name, self.default_entity_list_name + ) + params = { + k: v + for k, v in { + "$skip": skip, + "$top": top, + "$count": count, + "$filter": filter, + "$select": select, + }.items() + if v is not None + } + except PyODKError as err: + log.error(err, exc_info=True) + raise + + response = self.session.response_or_error( + method="GET", + url=self.session.urlformat( + self.urls.get_table, project_id=pid, el_name=eln, table_name="Entities" + ), + logger=log, + params=params, + ) + return response.json() diff --git a/pyodk/_endpoints/entity_lists.py b/pyodk/_endpoints/entity_lists.py new file mode 100644 index 0000000..a91a7c1 --- /dev/null +++ b/pyodk/_endpoints/entity_lists.py @@ -0,0 +1,79 @@ +import logging +from datetime import datetime + +from pyodk._endpoints import bases +from pyodk._utils import validators as pv +from pyodk._utils.session import Session +from pyodk.errors import PyODKError + +log = logging.getLogger(__name__) + + +class EntityList(bases.Model): + name: str + projectId: int + createdAt: datetime + approvalRequired: bool + + +class URLs(bases.Model): + class Config: + frozen = True + + list: str = "projects/{project_id}/datasets" + + +class EntityListService(bases.Service): + """ + Entity List-related functionality is accessed through `client.entity_lists`. + + For example: + + ```python + from pyodk.client import Client + + client = Client() + data = client.entity_lists.list() + ``` + + The structure this class works with is conceptually a list of lists, e.g. + + ``` + EntityList = list[Entity] + self.list() = list[EntityList] + ``` + """ + + __slots__ = ("urls", "session", "default_project_id") + + def __init__( + self, + session: Session, + default_project_id: int | None = None, + urls: URLs = None, + ): + self.urls: URLs = urls if urls is not None else URLs() + self.session: Session = session + self.default_project_id: int | None = default_project_id + + def list(self, project_id: int | None = None) -> list[EntityList]: + """ + Read Entity List details. + + :param project_id: The id of the project the Entity List belongs to. + + :return: A list of the object representation of all Entity Lists' details. + """ + try: + pid = pv.validate_project_id(project_id, self.default_project_id) + except PyODKError as err: + log.error(err, exc_info=True) + raise + + response = self.session.response_or_error( + method="GET", + url=self.session.urlformat(self.urls.list, project_id=pid), + logger=log, + ) + data = response.json() + return [EntityList(**r) for r in data] diff --git a/pyodk/_utils/validators.py b/pyodk/_utils/validators.py index 6d14e44..3b3281a 100644 --- a/pyodk/_utils/validators.py +++ b/pyodk/_utils/validators.py @@ -57,6 +57,14 @@ def validate_instance_id(*args: str) -> str: ) +def validate_entity_list_name(*args: str) -> str: + return wrap_error( + validator=v.str_validator, + key="entity_list_name", + value=coalesce(*args), + ) + + def validate_str(*args: str, key: str) -> str: return wrap_error( validator=v.str_validator, @@ -81,6 +89,14 @@ def validate_int(*args: int, key: str) -> int: ) +def validate_dict(*args: dict, key: str) -> int: + return wrap_error( + validator=v.dict_validator, + key=key, + value=coalesce(*args), + ) + + def validate_file_path(*args: str) -> Path: def validate_fp(f): p = v.path_validator(f) diff --git a/pyodk/client.py b/pyodk/client.py index f8a26a2..06c2104 100644 --- a/pyodk/client.py +++ b/pyodk/client.py @@ -1,6 +1,8 @@ from collections.abc import Callable from pyodk._endpoints.comments import CommentService +from pyodk._endpoints.entities import EntityService +from pyodk._endpoints.entity_lists import EntityListService from pyodk._endpoints.forms import FormService from pyodk._endpoints.projects import ProjectService from pyodk._endpoints.submissions import SubmissionService @@ -67,6 +69,12 @@ def __init__( self._comments: CommentService = CommentService( session=self.session, default_project_id=self.project_id ) + self.entities: EntityService = EntityService( + session=self.session, default_project_id=self.project_id + ) + self.entity_lists: EntityListService = EntityListService( + session=self.session, default_project_id=self.project_id + ) @property def project_id(self) -> int | None: diff --git a/tests/endpoints/test_entities.py b/tests/endpoints/test_entities.py new file mode 100644 index 0000000..55f4ba8 --- /dev/null +++ b/tests/endpoints/test_entities.py @@ -0,0 +1,48 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from pyodk._endpoints.entities import Entity +from pyodk._utils.session import Session +from pyodk.client import Client + +from tests.resources import CONFIG_DATA, entities_data + + +@patch("pyodk._utils.session.Auth.login", MagicMock()) +@patch("pyodk._utils.config.read_config", MagicMock(return_value=CONFIG_DATA)) +class TestEntities(TestCase): + def test_list__ok(self): + """Should return a list of Entity objects.""" + fixture = entities_data.test_entities + with patch.object(Session, "request") as mock_session: + mock_session.return_value.status_code = 200 + mock_session.return_value.json.return_value = fixture + with Client() as client: + observed = client.entities.list(entity_list_name="test") + self.assertEqual(2, len(observed)) + for i, o in enumerate(observed): + with self.subTest(i): + self.assertIsInstance(o, Entity) + + def test_create__ok(self): + """Should return an Entity object.""" + fixture = entities_data.test_entities + with patch.object(Session, "request") as mock_session: + mock_session.return_value.status_code = 200 + mock_session.return_value.json.return_value = fixture[0] + with Client() as client: + # Specify project + observed = client.entities.create( + project_id=2, + entity_list_name="test", + label="John (88)", + data=entities_data.test_entities_data, + ) + self.assertIsInstance(observed, Entity) + # Use default + observed = client.entities.create( + entity_list_name="test", + label="John (88)", + data=entities_data.test_entities_data, + ) + self.assertIsInstance(observed, Entity) diff --git a/tests/endpoints/test_entity_lists.py b/tests/endpoints/test_entity_lists.py new file mode 100644 index 0000000..517c121 --- /dev/null +++ b/tests/endpoints/test_entity_lists.py @@ -0,0 +1,25 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from pyodk._endpoints.entity_lists import EntityList +from pyodk._utils.session import Session +from pyodk.client import Client + +from tests.resources import CONFIG_DATA, entity_lists_data + + +@patch("pyodk._utils.session.Auth.login", MagicMock()) +@patch("pyodk._utils.config.read_config", MagicMock(return_value=CONFIG_DATA)) +class TestEntityLists(TestCase): + def test_list__ok(self): + """Should return a list of EntityList objects.""" + fixture = entity_lists_data.test_entity_lists + with patch.object(Session, "request") as mock_session: + mock_session.return_value.status_code = 200 + mock_session.return_value.json.return_value = fixture + with Client() as client: + observed = client.entity_lists.list() + self.assertEqual(2, len(observed)) + for i, o in enumerate(observed): + with self.subTest(i): + self.assertIsInstance(o, EntityList) diff --git a/tests/resources/entities_data.py b/tests/resources/entities_data.py new file mode 100644 index 0000000..512d937 --- /dev/null +++ b/tests/resources/entities_data.py @@ -0,0 +1,41 @@ +test_entities = [ + { + "uuid": "uuid:85cb9aff-005e-4edd-9739-dc9c1a829c44", + "createdAt": "2018-01-19T23:58:03.395Z", + "updatedAt": "2018-03-21T12:45:02.312Z", + "deletedAt": "2018-03-21T12:45:02.312Z", + "creatorId": 1, + "currentVersion": { + "label": "John (88)", + "current": True, + "createdAt": "2018-03-21T12:45:02.312Z", + "creatorId": 1, + "userAgent": "Enketo/3.0.4", + "version": 1, + "baseVersion": None, + "conflictingProperties": None, + }, + }, + { + "uuid": "uuid:85cb9aff-005e-4edd-9739-dc9c1a829c45", + "createdAt": "2018-01-19T23:58:03.395Z", + "updatedAt": "2018-03-21T12:45:02.312Z", + "deletedAt": "2018-03-21T12:45:02.312Z", + "creatorId": 1, + "conflict": "soft", + "currentVersion": { + "label": "John (89)", + "current": True, + "createdAt": "2018-03-21T12:45:02.312Z", + "creatorId": 1, + "userAgent": "Enketo/3.0.4", + "version": 1, + "baseVersion": None, + "conflictingProperties": None, + }, + }, +] +test_entities_data = { + "firstName": "John", + "age": "88", +} diff --git a/tests/resources/entity_lists_data.py b/tests/resources/entity_lists_data.py new file mode 100644 index 0000000..9f2a227 --- /dev/null +++ b/tests/resources/entity_lists_data.py @@ -0,0 +1,14 @@ +test_entity_lists = [ + { + "name": "people", + "createdAt": "2018-01-19T23:58:03.395Z", + "projectId": 1, + "approvalRequired": True, + }, + { + "name": "places", + "createdAt": "2018-01-19T23:58:03.396Z", + "projectId": 1, + "approvalRequired": False, + }, +] From 1715fddfc2c704c98f7f7c2debbf36ed4e6c4f1e Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Thu, 18 Apr 2024 00:03:43 +1000 Subject: [PATCH 2/3] add: e2e tests for entities and entity lists - fix: entity.py `create()` should send JSON not form-encoded data - fix: projects.py typo in comment - chg: readme.md make description generic rather than listing everything - add: test entity_lists.py creation of test fixture with properties - add: e2e test cases for entities and entity_lists functionality --- README.md | 2 +- pyodk/_endpoints/entities.py | 2 +- pyodk/_endpoints/projects.py | 2 +- tests/test_client.py | 35 ++++++++++++++++++++++++++ tests/utils/entity_lists.py | 48 ++++++++++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/utils/entity_lists.py diff --git a/README.md b/README.md index 8745dbc..2404367 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ For interactive testing, debugging, or sanity checking workflows, end-to-end tes 1. Create a test project in Central. 2. Create a test user in Central. It can be a site-wide Administrator. If it is not an Administrator, assign the user to the project with "Project Manager" privileges, so that forms and submissions in the test project can be uploaded and modified. 3. Save the user's credentials and the project ID in a `.pyodk_config.toml` (or equivalent) as described in the above section titled "Configure". -4. When the tests in `test_client.py` are run, the test setup method should automatically create a few forms and submissions for testing with. At a minimum these allow the tests to pass, but can also be used to interactively test or debug. +4. When the tests in `test_client.py` are run, the test setup method should automatically create a few fixtures for testing with. At a minimum these allow the tests to pass, but can also be used to interactively test or debug. ## Release diff --git a/pyodk/_endpoints/entities.py b/pyodk/_endpoints/entities.py index c06a90a..ad80a22 100644 --- a/pyodk/_endpoints/entities.py +++ b/pyodk/_endpoints/entities.py @@ -131,7 +131,7 @@ def create( method="POST", url=self.session.urlformat(self.urls.post, project_id=pid, el_name=eln), logger=log, - data=req_data, + json=req_data, ) data = response.json() return Entity(**data) diff --git a/pyodk/_endpoints/projects.py b/pyodk/_endpoints/projects.py index 8e4159a..f390d2d 100644 --- a/pyodk/_endpoints/projects.py +++ b/pyodk/_endpoints/projects.py @@ -128,7 +128,7 @@ def create_app_users( # The "App User" role_id should always be "2", so no need to look it up by name. # Ref: "https://github.com/getodk/central-backend/blob/9db0d792cf4640ec7329722984 # cebdee3687e479/lib/model/migrations/20181212-01-add-roles.js" - # See also roles data in `tests/resorces/projects_data.py`. + # See also roles data in `tests/resources/projects_data.py`. if forms is not None: for user in users: for form_id in forms: diff --git a/tests/test_client.py b/tests/test_client.py index 777195d..6e00f0a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -5,6 +5,7 @@ from tests.resources import RESOURCES, forms_data, submissions_data from tests.utils import utils +from tests.utils.entity_lists import create_new_or_get_entity_list from tests.utils.forms import ( create_new_form__md, create_new_form__xml, @@ -71,6 +72,22 @@ def create_test_submissions(client: Client | None = None) -> Client: return client +def create_test_entity_lists(client: Client | None = None) -> Client: + """ + Create test entity lists, if they don't already exist. + :param client: Client instance to use for API calls. + :return: The original client instance, or a new one if none was provided. + """ + if client is None: + client = Client() + create_new_or_get_entity_list( + client=client, + entity_list_name="pyodk_test_eln", + entity_props=["test_label", "another_prop"], + ) + return client + + @skip class TestUsage(TestCase): """Tests for experimenting with usage scenarios / general debugging / integration.""" @@ -82,6 +99,7 @@ def setUpClass(cls): cls.client = Client() create_test_forms(client=cls.client) create_test_submissions(client=cls.client) + create_test_entity_lists(client=cls.client) def test_direct(self): projects = self.client.projects.list() @@ -216,3 +234,20 @@ def test_submission_edit__non_ascii(self): instance_id=iid, comment=f"pyODK edit {now}", ) + + def test_entities__create_and_query(self): + """Should create a new entity, and query it afterwards via list() or get_table().""" + self.client.entities.default_entity_list_name = "pyodk_test_eln" + entity = self.client.entities.create( + label="test_label", + data={"test_label": "test_value", "another_prop": "another_value"}, + ) + entity_list = self.client.entities.list() + self.assertIn(entity, entity_list) + entity_data = self.client.entities.get_table(select="__id") + self.assertIn(entity.uuid, [d["__id"] for d in entity_data["value"]]) + + def test_entity_lists__list(self): + """Should return a list of entities""" + observed = self.client.entity_lists.list() + self.assertGreater(len(observed), 0) diff --git a/tests/utils/entity_lists.py b/tests/utils/entity_lists.py new file mode 100644 index 0000000..d6039b1 --- /dev/null +++ b/tests/utils/entity_lists.py @@ -0,0 +1,48 @@ +from pyodk._endpoints.entity_lists import EntityList, log +from pyodk.client import Client +from pyodk.errors import PyODKError + + +def create_new_or_get_entity_list( + client: Client, entity_list_name: str, entity_props: list[str] +) -> EntityList: + """ + Create a new entity list, or get the entity list metadata. + + :param client: Client instance to use for API calls. + :param entity_list_name: Name of the entity list. + :param entity_props: Properties to add to the entity list. + """ + try: + entity_list = client.session.response_or_error( + method="POST", + url=client.session.urlformat( + "projects/{pid}/datasets", + pid=client.project_id, + ), + logger=log, + json={"name": entity_list_name}, + ) + except PyODKError: + entity_list = client.session.get( + url=client.session.urlformat( + "projects/{pid}/datasets/{eln}", + pid=client.project_id, + eln=entity_list_name, + ), + ) + try: + for prop in entity_props: + client.session.response_or_error( + method="GET", + url=client.session.urlformat( + "projects/{pid}/datasets/{eln}/properties", + pid=client.project_id, + eln=entity_list_name, + ), + logger=log, + json={"name": prop}, + ) + except PyODKError: + pass + return EntityList(**entity_list.json()) From dc36fbb19ed7957f616a76ae11b40d31c7912bc7 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Thu, 18 Apr 2024 00:51:05 +1000 Subject: [PATCH 3/3] add: docs stubs for entity-related methods on the client - also tried to make the docstrings a bit clearer --- docs/entities.md | 3 +++ docs/entity_lists.md | 3 +++ mkdocs.yml | 4 +++- pyodk/_endpoints/entities.py | 4 +++- pyodk/_endpoints/entity_lists.py | 8 ++------ 5 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 docs/entities.md create mode 100644 docs/entity_lists.md diff --git a/docs/entities.md b/docs/entities.md new file mode 100644 index 0000000..fc33389 --- /dev/null +++ b/docs/entities.md @@ -0,0 +1,3 @@ +# Entities + +::: pyodk._endpoints.entities.EntityService \ No newline at end of file diff --git a/docs/entity_lists.md b/docs/entity_lists.md new file mode 100644 index 0000000..73b4d2d --- /dev/null +++ b/docs/entity_lists.md @@ -0,0 +1,3 @@ +# Entity Lists + +::: pyodk._endpoints.entity_lists.EntityListService \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 69807b5..5a6fd04 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,9 +11,11 @@ watch: nav: - Overview: index.md - Client: client.md + - .entities: entities.md + - .entity_lists: entity_lists.md - .forms: forms.md - - .submissions: submissions.md - .projects: projects.md + - .submissions: submissions.md - HTTP methods: http-methods.md - Examples: examples/README.md diff --git a/pyodk/_endpoints/entities.py b/pyodk/_endpoints/entities.py index ad80a22..0ac95b4 100644 --- a/pyodk/_endpoints/entities.py +++ b/pyodk/_endpoints/entities.py @@ -50,7 +50,9 @@ class EntityService(bases.Service): data = client.entities.list() ``` - An EntityList is a list of Entities, e.g. `list[Entity]`. + Conceptually, an Entity's parent object is an EntityList. Each EntityList may + have multiple Entities. In Python parlance, EntityLists are like classes, while + Entities are like instances. """ __slots__ = ("urls", "session", "default_project_id", "default_entity_list_name") diff --git a/pyodk/_endpoints/entity_lists.py b/pyodk/_endpoints/entity_lists.py index a91a7c1..bf0f9bb 100644 --- a/pyodk/_endpoints/entity_lists.py +++ b/pyodk/_endpoints/entity_lists.py @@ -36,12 +36,8 @@ class EntityListService(bases.Service): data = client.entity_lists.list() ``` - The structure this class works with is conceptually a list of lists, e.g. - - ``` - EntityList = list[Entity] - self.list() = list[EntityList] - ``` + Conceptually, an EntityList's parent object is a Project. Each Project may have + multiple EntityLists. """ __slots__ = ("urls", "session", "default_project_id")