diff --git a/README.md b/README.md index a2907e2..8745dbc 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,18 @@ You can run tests with: python -m unittest ``` +### Testing + +When adding or updating pyODK functionality, at a minimum add or update corresponding unit tests. The unit tests are filed in `tests/endpoints` or `tests`. These tests focus on pyODK functionality, such as ensuring that data de/serialisation works as expected, and that method logic results in the expected call patterns. The unit tests use mocks and static data, which are stored in `tests/resources`. These data are obtained by making an API call and saving the Python dict returned by `response.json()` as text. + +For interactive testing, debugging, or sanity checking workflows, end-to-end tests are stored in `tests/test_client.py`. These tests are not run by default because they require access to a live Central server. The ODK team use the Central staging instance https://staging.getodk.cloud/ which is already configured for testing. Below are the steps to set up a new project in Central to be able to run these tests. + +1. Create a test project in Central. +2. Create a test user in Central. It can be a site-wide Administrator. If it is not an Administrator, assign the user to the project with "Project Manager" privileges, so that forms and submissions in the test project can be uploaded and modified. +3. Save the user's credentials and the project ID in a `.pyodk_config.toml` (or equivalent) as described in the above section titled "Configure". +4. When the tests in `test_client.py` are run, the test setup method should automatically create a few forms and submissions for testing with. At a minimum these allow the tests to pass, but can also be used to interactively test or debug. + + ## Release 1. Run all linting and tests. diff --git a/pyodk/_endpoints/form_drafts.py b/pyodk/_endpoints/form_drafts.py index 7bf3981..918d9f6 100644 --- a/pyodk/_endpoints/form_drafts.py +++ b/pyodk/_endpoints/form_drafts.py @@ -34,20 +34,21 @@ def __init__( self.default_project_id: int | None = default_project_id self.default_form_id: str | None = default_form_id - def create( + def _prep_form_post( self, - file_path: str | None = None, + file_path: Path | str | None = None, ignore_warnings: bool | None = True, form_id: str | None = None, project_id: int | None = None, - ) -> bool: + ) -> (str, str, dict, dict): """ - Create a Form Draft. + Prepare / validate input arguments for POSTing a new form definition or version. :param file_path: The path to the file to upload. :param form_id: The xmlFormId of the Form being referenced. :param project_id: The id of the project this form belongs to. :param ignore_warnings: If True, create the form if there are XLSForm warnings. + :return: project_id, form_id, headers, params """ try: pid = pv.validate_project_id(project_id, self.default_project_id) @@ -81,6 +82,30 @@ def create( log.error(err, exc_info=True) raise + return pid, fid, headers, params + + def create( + self, + file_path: Path | str | None = None, + ignore_warnings: bool | None = True, + form_id: str | None = None, + project_id: int | None = None, + ) -> bool: + """ + Create a Form Draft. + + :param file_path: The path to the file to upload. + :param form_id: The xmlFormId of the Form being referenced. + :param project_id: The id of the project this form belongs to. + :param ignore_warnings: If True, create the form if there are XLSForm warnings. + """ + pid, fid, headers, params = self._prep_form_post( + file_path=file_path, + ignore_warnings=ignore_warnings, + form_id=form_id, + project_id=project_id, + ) + with open(file_path, "rb") if file_path is not None else nullcontext() as fd: response = self.session.response_or_error( method="POST", diff --git a/pyodk/_utils/session.py b/pyodk/_utils/session.py index fc8597e..3aacb2b 100644 --- a/pyodk/_utils/session.py +++ b/pyodk/_utils/session.py @@ -1,7 +1,7 @@ from logging import Logger from string import Formatter from typing import Any -from urllib.parse import quote_plus, urljoin +from urllib.parse import quote, urljoin from requests import PreparedRequest, Response from requests import Session as RequestsSession @@ -16,11 +16,25 @@ class URLFormatter(Formatter): """ - Makes a valid URL by sending each format input field through urllib.parse.quote_plus. + Makes a valid URL by sending each format input field through urllib.parse.quote. + + To parse/un-parse URLs, currently (v2023.5) Central uses JS default functions + encodeURIComponent and decodeURIComponent, which comply with RFC2396. The more recent + RFC3986 reserves hex characters 2A (asterisk), 27 (single quote), 28 (left + parenthesis), and 29 (right parenthesis). Python 3.7+ urllib.parse complies with + RFC3986 so in order for pyODK to behave as Central expects, these additional 4 + characters are specified as "safe" in `format_field()` to not percent-encode them. + + Currently (v2023.5) Central primarily supports the default submission instanceID + format per the XForm spec, namely "uuid:" followed by the 36 character UUID string. + In many endpoints, custom UUIDs (including non-ASCII/UTF-8 chars) will work, but in + some places they won't. For example the Central page for viewing submission details + fails on the Submissions OData call, because the OData function to filter by ID + (`Submission('instanceId')`) only works for the default instanceID format. """ def format_field(self, value: Any, format_spec: str) -> Any: - return format(quote_plus(str(value)), format_spec) + return format(quote(str(value), safe="*'()"), format_spec) _URL_FORMATTER = URLFormatter() diff --git a/pyproject.toml b/pyproject.toml index 63dc5cb..80622e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ # Install with `pip install pyodk[dev]`. dev = [ "ruff==0.3.4", # Format and lint + "openpyxl==3.1.2" # Create test XLSX files ] docs = [ "mkdocs==1.5.3", diff --git a/tests/endpoints/test_auth.py b/tests/endpoints/test_auth.py index 5e8c8dd..b87b695 100644 --- a/tests/endpoints/test_auth.py +++ b/tests/endpoints/test_auth.py @@ -7,8 +7,8 @@ from pyodk.errors import PyODKError from requests import Session -from tests import utils from tests.resources import CONFIG_DATA +from tests.utils.utils import get_temp_dir @patch("pyodk._utils.config.read_config", MagicMock(return_value=CONFIG_DATA)) @@ -75,7 +75,7 @@ def test_get_token__ok__new_cache(self): AuthService, get_new_token=MagicMock(return_value="123"), ), - utils.get_temp_dir() as tmp, + get_temp_dir() as tmp, ): cache_path = (tmp / "test_cache.toml").as_posix() client = Client(cache_path=cache_path) @@ -99,7 +99,7 @@ def test_get_token__error__new_cache_bad_response(self): verify_token=verify_mock, get_new_token=get_new_mock, ), - utils.get_temp_dir() as tmp, + get_temp_dir() as tmp, self.assertRaises(PyODKError) as err, ): cache_path = tmp / "test_cache.toml" @@ -115,7 +115,7 @@ def test_get_token__ok__existing_cache(self): AuthService, verify_token=MagicMock(return_value="123"), ), - utils.get_temp_dir() as tmp, + get_temp_dir() as tmp, ): cache_path = (tmp / "test_cache.toml").as_posix() client = Client(cache_path=cache_path) @@ -138,7 +138,7 @@ def test_get_token__error__existing_cache_bad_response(self): verify_token=verify_mock, get_new_token=MagicMock(return_value="123"), ), - utils.get_temp_dir() as tmp, + get_temp_dir() as tmp, ): cache_path = (tmp / "test_cache.toml").as_posix() client = Client(cache_path=cache_path) diff --git a/tests/resources/forms/non_ascii_form_id.xlsx b/tests/resources/forms/non_ascii_form_id.xlsx deleted file mode 100644 index fb488cb..0000000 Binary files a/tests/resources/forms/non_ascii_form_id.xlsx and /dev/null differ diff --git a/tests/resources/forms/pull_data.xlsx b/tests/resources/forms/pull_data.xlsx deleted file mode 100644 index 1df7f77..0000000 Binary files a/tests/resources/forms/pull_data.xlsx and /dev/null differ diff --git a/tests/resources/forms/range_draft.xml b/tests/resources/forms/range_draft.xml index 692edc3..5f6f6e5 100644 --- a/tests/resources/forms/range_draft.xml +++ b/tests/resources/forms/range_draft.xml @@ -8,7 +8,7 @@ range_draft - + diff --git "a/tests/resources/forms/\342\234\205.xlsx" "b/tests/resources/forms/\342\234\205.xlsx" deleted file mode 100644 index ce3274c..0000000 Binary files "a/tests/resources/forms/\342\234\205.xlsx" and /dev/null differ diff --git a/tests/resources/forms_data.py b/tests/resources/forms_data.py index 5817b93..b997641 100644 --- a/tests/resources/forms_data.py +++ b/tests/resources/forms_data.py @@ -1,3 +1,6 @@ +from datetime import datetime +from pathlib import Path + test_forms = { "project_id": 8, "response_data": [ @@ -71,3 +74,44 @@ }, ], } + + +def get_xml__range_draft(version: str | None = None) -> str: + if version is None: + version = datetime.now().isoformat() + with open(Path(__file__).parent / "forms" / "range_draft.xml") as fd: + return fd.read().format(version=version) + + +def get_md__pull_data(version: str | None = None) -> str: + if version is None: + version = datetime.now().isoformat() + return f""" + | settings | + | | version | + | | {version} | + | survey | | | | | + | | type | name | label | calculation | + | | calculate | fruit | | pulldata('fruits', 'name', 'name_key', 'mango') | + | | note | note_fruit | The fruit ${{fruit}} pulled from csv | | + """ + + +md__symbols = """ +| settings | +| | form_title | form_id | version | +| | a non_ascii_form_id | ''=+/*-451%/% | 1 | +| survey | | | | | +| | type | name | label | calculation | +| | calculate | fruit | | pulldata('fruits', 'name', 'name_key', 'mango') | +| | note | note_fruit | The fruit ${{fruit}} pulled from csv | | +""" +md__dingbat = """ +| settings | +| | form_title | form_id | version | +| | ✅ | ✅ | 1 | +| survey | | | | | +| | type | name | label | calculation | +| | calculate | fruit | | pulldata('fruits', 'name', 'name_key', 'mango') | +| | note | note_fruit | The fruit ${{fruit}} pulled from csv | | +""" diff --git a/tests/resources/submissions_data.py b/tests/resources/submissions_data.py index e377c68..7561f38 100644 --- a/tests/resources/submissions_data.py +++ b/tests/resources/submissions_data.py @@ -45,3 +45,34 @@ 36 """ + + +def get_xml__fruits( + form_id: str, + version: str, + instance_id: str, + deprecated_instance_id: str | None = None, + selected_fruit: str = "Papaya", +) -> str: + """ + Get Submission XML for the "fruits" form that uses an external data list. + + :param form_id: The xmlFormId of the Form being referenced. + :param version: The version of the form that the submission is for. + :param instance_id: The instanceId of the Submission being referenced. + :param deprecated_instance_id: If the submission is an edit, then the instance_id of + the submission being replaced must be provided. + :param selected_fruit: Which delicious tropical fruit do you like? + """ + iidd = "" + if deprecated_instance_id is not None: + iidd = f"{deprecated_instance_id}" + return f""" + + {iidd} + {instance_id} + + {selected_fruit} + + + """ diff --git a/tests/test_client.py b/tests/test_client.py index 1ad39b2..777195d 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -3,27 +3,100 @@ from pyodk.client import Client -from tests.resources import RESOURCES +from tests.resources import RESOURCES, forms_data, submissions_data +from tests.utils import utils +from tests.utils.forms import ( + create_new_form__md, + create_new_form__xml, + get_latest_form_version, +) +from tests.utils.md_table import md_table_to_temp_dir +from tests.utils.submissions import ( + create_new_or_get_last_submission, + create_or_update_submission_with_comment, +) + + +def create_test_forms(client: Client | None = None) -> Client: + """ + Create test forms if they don't already exist. + + :param client: Client instance to use for API calls. + :return: The original client instance, or a new one if none was provided. + """ + if client is None: + client = Client() + create_new_form__xml( + client=client, + form_id="range_draft", + form_def=forms_data.get_xml__range_draft(), + ) + create_new_form__md( + client=client, + form_id="pull_data", + form_def=forms_data.get_md__pull_data(), + ) + create_new_form__md( + client=client, + form_id="non_ascii_form_id", + form_def=forms_data.md__symbols, + ) + create_new_form__md( + client=client, + form_id="✅", + form_def=forms_data.md__dingbat, + ) + return client + + +def create_test_submissions(client: Client | None = None) -> Client: + """ + Create test submissions, if they don't already exist. + + :param client: Client instance to use for API calls. + :return: The original client instance, or a new one if none was provided. + """ + if client is None: + client = Client() + create_or_update_submission_with_comment( + client=client, + form_id="pull_data", + instance_id="uuid:07ee9b2f-2271-474c-b9f3-c92ffba80c79", + ) + create_or_update_submission_with_comment( + client=client, + form_id="pull_data", + instance_id="uuid:4e2d1f60-aa3a-4065-bb97-af69b0cc8187", + ) + return client @skip class TestUsage(TestCase): """Tests for experimenting with usage scenarios / general debugging / integration.""" + client: Client | None = None + + @classmethod + def setUpClass(cls): + cls.client = Client() + create_test_forms(client=cls.client) + create_test_submissions(client=cls.client) + def test_direct(self): - client = Client() - projects = client.projects.list() - forms = client.forms.list() - submissions = client.submissions.list(form_id=forms[3].xmlFormId) - form_data = client.submissions.get_table(form_id=forms[3].xmlFormId) - form_data_params = client.submissions.get_table( - form_id="range", + projects = self.client.projects.list() + forms = self.client.forms.list() + submissions = self.client.submissions.list(form_id="pull_data") + form_data = self.client.submissions.get_table(form_id="pull_data") + form_data_params = self.client.submissions.get_table( + form_id="pull_data", table_name="Submissions", count=True, + select="__id,meta/instanceID,__system/formVersion,fruit", ) - comments = client.submissions.list_comments( - form_id="range", - instance_id="uuid:2c296eae-2708-4a89-bfe7-0f2d440b7fe8", + comments = self.client.submissions.list_comments( + form_id="pull_data", + instance_id=next(s.instanceId for s in submissions), ) print([projects, forms, submissions, form_data, form_data_params, comments]) @@ -36,104 +109,110 @@ def test_direct_context(self): # Below tests assume project has forms by these names already published. def test_form_update__new_definition(self): """Should create a new version with the new definition.""" - with Client() as client: - client.forms.update( + with utils.get_temp_file(suffix=".xml") as fp: + fp.write_text(forms_data.get_xml__range_draft()) + self.client.forms.update( form_id="range_draft", - definition=(RESOURCES / "forms" / "range_draft.xml").as_posix(), + definition=fp.as_posix(), ) def test_form_update__new_definition_and_attachments(self): """Should create a new version with new definition and attachment.""" - with Client() as client: - client.forms.update( + # To test the API without a version_updater, a timestamped version is created. + with md_table_to_temp_dir( + form_id="pull_data", mdstr=forms_data.get_md__pull_data() + ) as fp: + self.client.forms.update( form_id="pull_data", - definition=(RESOURCES / "forms" / "pull_data.xlsx").as_posix(), + definition=fp.as_posix(), attachments=[(RESOURCES / "forms" / "fruits.csv").as_posix()], ) def test_form_update__new_definition_and_attachments__non_ascii_dingbat(self): """Should create a new version with new definition and attachment.""" - with Client() as client: - client.forms.update( + with md_table_to_temp_dir( + form_id="✅", mdstr=forms_data.get_md__pull_data() + ) as fp: + self.client.forms.update( form_id="✅", - definition=(RESOURCES / "forms" / "✅.xlsx").as_posix(), + definition=fp.as_posix(), attachments=[(RESOURCES / "forms" / "fruits.csv").as_posix()], ) - form = client.forms.get("✅") + form = self.client.forms.get("✅") self.assertEqual(form.xmlFormId, "✅") def test_form_update__with_version_updater__non_ascii_specials(self): """Should create a new version with new definition and attachment.""" - with Client() as client: - client.forms.update( - form_id="'=+/*-451%/%", - attachments=[], - version_updater=lambda v: datetime.now().isoformat(), - ) + self.client.forms.update( + form_id="'=+/*-451%/%", + attachments=[], + version_updater=lambda v: datetime.now().isoformat(), + ) def test_form_update__attachments(self): """Should create a new version with new attachment.""" - with Client() as client: - client.forms.update( - form_id="pull_data", - attachments=[(RESOURCES / "forms" / "fruits.csv").as_posix()], - ) + self.client.forms.update( + form_id="pull_data", + attachments=[(RESOURCES / "forms" / "fruits.csv").as_posix()], + ) def test_form_update__attachments__with_version_updater(self): """Should create a new version with new attachment and updated version.""" - with Client() as client: - client.forms.update( - form_id="pull_data", - attachments=[(RESOURCES / "forms" / "fruits.csv").as_posix()], - version_updater=lambda v: v + "_1", - ) + self.client.forms.update( + form_id="pull_data", + attachments=[(RESOURCES / "forms" / "fruits.csv").as_posix()], + version_updater=lambda v: v + "_1", + ) def test_project_create_app_users__names_only(self): """Should create project app users.""" - client = Client() - client.projects.create_app_users(display_names=["test_role3", "test_user3"]) + self.client.projects.create_app_users(display_names=["test_role3", "test_user3"]) def test_project_create_app_users__names_and_forms(self): """Should create project app users, and assign forms to them.""" - client = Client() - client.projects.create_app_users( + self.client.projects.create_app_users( display_names=["test_assign3", "test_assign_23"], - forms=["range", "pull_data"], + forms=["range_draft", "pull_data"], ) def test_submission_create__non_ascii(self): """Should create an instance of the form, encoded to utf-8.""" - client = Client() - xml = """ - - - ~!@#$%^&*()_+=-✅✅ - - Banana - - - """ - client.submissions.create(xml=xml, form_id="'=+/*-451%/%") - submission = client.submissions.get( - form_id="'=+/*-451%/%", instance_id="~!@#$%^&*()_+=-✅✅" + form_id = "'=+/*-451%/%" + iid = f"""scna+~!@#$%^&*()_+=-✅✅+{datetime.now().isoformat()}""" + + self.client.submissions.create( + xml=submissions_data.get_xml__fruits( + form_id=form_id, + version=get_latest_form_version(client=self.client, form_id=form_id), + instance_id=iid, + ), + form_id=form_id, ) - self.assertEqual("~!@#$%^&*()_+=-✅✅", submission.instanceId) + submission = self.client.submissions.get(form_id=form_id, instance_id=iid) + self.assertEqual(iid, submission.instanceId) def test_submission_edit__non_ascii(self): """Should edit an existing instance of the form, encoded to utf-8.""" - client = Client() # The "instance_id" remains the id of the first submission, not the # instanceID/deprecatedID used in the XML. - xml = """ - - - ~!@#$%^&*()_+=-✘✘ - ~!@#$%^&*()_+=-✘✘✘ - - Papaya - - - """ - client.submissions.edit( - xml=xml, form_id="'=+/*-451%/%", instance_id="~!@#$%^&*()_+=-✅✅" + form_id = "'=+/*-451%/%" + iid = """sena_~~!@#$%^&*()_+=-✅✅""" + + # So we have a submission to edit, create one or find the most recent prior edit. + old_iid = create_new_or_get_last_submission( + client=self.client, + form_id=form_id, + instance_id=iid, + ) + now = datetime.now().isoformat() + self.client.submissions.edit( + xml=submissions_data.get_xml__fruits( + form_id=form_id, + version=get_latest_form_version(client=self.client, form_id=form_id), + instance_id=iid + now, + deprecated_instance_id=old_iid, + ), + form_id=form_id, + instance_id=iid, + comment=f"pyODK edit {now}", ) diff --git a/tests/test_config.py b/tests/test_config.py index 5e4d31f..9b4ba53 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,8 @@ from pyodk._utils import config from pyodk.errors import PyODKError -from tests import resources, utils +from tests import resources +from tests.utils.utils import get_temp_dir class TestConfig(TestCase): @@ -51,7 +52,7 @@ def test_read_toml__error__non_existent(self): def test_write_cache__ok(self): """Should write the cache data when no path is specified.""" - with utils.get_temp_dir() as tmp: + with get_temp_dir() as tmp: path = tmp / "my_cache.toml" with patch.dict(os.environ, {"PYODK_CACHE_FILE": path.as_posix()}): self.assertFalse(path.exists()) @@ -60,7 +61,7 @@ def test_write_cache__ok(self): def test_write_cache__with_path(self): """Should write the cache data when a path is specified.""" - with utils.get_temp_dir() as tmp: + with get_temp_dir() as tmp: path = tmp / "my_cache.toml" self.assertFalse(path.exists()) config.write_cache(key="token", value="1234abcd", cache_path=path.as_posix()) diff --git a/tests/test_session.py b/tests/test_session.py index bafe41f..321e07f 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -29,7 +29,7 @@ def test_urlformat(self): # integer ({"project_id": 1, "form_id": 1}, "projects/1/forms/1"), # latin symbols - ({"project_id": 1, "form_id": "+-_*%*"}, "projects/1/forms/%2B-_%2A%25%2A"), + ({"project_id": 1, "form_id": "+-_*%*"}, "projects/1/forms/%2B-_*%25*"), # lower case e, with combining acute accent (2 symbols) ({"project_id": 1, "form_id": "tést"}, "projects/1/forms/te%CC%81st"), # lower case e with acute (1 symbol) @@ -49,7 +49,9 @@ def test_urlquote(self): # integer ("1.xls", "1"), # latin symbols - ("+-_*%*.xls", "%2B-_%2A%25%2A"), + ("+-_*%*.xls", "%2B-_*%25*"), + # spaces + ("my form.xlsx", "my%20form"), # lower case e, with combining acute accent (2 symbols) ("tést.xlsx", "te%CC%81st"), # lower case e with acute (1 symbol) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/forms.py b/tests/utils/forms.py new file mode 100644 index 0000000..3bdb6e2 --- /dev/null +++ b/tests/utils/forms.py @@ -0,0 +1,87 @@ +from pathlib import Path +from typing import IO + +from pyodk._endpoints.form_drafts import FormDraftService +from pyodk.client import Client +from pyodk.errors import PyODKError + +from tests.utils import utils +from tests.utils.md_table import md_table_to_temp_dir + + +def create_new_form(client: Client, file_path: Path | str, form_id: str, form_data: IO): + """ + Create a new form. Ignores any pyODK errors. + + :param client: Client instance to use for API calls. + :param file_path: Path of the form definition. + :param form_id: The xmlFormId of the Form being referenced. + :param form_data: The form file descriptor which can be read() from. + :return: + """ + try: + fd = FormDraftService( + session=client.session, default_project_id=client.project_id + ) + pid, fid, headers, params = fd._prep_form_post( + file_path=file_path, form_id=form_id + ) + params["publish"] = True + client.post( + url=client.session.urlformat("projects/{pid}/forms", pid=client.project_id), + headers=headers, + params=params, + data=form_data, + ) + except PyODKError: + pass + + +def create_new_form__md(client: Client, form_id: str, form_def: str): + """ + Create a new form from a MarkDown string. + + :param client: Client instance to use for API calls. + :param form_id: The xmlFormId of the Form being referenced. + :param form_def: The form definition MarkDown. + """ + with ( + md_table_to_temp_dir(form_id=form_id, mdstr=form_def) as fp, + open(fp, "rb") as form_data, + ): + create_new_form(client=client, file_path=fp, form_id=form_id, form_data=form_data) + + +def create_new_form__xml(client: Client, form_id: str, form_def: str): + """ + Create a new form from a XML string. + + :param client: Client instance to use for API calls. + :param form_id: The xmlFormId of the Form being referenced. + :param form_def: The form definition XML. + """ + with utils.get_temp_file(suffix=".xml") as fp: + fp.write_text(form_def) + with open(fp, "rb") as form_data: + create_new_form( + client=client, file_path=fp, form_id=form_id, form_data=form_data + ) + + +def get_latest_form_version(client: Client, form_id: str) -> str: + """ + Get the version name of the most recently published version of the form. + + :param client: Client instance to use for API calls. + :param form_id: The xmlFormId of the Form being referenced. + """ + versions = client.session.get( + client.session.urlformat( + "projects/{pid}/forms/{fid}/versions", + pid=client.project_id, + fid=form_id, + ) + ) + return sorted( + (s for s in versions.json()), key=lambda s: s["publishedAt"], reverse=True + )[0]["version"] diff --git a/tests/utils/md_table.py b/tests/utils/md_table.py new file mode 100644 index 0000000..17a8467 --- /dev/null +++ b/tests/utils/md_table.py @@ -0,0 +1,90 @@ +""" +Markdown table utility functions. +""" + +import re +from contextlib import contextmanager +from pathlib import Path + +from openpyxl import Workbook + +from tests.utils.utils import get_temp_dir + + +def _strp_cell(cell): + val = cell.strip() + if val == "": + return None + val = val.replace(r"\|", "|") + return val + + +def _extract_array(mdtablerow): + match = re.match(r"\s*\|(.*)\|\s*", mdtablerow) + if match: + mtchstr = match.groups()[0] + if re.match(r"^[\|-]+$", mtchstr): + return False + else: + return [_strp_cell(c) for c in re.split(r"(? list[tuple[str, list[list[str]]]]: + ss_arr = [] + for item in mdstr.split("\n"): + arr = _extract_array(item) + if arr: + ss_arr.append(arr) + sheet_name = False + sheet_arr = False + sheets = [] + for row in ss_arr: + if row[0] is not None: + if sheet_arr: + sheets.append((sheet_name, sheet_arr)) + sheet_arr = [] + sheet_name = row[0] + excluding_first_col = row[1:] + if sheet_name and not _is_null_row(excluding_first_col): + sheet_arr.append(excluding_first_col) + sheets.append((sheet_name, sheet_arr)) + + return sheets + + +def md_table_to_workbook(mdstr: str) -> Workbook: + """ + Convert Markdown table string to an openpyxl.Workbook. Call wb.save() to persist. + """ + md_data = md_table_to_ss_structure(mdstr=mdstr) + wb = Workbook(write_only=True) + for key, rows in md_data: + sheet = wb.create_sheet(title=key) + for r in rows: + sheet.append(r) + return wb + + +@contextmanager +def md_table_to_temp_dir(form_id: str, mdstr: str) -> Path: + """ + Convert MarkDown table string to a XLSX file saved in a temp directory. + + :param form_id: The xmlFormId of the Form being referenced. + :param mdstr: The MarkDown table string. + :return: The path of the XLSX file. + """ + with get_temp_dir() as td: + fp = Path(td) / f"{form_id}.xlsx" + md_table_to_workbook(mdstr).save(fp.as_posix()) + yield fp diff --git a/tests/utils/submissions.py b/tests/utils/submissions.py new file mode 100644 index 0000000..371505f --- /dev/null +++ b/tests/utils/submissions.py @@ -0,0 +1,70 @@ +from uuid import uuid4 + +from pyodk.client import Client +from pyodk.errors import PyODKError + +from tests.resources import submissions_data +from tests.utils.forms import get_latest_form_version + + +def create_new_or_get_last_submission( + client: Client, form_id: str, instance_id: str +) -> str: + """ + Create a new submission, or get the most recent version, and return it's instance_id. + + :param client: Client instance to use for API calls. + :param form_id: The xmlFormId of the Form being referenced. + :param instance_id: The instanceId of the Submission being referenced. + :return: The created instance_id or the instance_id of the most recent version. + """ + try: + old_iid = client.submissions.create( + xml=submissions_data.get_xml__fruits( + form_id=form_id, + version=get_latest_form_version(client=client, form_id=form_id), + instance_id=instance_id, + ), + form_id=form_id, + ).instanceId + except PyODKError: + subvs = client.session.get( + client.session.urlformat( + "projects/{pid}/forms/{fid}/submissions/{iid}/versions", + pid=client.project_id, + fid=form_id, + iid=instance_id, + ), + ) + old_iid = sorted( + (s for s in subvs.json()), key=lambda s: s["createdAt"], reverse=True + )[0]["instanceId"] + return old_iid + + +def create_or_update_submission_with_comment( + client: Client, form_id: str, instance_id: str +): + """ + Create and/or update a submission, adding a comment with the edit. + + :param client: Client instance to use for API calls. + :param form_id: The xmlFormId of the Form being referenced. + :param instance_id: The instanceId of the Submission being referenced. + """ + pd_iid = create_new_or_get_last_submission( + client=client, + form_id=form_id, + instance_id=instance_id, + ) + client.submissions.edit( + xml=submissions_data.get_xml__fruits( + form_id=form_id, + version=get_latest_form_version(client=client, form_id=form_id), + instance_id=uuid4().hex, + deprecated_instance_id=pd_iid, + ), + form_id=form_id, + instance_id=instance_id, + comment="pyODK edit", + ) diff --git a/tests/utils.py b/tests/utils/utils.py similarity index 65% rename from tests/utils.py rename to tests/utils/utils.py index 82ca65c..aee6891 100644 --- a/tests/utils.py +++ b/tests/utils/utils.py @@ -5,8 +5,14 @@ @contextmanager -def get_temp_file() -> Path: - temp_file = tempfile.NamedTemporaryFile(delete=False) +def get_temp_file(**kwargs) -> Path: + """ + Create a temporary file. + + :param kwargs: File handling options passed through to NamedTemporaryFile. + :return: The path of the temporary file. + """ + temp_file = tempfile.NamedTemporaryFile(delete=False, **kwargs) temp_file.close() temp_path = Path(temp_file.name) try: