add: document and mostly automate e2e tests, fix percent-encoding

- updated readme with section on testing and setting up for E2E
- automated create/update of forms and submissions for E2E tests
- copied over md_table functionality from pyxform to allow specifying
  test forms as markdown instead of accumulating XLSX files
- added openpyxl as a dev dependency to support conversion of md_table
  XLSForm data to XLSX format
- updated test_client.py / TestUsage.test_direct to use the new
  `$select` parameter, also showing how to select sub-properties
- fixed percent-encoding in `session.py` to match Central behaviour,
  see new docstring comments for details, update tests accordingly.

README.md

@@ -164,6 +164,18 @@ You can run tests with:
 python -m unittest
 ```
 
+### Testing
+
+When adding or updating pyODK functionality, at a minimum add or update corresponding unit tests. The unit tests are filed in `tests/endpoints` or `tests`. These tests focus on pyODK functionality, such as ensuring that data de/serialisation works as expected, and that method logic results in the expected call patterns. The unit tests use mocks and static data, which are stored in `tests/resources`. These data are obtained by making an API call and saving the Python dict returned by `response.json()` as text.
+
+For interactive testing, debugging, or sanity checking workflows, end-to-end tests are stored in `tests/test_client.py`. These tests are not run by default because they require access to a live Central server. The ODK team use the Central staging instance https://staging.getodk.cloud/ which is already configured for testing. Below are the steps to set up a new project in Central to be able to run these tests.
+
+1. Create a test project in Central.
+2. Create a test user in Central. It can be a site-wide Administrator. If it is not an Administrator, assign the user to the project with "Project Manager" privileges, so that forms and submissions in the test project can be uploaded and modified.
+3. Save the user's credentials and the project ID in a `.pyodk_config.toml` (or equivalent) as described in the above section titled "Configure".
+4. When the tests in `test_client.py` are run, the test setup method should automatically create a few forms and submissions for testing with. At a minimum these allow the tests to pass, but can also be used to interactively test or debug.
+
+
 ## Release
 
 1. Run all linting and tests.

pyodk/_endpoints/form_drafts.py

@@ -34,20 +34,21 @@ def __init__(
         self.default_project_id: int | None = default_project_id
         self.default_form_id: str | None = default_form_id
 
-    def create(
+    def _prep_form_post(
         self,
-        file_path: str | None = None,
+        file_path: Path | str | None = None,
         ignore_warnings: bool | None = True,
         form_id: str | None = None,
         project_id: int | None = None,
-    ) -> bool:
+    ) -> (str, str, dict, dict):
         """
-        Create a Form Draft.
+        Prepare / validate input arguments for POSTing a new form definition or version.
 
         :param file_path: The path to the file to upload.
         :param form_id: The xmlFormId of the Form being referenced.
         :param project_id: The id of the project this form belongs to.
         :param ignore_warnings: If True, create the form if there are XLSForm warnings.
+        :return: project_id, form_id, headers, params
         """
         try:
             pid = pv.validate_project_id(project_id, self.default_project_id)
@@ -81,6 +82,30 @@ def create(
             log.error(err, exc_info=True)
             raise
 
+        return pid, fid, headers, params
+
+    def create(
+        self,
+        file_path: Path | str | None = None,
+        ignore_warnings: bool | None = True,
+        form_id: str | None = None,
+        project_id: int | None = None,
+    ) -> bool:
+        """
+        Create a Form Draft.
+
+        :param file_path: The path to the file to upload.
+        :param form_id: The xmlFormId of the Form being referenced.
+        :param project_id: The id of the project this form belongs to.
+        :param ignore_warnings: If True, create the form if there are XLSForm warnings.
+        """
+        pid, fid, headers, params = self._prep_form_post(
+            file_path=file_path,
+            ignore_warnings=ignore_warnings,
+            form_id=form_id,
+            project_id=project_id,
+        )
+
         with open(file_path, "rb") if file_path is not None else nullcontext() as fd:
             response = self.session.response_or_error(
                 method="POST",

pyodk/_utils/session.py

@@ -1,7 +1,7 @@
 from logging import Logger
 from string import Formatter
 from typing import Any
-from urllib.parse import quote_plus, urljoin
+from urllib.parse import quote, urljoin
 
 from requests import PreparedRequest, Response
 from requests import Session as RequestsSession
@@ -16,11 +16,25 @@
 
 class URLFormatter(Formatter):
     """
-    Makes a valid URL by sending each format input field through urllib.parse.quote_plus.
+    Makes a valid URL by sending each format input field through urllib.parse.quote.
+
+    To parse/un-parse URLs, currently (v2023.5) Central uses JS default functions
+    encodeURIComponent and decodeURIComponent, which comply with RFC2396. The more recent
+    RFC3986 reserves hex characters 2A (asterisk), 27 (single quote), 28 (left
+    parenthesis), and 29 (right parenthesis). Python 3.7+ urllib.parse complies with
+    RFC3986 so in order for pyODK to behave as Central expects, these additional 4
+    characters are specified as "safe" in `format_field()` to not percent-encode them.
+
+    Currently (v2023.5) Central primarily supports the default submission instanceID
+    format per the XForm spec, namely "uuid:" followed by the 36 character UUID string.
+    In many endpoints, custom UUIDs (including non-ASCII/UTF-8 chars) will work, but in
+    some places they won't. For example the Central page for viewing submission details
+    fails on the Submissions OData call, because the OData function to filter by ID
+    (`Submission('instanceId')`) only works for the default instanceID format.
     """
 
     def format_field(self, value: Any, format_spec: str) -> Any:
-        return format(quote_plus(str(value)), format_spec)
+        return format(quote(str(value), safe="*'()"), format_spec)
 
 
 _URL_FORMATTER = URLFormatter()

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
 # Install with `pip install pyodk[dev]`.
 dev = [
     "ruff==0.3.4",        # Format and lint
+    "openpyxl==3.1.2"     # Create test XLSX files
 ]
 docs = [
     "mkdocs==1.5.3",

tests/endpoints/test_auth.py

@@ -7,8 +7,8 @@
 from pyodk.errors import PyODKError
 from requests import Session
 
-from tests import utils
 from tests.resources import CONFIG_DATA
+from tests.utils.utils import get_temp_dir
 
 
 @patch("pyodk._utils.config.read_config", MagicMock(return_value=CONFIG_DATA))
@@ -75,7 +75,7 @@ def test_get_token__ok__new_cache(self):
                 AuthService,
                 get_new_token=MagicMock(return_value="123"),
             ),
-            utils.get_temp_dir() as tmp,
+            get_temp_dir() as tmp,
         ):
             cache_path = (tmp / "test_cache.toml").as_posix()
             client = Client(cache_path=cache_path)
@@ -99,7 +99,7 @@ def test_get_token__error__new_cache_bad_response(self):
                 verify_token=verify_mock,
                 get_new_token=get_new_mock,
             ),
-            utils.get_temp_dir() as tmp,
+            get_temp_dir() as tmp,
             self.assertRaises(PyODKError) as err,
         ):
             cache_path = tmp / "test_cache.toml"
@@ -115,7 +115,7 @@ def test_get_token__ok__existing_cache(self):
                 AuthService,
                 verify_token=MagicMock(return_value="123"),
             ),
-            utils.get_temp_dir() as tmp,
+            get_temp_dir() as tmp,
         ):
             cache_path = (tmp / "test_cache.toml").as_posix()
             client = Client(cache_path=cache_path)
@@ -138,7 +138,7 @@ def test_get_token__error__existing_cache_bad_response(self):
                 verify_token=verify_mock,
                 get_new_token=MagicMock(return_value="123"),
             ),
-            utils.get_temp_dir() as tmp,
+            get_temp_dir() as tmp,
         ):
             cache_path = (tmp / "test_cache.toml").as_posix()
             client = Client(cache_path=cache_path)

tests/resources/forms/range_draft.xml

@@ -8,7 +8,7 @@
         <h:title>range_draft</h:title>
         <model odk:xforms-version="1.0.0">
             <instance>
-                <data id="range_draft" version="2023012001">
+                <data id="range_draft" version="{version}">
                     <start/>
                     <end/>
                     <Enter_a_number_within_a_specified_range/>

tests/resources/forms_data.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+from pathlib import Path
+
 test_forms = {
     "project_id": 8,
     "response_data": [
@@ -71,3 +74,44 @@
         },
     ],
 }
+
+
+def get_xml__range_draft(version: str | None = None) -> str:
+    if version is None:
+        version = datetime.now().isoformat()
+    with open(Path(__file__).parent / "forms" / "range_draft.xml") as fd:
+        return fd.read().format(version=version)
+
+
+def get_md__pull_data(version: str | None = None) -> str:
+    if version is None:
+        version = datetime.now().isoformat()
+    return f"""
+    | settings |
+    |          | version   |
+    |          | {version} |
+    | survey |           |            |           |             |
+    |        | type      | name       | label     | calculation |
+    |        | calculate | fruit      |           | pulldata('fruits', 'name', 'name_key', 'mango') |
+    |        | note      | note_fruit | The fruit ${{fruit}} pulled from csv |                      |
+    """
+
+
+md__symbols = """
+| settings |
+|          | form_title          | form_id       | version |
+|          | a non_ascii_form_id | ''=+/*-451%/% | 1       |
+| survey |           |            |           |             |
+|        | type      | name       | label     | calculation |
+|        | calculate | fruit      |           | pulldata('fruits', 'name', 'name_key', 'mango') |
+|        | note      | note_fruit | The fruit ${{fruit}} pulled from csv |                      |
+"""
+md__dingbat = """
+| settings |
+|          | form_title  | form_id | version |
+|          | ✅          | ✅     | 1       |
+| survey |           |            |           |             |
+|        | type      | name       | label     | calculation |
+|        | calculate | fruit      |           | pulldata('fruits', 'name', 'name_key', 'mango') |
+|        | note      | note_fruit | The fruit ${{fruit}} pulled from csv |                      |
+"""

tests/resources/submissions_data.py

@@ -45,3 +45,34 @@
   <age>36</age>
 </data>
 """
+
+
+def get_xml__fruits(
+    form_id: str,
+    version: str,
+    instance_id: str,
+    deprecated_instance_id: str | None = None,
+    selected_fruit: str = "Papaya",
+) -> str:
+    """
+    Get Submission XML for the "fruits" form that uses an external data list.
+
+    :param form_id: The xmlFormId of the Form being referenced.
+    :param version: The version of the form that the submission is for.
+    :param instance_id: The instanceId of the Submission being referenced.
+    :param deprecated_instance_id: If the submission is an edit, then the instance_id of
+      the submission being replaced must be provided.
+    :param selected_fruit: Which delicious tropical fruit do you like?
+    """
+    iidd = ""
+    if deprecated_instance_id is not None:
+        iidd = f"<deprecatedID>{deprecated_instance_id}</deprecatedID>"
+    return f"""
+    <data id="{form_id}" version="{version}">
+      <meta>{iidd}
+        <instanceID>{instance_id}</instanceID>
+      </meta>
+      <fruit>{selected_fruit}</fruit>
+      <note_fruit/>
+    </data>
+    """