MongoDB/PyMongo: Add software tests and CI configuration

It needs to balance SQLAlchemy 1.x vs. 2.x throughout the toolkit test
cases, because JessiQL still uses SQLAlchemy 1.x.

.github/workflows/influxdb.yml

@@ -66,7 +66,7 @@ jobs:
         cache: 'pip'
         cache-dependency-path: 'pyproject.toml'
 
-    - name: Setup project
+    - name: Set up project
       run: |
 
         # `setuptools 0.64.0` adds support for editable install hooks (PEP 660).

.github/workflows/main.yml

@@ -20,7 +20,8 @@ concurrency:
 
 jobs:
 
-  tests:
+
+  tests-main:
 
     runs-on: ${{ matrix.os }}
     strategy:
@@ -43,7 +44,10 @@ jobs:
           - 4200:4200
           - 5432:5432
 
-    name: Python ${{ matrix.python-version }} on OS ${{ matrix.os }}
+    name: "
+    Common:
+    Python ${{ matrix.python-version }} on OS ${{ matrix.os }}
+    "
     steps:
 
     - name: Acquire sources
@@ -57,7 +61,7 @@ jobs:
         cache: 'pip'
         cache-dependency-path: 'pyproject.toml'
 
-    - name: Setup project
+    - name: Set up project
       run: |
 
         # `setuptools 0.64.0` adds support for editable install hooks (PEP 660).
@@ -79,3 +83,66 @@ jobs:
         env_vars: OS,PYTHON
         name: codecov-umbrella
         fail_ci_if_error: false
+
+
+  tests-pymongo:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.9", "3.12"]
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
+      # Do not tear down Testcontainers
+      TC_KEEPALIVE: true
+
+    # https://docs.github.com/en/actions/using-containerized-services/about-service-containers
+    services:
+      cratedb:
+        image: crate/crate:nightly
+        ports:
+          - 4200:4200
+          - 5432:5432
+
+    name: "
+    PyMongo: 
+    Python ${{ matrix.python-version }} on OS ${{ matrix.os }}"
+    steps:
+
+    - name: Acquire sources
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+        cache: 'pip'
+        cache-dependency-path: 'pyproject.toml'
+
+    - name: Set up project
+      run: |
+
+        # `setuptools 0.64.0` adds support for editable install hooks (PEP 660).
+        # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400
+        pip install "setuptools>=64" --upgrade
+
+        # Install package in editable mode.
+        pip install --use-pep517 --prefer-binary --editable=.[pymongo,test,develop]
+
+    - name: Run linter and software tests
+      run: |
+        poe check
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        files: ./coverage.xml
+        flags: pymongo
+        env_vars: OS,PYTHON
+        name: codecov-umbrella
+        fail_ci_if_error: false

.github/workflows/mongodb.yml

@@ -60,7 +60,7 @@ jobs:
         cache: 'pip'
         cache-dependency-path: 'pyproject.toml'
 
-    - name: Setup project
+    - name: Set up project
       run: |
 
         # `setuptools 0.64.0` adds support for editable install hooks (PEP 660).

.github/workflows/oci.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Acquire sources
         uses: actions/checkout@v4
 
-      - name: Setup Python
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.11"

cratedb_toolkit/adapter/pymongo/backlog.md

@@ -6,6 +6,7 @@
 - Add documentation.
 - Add missing essential querying features: Examples: sort order, skip, limit
 - Add missing essential methods. Example: `db.my_collection.drop()`.
+- Make write-synchronization behavior (refresh table) configurable.
 
 ## Iteration +2
 

cratedb_toolkit/retention/store.py

@@ -9,7 +9,11 @@
 from sqlalchemy import MetaData, Table
 from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.orm import Session
-from sqlalchemy.sql.selectable import NamedFromClause
+
+try:
+    from sqlalchemy.sql.selectable import NamedFromClause  # type: ignore[attr-defined]
+except ImportError as ex:
+    raise NotImplementedError("This module only works on SQLAlchemy 2.x") from ex
 
 from cratedb_toolkit.retention.model import JobSettings, RetentionPolicy, RetentionStrategy
 from cratedb_toolkit.util.database import DatabaseAdapter, sa_is_empty
@@ -41,7 +45,7 @@ def get_tags_constraints(self, tags: t.Union[t.List[str], t.Set[str]]):
         for tag in tags:
             if not tag:
                 continue
-            constraint = table.c[self.tag_column][tag] != sa.Null()
+            constraint = table.c[self.tag_column][tag] != sa.Null()  # type: ignore[attr-defined]
             constraints.append(constraint)
         return sa.and_(sa.true(), *constraints)
 

cratedb_toolkit/sqlalchemy/patch.py

@@ -15,7 +15,7 @@ def patch_inspector():
     FIXME: Bug in CrateDB SQLAlchemy dialect?
     """
 
-    def get_effective_schema(engine: sa.Engine):
+    def get_effective_schema(engine: sa.engine.Engine):
         schema_name_raw = engine.url.query.get("schema")
         schema_name = None
         if isinstance(schema_name_raw, str):
@@ -28,7 +28,9 @@ def get_effective_schema(engine: sa.Engine):
 
     get_table_names_dist = CrateDialect.get_table_names
 
-    def get_table_names(self, connection: sa.Connection, schema: t.Optional[str] = None, **kw: t.Any) -> t.List[str]:
+    def get_table_names(
+        self, connection: sa.engine.Connection, schema: t.Optional[str] = None, **kw: t.Any
+    ) -> t.List[str]:
         if schema is None:
             schema = get_effective_schema(connection.engine)
         return get_table_names_dist(self, connection=connection, schema=schema, **kw)

examples/pymongo_adapter.py

@@ -24,7 +24,9 @@
 ==========
 - https://github.com/mongodb/mongo-python-driver
 """
+import datetime as dt
 import logging
+import time
 
 import pymongo
 from pymongo.database import Database
@@ -35,7 +37,7 @@
 logger = logging.getLogger(__name__)
 
 
-def main():
+def mongodb_workload():
     client = pymongo.MongoClient(
         "localhost", 27017, timeoutMS=100, connectTimeoutMS=100, socketTimeoutMS=100, serverSelectionTimeoutMS=100
     )
@@ -46,45 +48,70 @@ def main():
     logger.info(f"Using database: {db.name}")
     logger.info(f"Using collection: {db.my_collection}")
 
-    # Insert records.
-    inserted_id = db.my_collection.insert_one({"x": 5}).inserted_id
-    logger.info(f"Inserted object: {inserted_id!r}")
+    # TODO: Dropping a collection is not implemented yet.
+    # db.my_collection.drop()  # noqa: ERA001
+
+    # Insert document.
+    documents = [
+        {
+            "author": "Mike",
+            "text": "My first blog post!",
+            "tags": ["mongodb", "python", "pymongo"],
+            "date": dt.datetime.now(tz=dt.timezone.utc),
+        },
+        {
+            "author": "Eliot",
+            "title": "MongoDB is fun",
+            "text": "and pretty easy too!",
+            "date": dt.datetime(2009, 11, 10, 10, 45),
+        },
+    ]
+    result = db.my_collection.insert_many(documents)
+    logger.info(f"Inserted document identifiers: {result.inserted_ids!r}")
 
     # FIXME: Refresh table.
+    time.sleep(1)
 
-    # Query records.
+    # Query documents.
     document_count = db.my_collection.count_documents({})
     logger.info(f"Total document count: {document_count}")
 
     # Find single document.
-    document = db.my_collection.find_one()
+    document = db.my_collection.find_one({"author": "Mike"})
     logger.info(f"[find_one] Response document: {document}")
 
-    # Assorted basic find operations, with sorting and paging.
-    print("results:", end=" ")
+    # Run a few basic retrieval operations, with sorting and paging.
+    print("Whole collection")
     for item in db.my_collection.find():
-        print(item["x"], end=", ")
+        print(item)
     print()
 
-    print("results:", end=" ")
-    for item in db.my_collection.find().sort("x", pymongo.ASCENDING):
-        print(item["x"], end=", ")
+    print("Sort ascending")
+    for item in db.my_collection.find().sort("author", pymongo.ASCENDING):
+        print(item)
     print()
 
-    print("results:", end=" ")
-    for item in db.my_collection.find().sort("x", pymongo.DESCENDING):
-        print(item["x"], end=", ")
+    print("Sort descending")
+    for item in db.my_collection.find().sort("author", pymongo.DESCENDING):
+        print(item)
     print()
 
-    results = [item["x"] for item in db.my_collection.find().limit(2).skip(1)]
-    print("results:", results)
-    print("length:", len(results))
+    print("Paging")
+    for item in db.my_collection.find().limit(2).skip(1):
+        print(item)
+    print()
 
 
-if __name__ == "__main__":
+def main(dburi: str = None):
+    dburi = dburi or "crate://crate@localhost:4200"
+
     # setup_logging(level=logging.DEBUG, width=42)  # noqa: ERA001
     setup_logging(level=logging.INFO, width=20)
 
     # Context manager use.
-    with PyMongoCrateDbAdapter(dburi="crate://crate@localhost:4200"):
-        main()
+    with PyMongoCrateDbAdapter(dburi=dburi):
+        mongodb_workload()
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -93,8 +93,8 @@ dependencies = [
   "croud",
   'importlib-metadata; python_version <= "3.7"',
   "python-dotenv<2",
-  "sqlalchemy",
   "sqlparse<0.5",
+  "verlib2==0.2",
 ]
 [project.optional-dependencies]
 all = [
@@ -115,6 +115,7 @@ io = [
   "cr8",
   "dask<=2023.10.1,>=2020",
   "pandas<3,>=1",
+  "sqlalchemy>=2",
 ]
 mongodb = [
   "cr8",
@@ -125,8 +126,9 @@ mongodb = [
 ]
 pymongo = [
   "jessiql==1.0.0rc1",
+  "pandas<3,>=2",
   "pymongo<5,>=3.10.1",
-  "sqlalchemy<2.0",
+  "sqlalchemy<2",
 ]
 release = [
   "build<2",

tests/adapter/test_pymongo.py

@@ -1,10 +1,15 @@
+# ruff: noqa: E402
 import datetime as dt
 import typing as t
 from unittest import mock
 
 import pymongo
 import pytest
 
+from tests.conftest import check_sqlalchemy1
+
+check_sqlalchemy1(allow_module_level=True)
+
 from cratedb_toolkit.adapter.pymongo import PyMongoCrateDbAdapter
 from cratedb_toolkit.adapter.pymongo.util import AmendedObjectId
 from cratedb_toolkit.util.date import truncate_milliseconds
@@ -208,6 +213,15 @@ def test_pymongo_roundtrip_document(
     assert document_loaded == document_original
 
 
+def test_example_program(cratedb: CrateDBFixture):
+    """
+    Verify that the program `examples/pymongo_adapter.py` works.
+    """
+    from examples.pymongo_adapter import main
+
+    main(dburi=cratedb.database.dburi)
+
+
 def test_pymongo_tutorial(
     pymongo_cratedb: PyMongoCrateDbAdapter, pymongo_client: pymongo.MongoClient, cratedb: CrateDBFixture, sync_writes
 ):

tests/conftest.py

@@ -2,6 +2,8 @@
 # Distributed under the terms of the AGPLv3 license, see LICENSE.
 import pytest
 import responses
+import sqlalchemy as sa
+from verlib2 import Version
 
 from cratedb_toolkit.testing.testcontainers.cratedb import CrateDBContainer
 from cratedb_toolkit.util import DatabaseAdapter
@@ -118,4 +120,40 @@ def cloud_cluster_mock():
     )
 
 
+IS_SQLALCHEMY1 = Version(sa.__version__) < Version("2")
+IS_SQLALCHEMY2 = Version(sa.__version__) >= Version("2")
+
+
+@pytest.fixture(scope="module")
+def needs_sqlalchemy1_module():
+    """
+    Use this for annotating pytest test case functions testing subsystems which need SQLAlchemy 1.x.
+    """
+    check_sqlalchemy1()
+
+
+def check_sqlalchemy1(**kwargs):
+    """
+    Skip pytest test cases or modules testing subsystems which need SQLAlchemy 1.x.
+    """
+    if not IS_SQLALCHEMY1:
+        raise pytest.skip("This feature or subsystem needs SQLAlchemy 1.x", **kwargs)
+
+
+@pytest.fixture
+def needs_sqlalchemy2():
+    """
+    Use this for annotating pytest test case functions testing subsystems which need SQLAlchemy 2.x.
+    """
+    check_sqlalchemy2()
+
+
+def check_sqlalchemy2(**kwargs):
+    """
+    Skip pytest test cases or modules testing subsystems which need SQLAlchemy 2.x.
+    """
+    if not IS_SQLALCHEMY2:
+        raise pytest.skip("This feature or subsystem needs SQLAlchemy 2.x", **kwargs)
+
+
 setup_logging()