Dependencies: Update to lorrystream 0.0.4 and commons-codec 0.0.6. Adjust code. Add software tests.

.github/workflows/main.yml

@@ -155,3 +155,68 @@ jobs:
         env_vars: OS,PYTHON
         name: codecov-umbrella
         fail_ci_if_error: true
+
+
+  tests-kinesis:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.9", "3.12"]
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
+      # Do not tear down Testcontainers
+      TC_KEEPALIVE: true
+
+    # https://docs.github.com/en/actions/using-containerized-services/about-service-containers
+    services:
+      cratedb:
+        image: crate/crate:nightly
+        ports:
+          - 4200:4200
+          - 5432:5432
+
+    name: "
+    Kinesis:
+    Python ${{ matrix.python-version }} on OS ${{ matrix.os }}"
+    steps:
+
+    - name: Acquire sources
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+        cache: 'pip'
+        cache-dependency-path: 'pyproject.toml'
+
+    - name: Set up project
+      run: |
+
+        # `setuptools 0.64.0` adds support for editable install hooks (PEP 660).
+        # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400
+        pip install "setuptools>=64" --upgrade
+
+        # Install package in editable mode.
+        pip install --use-pep517 --prefer-binary --editable=.[kinesis,test,develop]
+
+    - name: Run linter and software tests
+      run: |
+        pytest -m kinesis
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      with:
+        files: ./coverage.xml
+        flags: kinesis
+        env_vars: OS,PYTHON
+        name: codecov-umbrella
+        fail_ci_if_error: true

CHANGES.md

@@ -15,6 +15,8 @@
 - MongoDB: Make `migr8 extract` and `migr8 export` accept the `--limit` option
 - MongoDB: Fix indentation in prettified SQL output of `migr8 translate`
 - MongoDB: Add capability to give type hints and add transformations
+- Dependencies: Adjust code for lorrystream version 0.0.3
+- Dependencies: Update to lorrystream 0.0.4 and commons-codec 0.0.6
 
 ## 2024/07/25 v0.0.16
 - `ctk load table`: Added support for MongoDB Change Streams

cratedb_toolkit/iac/aws.py

@@ -1,9 +1,10 @@
+from lorrystream.carabas.aws import DynamoDBKinesisPipe, RDSPostgreSQLDMSKinesisPipe
 from lorrystream.carabas.aws.function.model import LambdaFactory
 from lorrystream.carabas.aws.function.oci import LambdaPythonImage
-from lorrystream.carabas.aws.stack import DynamoDBKinesisPipe
 
 __all__ = [
+    "DynamoDBKinesisPipe",
     "LambdaFactory",
     "LambdaPythonImage",
-    "DynamoDBKinesisPipe",
+    "RDSPostgreSQLDMSKinesisPipe",
 ]

cratedb_toolkit/io/processor/kinesis_lambda.py

@@ -25,7 +25,7 @@
 # /// script
 # requires-python = ">=3.9"
 # dependencies = [
-#   "commons-codec==0.0.3",
+#   "commons-codec==0.0.6",
 #   "sqlalchemy-cratedb==0.38.0",
 # ]
 # ///

pyproject.toml

@@ -151,10 +151,10 @@ io = [
   "sqlalchemy>=2",
 ]
 kinesis = [
-  "lorrystream @ git+https://github.com/daq-tools/lorrystream.git@kinesis",
+  "lorrystream[carabas]==0.0.4",
 ]
 mongodb = [
-  "commons-codec[mongodb,zyp]==0.0.4",
+  "commons-codec[mongodb,zyp]==0.0.6",
   "cratedb-toolkit[io]",
   "orjson<4,>=3.3.1",
   "pymongo<5,>=3.10.1",
@@ -255,6 +255,7 @@ xfail_strict = true
 markers = [
   "examples",
   "influxdb",
+  "kinesis",
   "mongodb",
   "pymongo",
   "slow",

release/oci/Dockerfile

@@ -6,8 +6,8 @@
 
 FROM python:3.11-slim-bookworm
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV TERM linux
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TERM=linux
 
 # Install Git, it is needed for `versioningit`.
 RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache

tests/io/test_iac.py

@@ -0,0 +1,15 @@
+# ruff: noqa: F401
+import pytest
+
+pytestmark = pytest.mark.kinesis
+
+pytest.importorskip("lorrystream", reason="Only works with LorryStream installed")
+
+
+def test_iac_imports():
+    from cratedb_toolkit.iac.aws import (
+        DynamoDBKinesisPipe,
+        LambdaFactory,
+        LambdaPythonImage,
+        RDSPostgreSQLDMSKinesisPipe,
+    )

tests/io/test_processor.py

@@ -0,0 +1,35 @@
+import os
+import sys
+
+import pytest
+
+pytestmark = pytest.mark.kinesis
+
+pytest.importorskip("commons_codec", reason="Only works with commons-codec installed")
+
+
+@pytest.fixture
+def reset_handler():
+    try:
+        del sys.modules["cratedb_toolkit.io.processor.kinesis_lambda"]
+    except KeyError:
+        pass
+
+
+def test_processor_invoke_no_records(reset_handler, mocker, caplog):
+    """
+    Roughly verify that the unified Lambda handler works.
+    """
+
+    # Configure environment variables.
+    handler_environment = {
+        "MESSAGE_FORMAT": "dms",
+    }
+    mocker.patch.dict(os.environ, handler_environment)
+
+    from cratedb_toolkit.io.processor.kinesis_lambda import handler
+
+    event = {"Records": []}
+    handler(event, None)
+
+    assert "Successfully processed 0 records" in caplog.messages