Parallelize integration tests

.github/workflows/ci.yaml

@@ -38,29 +38,86 @@ jobs:
     name: Build charms
     uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@main
 
+  collect-tests:
+    name: Collect integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install tox
+        # TODO: Consider replacing with custom image on self-hosted runner OR pinning version
+        run: python3 -m pip install tox
+      - name: Select tests
+        id: select-tests
+        run: |
+          if [ "${{ github.event_name }}" == "schedule" ]
+          then
+            echo Running flaky and non-flaky tests
+            echo "mark_expression=" >> $GITHUB_OUTPUT
+          else
+            echo Skipping flaky tests
+            echo "mark_expression=not flaky" >> $GITHUB_OUTPUT
+          fi
+      - name: Collect tests
+        id: collect-tests
+        shell: python
+        run: |
+          import json
+          import os
+          import subprocess
+
+          output = subprocess.check_output(
+              [
+                  "tox",
+                  "-qq",
+                  "run",
+                  "-e",
+                  "integration",
+                  "--",
+                  "pytest",
+                  "--quiet",
+                  "--color=no",
+                  "--collect-only",
+                  "tests/integration/",
+                  "-m", 
+                  "${{ steps.select-tests.outputs.mark_expression }}",
+              ],
+              encoding="utf-8",
+          )
+          """
+          Example output (every line before the first empty line is a test):
+          tests/integration/test_tls.py::test_connection_before_tls
+          tests/integration/high_availability/test_replication.py::test_consistent_data_replication_across_cluster
+          tests/integration/high_availability/test_replication.py::test_kill_primary_check_reelection
+
+          3 tests collected in 0.02s
+          """
+          output = output.split("\n")
+          tests = output[: output.index("")]
+          with open(os.environ["GITHUB_OUTPUT"], "a") as file:
+              file.write(f"tests={json.dumps(tests)}")
+    outputs:
+      tests: ${{ steps.collect-tests.outputs.tests }}
+
+
+
   integration-test:
     strategy:
       fail-fast: false
       matrix:
-        tox-environments:
-          - integration-ha
-          - integration-db-router
-          - integration-shared-db
-          - integration-database
-          - integration-mysql-interface
-          - integration-healing
-          - integration-tls
+        tests: ${{ fromJSON(needs.collect-tests.outputs.tests) }}
         ubuntu-versions:
           # Update whenever charmcraft.yaml is changed
           - series: focal
             bases-index: 0
           - series: jammy
             bases-index: 1
-    name: ${{ matrix.tox-environments }} | ${{ matrix.ubuntu-versions.series }}
+    name: ${{ matrix.tests }} | ${{ matrix.ubuntu-versions.series }}
     needs:
       - lint
       - unit-test
       - build
+      - collect-tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -86,7 +143,7 @@ jobs:
             echo Skipping flaky tests
             echo "mark_expression=not flaky" >> $GITHUB_OUTPUT
           fi
-      - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --mysql-charm-series=${{ matrix.ubuntu-versions.series }} --mysql-charm-bases-index=${{ matrix.ubuntu-versions.bases-index }}
+      - name: Run integration test
+        run: tox run -e integration -- pytest -v --tb native --log-cli-level=INFO -s ${{ matrix.tests }} --mysql-charm-series=${{ matrix.ubuntu-versions.series }} --mysql-charm-bases-index=${{ matrix.ubuntu-versions.bases-index }}
         env:
           CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}

CONTRIBUTING.md

@@ -34,7 +34,7 @@ source venv/bin/activate
 tox run -e format        # update your code according to linting rules
 tox run -e lint          # code style
 tox run -e unit          # unit tests
-tox run -m integration   # integration tests
+tox run -e integration   # integration tests
 tox                      # runs 'lint' and 'unit' environments
 ```
 

pyproject.toml

@@ -12,10 +12,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = [
-    "dev: mark to test in development tests",
-    "flaky"
-]
+markers = ["flaky"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/high_availability/test_replication.py

@@ -11,7 +11,6 @@
 from pytest_operator.plugin import OpsTest
 
 from ..helpers import (
-    app_name,
     cluster_name,
     execute_queries_on_unit,
     generate_random_string,
@@ -107,10 +106,10 @@ async def test_kill_primary_check_reelection(ops_test: OpsTest, mysql_charm_seri
 
 
 @pytest.mark.abort_on_fail
-async def test_scaling_without_data_loss(ops_test: OpsTest) -> None:
+async def test_scaling_without_data_loss(ops_test: OpsTest, mysql_charm_series: str) -> None:
     """Test that data is preserved during scale up and scale down."""
     # Insert values into test table from the primary unit
-    app = await app_name(ops_test)
+    app, _ = await high_availability_test_setup(ops_test, mysql_charm_series)
     application = ops_test.model.applications[app]
 
     random_unit = application.units[0]
@@ -190,7 +189,7 @@ async def test_cluster_isolation(ops_test: OpsTest, mysql_charm_series: str) ->
     the application name for each cluster, retrieve and compare these records, asserting they are
     not the same.
     """
-    app = await app_name(ops_test)
+    app, _ = await high_availability_test_setup(ops_test, mysql_charm_series)
     apps = [app, ANOTHER_APP_NAME]
 
     # Build and deploy secondary charm

tests/integration/test_tls.py

@@ -11,13 +11,11 @@
 from constants import CLUSTER_ADMIN_USERNAME, TLS_SSL_CERT_FILE
 
 from .helpers import (
-    app_name,
     get_process_pid,
     get_system_user_password,
     get_tls_ca,
     get_unit_ip,
     is_connection_possible,
-    scale_application,
     unit_file_md5,
 )
 
@@ -29,17 +27,8 @@
 TLS_APP_NAME = "tls-certificates-operator"
 
 
-@pytest.mark.abort_on_fail
-async def test_build_and_deploy(ops_test: OpsTest, mysql_charm_series: str) -> None:
-    """Build the charm and deploy 3 units to ensure a cluster is formed."""
-    if app := await app_name(ops_test):
-        if len(ops_test.model.applications[app].units) == 3:
-            return
-        else:
-            async with ops_test.fast_forward():
-                await scale_application(ops_test, app, 3)
-            return
-
+@pytest.fixture
+async def deploy(ops_test: OpsTest, mysql_charm_series: str) -> str:
     charm = await ops_test.build_charm(".")
 
     await ops_test.model.deploy(
@@ -61,11 +50,19 @@ async def test_build_and_deploy(ops_test: OpsTest, mysql_charm_series: str) -> N
             timeout=15 * 60,
         )
 
+    return APP_NAME
+
+
+@pytest.mark.abort_on_fail
+async def test_build_and_deploy(deploy: str) -> None:
+    """Build the charm and deploy 3 units to ensure a cluster is formed."""
+    pass
+
 
 @pytest.mark.abort_on_fail
-async def test_connection_before_tls(ops_test: OpsTest) -> None:
+async def test_connection_before_tls(ops_test: OpsTest, deploy: str) -> None:
     """Ensure connections (with and without ssl) are possible before relating with TLS operator."""
-    app = await app_name(ops_test)
+    app = deploy
     all_units = ops_test.model.applications[app].units
 
     # set global config dict once
@@ -91,9 +88,9 @@ async def test_connection_before_tls(ops_test: OpsTest) -> None:
 
 
 @pytest.mark.abort_on_fail
-async def test_enable_tls(ops_test: OpsTest) -> None:
+async def test_enable_tls(ops_test: OpsTest, deploy: str) -> None:
     """Test for encryption enablement when relation to TLS charm."""
-    app = await app_name(ops_test)
+    app = deploy
     all_units = ops_test.model.applications[app].units
 
     # Deploy TLS Certificates operator.
@@ -133,12 +130,12 @@ async def test_enable_tls(ops_test: OpsTest) -> None:
 
 
 @pytest.mark.abort_on_fail
-async def test_rotate_tls_key(ops_test: OpsTest) -> None:
+async def test_rotate_tls_key(ops_test: OpsTest, deploy: str) -> None:
     """Verify rotating tls private keys restarts cluster with new certificates.
 
     This test rotates tls private keys to randomly generated keys.
     """
-    app = await app_name(ops_test)
+    app = deploy
     all_units = ops_test.model.applications[app].units
     # dict of values for cert file md5sum and mysql service PID. After resetting the
     # private keys these certificates should be updated and the mysql service should be
@@ -190,9 +187,9 @@ async def test_rotate_tls_key(ops_test: OpsTest) -> None:
 
 
 @pytest.mark.abort_on_fail
-async def test_disable_tls(ops_test: OpsTest) -> None:
+async def test_disable_tls(ops_test: OpsTest, deploy: str) -> None:
     # Remove the relation
-    app = await app_name(ops_test)
+    app = deploy
     all_units = ops_test.model.applications[app].units
 
     logger.info("Removing relation")

tox.ini

@@ -5,10 +5,6 @@
 no_package = True
 skip_missing_interpreters = True
 env_list = lint, unit
-labels =
-    integration = integration-ha, integration-db-router, integration-shared-db, \
-        integration-database, integration-mysql-interface, integration-healing, \
-        integration-tls
 
 [vars]
 src_path = {tox_root}/src
@@ -80,55 +76,7 @@ pass_env = GITHUB_OUTPUT
 commands =
     python -c "from tests.integration.read_charm_yaml import create_build_matrix; create_build_matrix()"
 
-[testenv:integration-ha]
-description = Run HA integration tests
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/high_availability/test_replication.py
-
-[testenv:integration-db-router]
-description = Run legacy db-router relation integration tests
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/relations/test_db_router.py
-
-[testenv:integration-shared-db]
-description = Run legacy shared-db relation integration tests
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/relations/test_shared_db.py
-
-[testenv:integration-database]
+[testenv:integration]
 description = Run integration tests
 pass_env =
     {[testenv]pass_env}
@@ -142,71 +90,4 @@ deps =
     pyyaml
     -r {tox_root}/requirements.txt
 commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/relations/test_database.py
-
-[testenv:integration-mysql-interface]
-description = Run integration tests for legacy mysql interface
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/relations/test_relation_mysql_legacy.py
-
-[testenv:integration-healing]
-description = Run self-healing tests
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/high_availability/test_self_healing.py
-
-[testenv:integration-tls]
-description = Run TLS tests
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_tls.py
-
-# Used to mark tests run locally (in order to target specific tests to run)
-[testenv:dev]
-description = Run in development test
-pass_env =
-    {[testenv]pass_env}
-    CI
-    CI_PACKED_CHARMS
-deps =
-    juju==2.9.38.1
-    mysql-connector-python
-    pytest
-    pdbpp
-    pytest-operator
-    pyyaml
-    -r {tox_root}/requirements.txt
-commands =
-    pytest -v --tb native --ignore={[vars]tests_path}/unit --log-cli-level=INFO -s {posargs} -m "dev"
-
+    {posargs}