Merge pull request #822 from Aiven-Open/matyaskuti/randomize_test_order

Randomize test run order

.github/workflows/tests.yml

@@ -37,7 +37,7 @@ jobs:
 
     - run: make install version
     - run: make unit-tests
-    - run: make integration-tests
+    - run: make integration-tests PYTEST_ARGS="--random-order"
 
     - name: Archive logs
       uses: actions/upload-artifact@v4

requirements/requirements-dev.in

@@ -9,6 +9,7 @@ filelock
 pytest
 pytest-xdist[psutil]
 pytest-timeout
+pytest-random-order
 psutil
 requests
 hypothesis

requirements/requirements-dev.txt

@@ -167,9 +167,12 @@ pyrepl==0.9.0
     # via fancycompleter
 pytest==7.4.4
     # via
-    #   -r requirements-dev.in
+    #   -r requirements/requirements-dev.in
+    #   pytest-random-order
     #   pytest-timeout
     #   pytest-xdist
+pytest-random-order==1.1.1
+    # via -r requirements/requirements-dev.in
 pytest-timeout==2.2.0
     # via -r requirements-dev.in
 pytest-xdist[psutil]==3.6.1

tests/integration/test_rest_consumer.py

@@ -118,7 +118,9 @@ async def test_subscription(rest_async_client, admin_client, producer, trail):
     data = res.json()
     assert "topics" in data and len(data["topics"]) == 0, f"expecting {data} to be empty"
     # one pattern sub will get all 3
-    prefix = f"{hash(random.random())}"
+    # use bool(trail) to make topic prefix distinct as there has been collision when
+    # test order is randomized.
+    prefix = f"test_subscription_{bool(trail)}{hash(random.random())}"
     pattern_topics = [new_topic(admin_client, prefix=f"{prefix}{i}") for i in range(3)]
     await wait_for_topics(rest_async_client, topic_names=pattern_topics, timeout=20, sleep=1)
     res = await rest_async_client.post(sub_path, json={"topic_pattern": f"{prefix}.*"}, headers=REST_HEADERS["json"])

tests/unit/test_schema_registry_api.py

@@ -12,12 +12,7 @@
 
 
 async def test_forward_when_not_ready():
-    with patch("karapace.schema_registry_apis.aiohttp.ClientSession") as client_session_class, patch(
-        "karapace.schema_registry_apis.KarapaceSchemaRegistry"
-    ) as schema_registry_class:
-        client_session = Mock()
-        client_session_class.return_value = client_session
-
+    with patch("karapace.schema_registry_apis.KarapaceSchemaRegistry") as schema_registry_class:
         schema_reader_mock = Mock()
         ready_property_mock = PropertyMock(return_value=False)
         schema_registry = Mock()
@@ -34,7 +29,6 @@ async def test_forward_when_not_ready():
         close_func = Mock()
         close_func.return_value = close_future_result
         schema_registry.close = close_func
-        client_session.close = close_func
 
         controller = KarapaceSchemaRegistryController(config=set_config_defaults(DEFAULTS))
         mock_forward_func_future = asyncio.Future()