chore: disable flakey tests in the default runs

.github/workflows/flakey-integration-tests.yaml

@@ -0,0 +1,20 @@
+name: Flakey integration tests
+
+on: workflow_dispatch
+
+jobs:
+  integration:
+    name: Integration
+    timeout-minutes: 150
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: lxd
+          juju-channel: 3.4/stable
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: astral-sh/setup-uv@v3
+      - run: uvx tox -e integration-flakey

tests/integration/test_application.py

@@ -21,6 +21,7 @@
 from ..utils import INTEGRATION_TEST_DIR
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_action():
     async with base.CleanModel() as model:
@@ -181,6 +182,7 @@ async def test_upgrade_charm_switch():
         assert app.data['charm-url'] == 'ubuntu'
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_upgrade_local_charm():
     async with base.CleanModel() as model:
@@ -194,6 +196,7 @@ async def test_upgrade_local_charm():
         assert app.data['charm-url'] == 'local:focal/ubuntu-0'
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_upgrade_local_charm_resource():
     async with base.CleanModel() as model:
@@ -313,6 +316,7 @@ async def test_trusted():
         assert trusted is False
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_app_destroy():
     async with base.CleanModel() as model:
@@ -328,6 +332,7 @@ async def test_app_destroy():
         assert a_name not in model.applications
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_app_remove_wait_flag():
     async with base.CleanModel() as model:
@@ -358,6 +363,7 @@ async def test_app_charm_name():
         assert 'ubuntu' == app.charm_name
 
 
+@pytest.mark.skip(reason="always fails")
 @base.bootstrapped
 async def test_app_relation_destroy_block_until_done():
     async with base.CleanModel() as model:

tests/integration/test_charmhub.py

@@ -118,6 +118,7 @@ async def test_subordinate_charm_zero_units():
         assert len(app2.units) == 0
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_subordinate_false_field_exists():
     async with base.CleanModel() as model:

tests/integration/test_machine.py

@@ -39,6 +39,7 @@ async def test_status():
             timeout=480)
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_machine_ssh():
     async with base.CleanModel() as model:

tests/integration/test_model.py

@@ -307,6 +307,7 @@ async def test_deploy_local_bundle_with_overlay_multi():
         assert 'ghost' not in model.applications
 
 
+@pytest.mark.skip(reason="always fails")
 @base.bootstrapped
 @pytest.mark.bundle
 async def test_deploy_bundle_with_overlay_as_argument():
@@ -340,6 +341,7 @@ async def test_deploy_bundle_with_multi_overlay_as_argument():
         assert 'mysql' in model.applications
 
 
+@pytest.mark.skip(reason="always fails")
 @base.bootstrapped
 @pytest.mark.bundle
 async def test_deploy_bundle_with_multiple_overlays_with_include_files():
@@ -430,6 +432,7 @@ async def test_deploy_from_ch_with_invalid_series():
             pass
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_deploy_with_base():
     async with base.CleanModel() as model:
@@ -628,6 +631,7 @@ def wait_for_network(container, timeout=30):
         profile.delete()
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_add_manual_machine_ssh():
     """Test manual machine provisioning with a non-root user
@@ -637,6 +641,7 @@ async def test_add_manual_machine_ssh():
     await add_manual_machine_ssh(is_root=False)
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_add_manual_machine_ssh_root():
     """Test manual machine provisioning with the root user"""
@@ -733,6 +738,7 @@ async def test_local_oci_image_resource_charm():
         assert charm.units[0].workload_status == 'active'
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_local_file_resource_charm():
     charm_path = INTEGRATION_TEST_DIR / 'file-resource-charm'
@@ -749,6 +755,7 @@ async def test_local_file_resource_charm():
         assert ress['file-res']
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_attach_resource():
     charm_path = TESTS_DIR / 'integration' / 'file-resource-charm'
@@ -845,6 +852,7 @@ async def test_wait_for_idle_without_units():
             await model.wait_for_idle(timeout=10)
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 @pytest.mark.wait_for_idle
 async def test_wait_for_idle_with_not_enough_units():
@@ -859,6 +867,7 @@ async def test_wait_for_idle_with_not_enough_units():
             await model.wait_for_idle(timeout=5 * 60, wait_for_at_least_units=3)
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 @pytest.mark.wait_for_idle
 async def test_wait_for_idle_more_units_than_needed():
@@ -968,6 +977,7 @@ async def test_wait_for_idle_with_exact_units_scale_down_zero():
         assert (end_time - start_time) > 0.001
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_destroy_units():
     async with base.CleanModel() as model:
@@ -1091,6 +1101,7 @@ async def test_application_annotations():
         assert annotations == expected
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_unit_annotations():
 

tests/integration/test_unit.py

@@ -62,6 +62,7 @@ async def test_unit_public_address():
             assert addr is not None
 
 
+@pytest.mark.flakey
 @base.bootstrapped
 async def test_run():
     from juju.action import Action

tox.ini

@@ -12,6 +12,8 @@ markers =
     serial: mark a test that must run by itself
     wait_for_idle: mark a test that waits for the model to be idle
     bundle: mark a test that uses a bundle
+    # https://github.com/juju/python-libjuju/issues/1108#issuecomment-2387154292
+    flakey: exclude the test from the default set
 asyncio_mode = auto
 filterwarnings =
     ignore::DeprecationWarning:websockets
@@ -63,7 +65,14 @@ envdir = {toxworkdir}/py3
 commands =
     pip install urllib3<2
     pip install pylxd
-    python -m pytest --tb native -ra -v -n auto -k 'integration' -m 'not serial' {posargs}
+    python -m pytest --tb native -ra -v -n auto -k 'integration' -m 'not serial and not flakey' {posargs}
+
+[testenv:integration-flakey]
+envdir = {toxworkdir}/py3
+commands =
+    pip install urllib3<2
+    pip install pylxd
+    python -m pytest --tb native -ra -v -n auto -k 'integration' -m 'not serial and flakey' {posargs}
 
 [testenv:unit]
 envdir = {toxworkdir}/py3