diff --git a/.github/workflows/flakey-integration-tests.yaml b/.github/workflows/flakey-integration-tests.yaml new file mode 100644 index 000000000..8a0d398df --- /dev/null +++ b/.github/workflows/flakey-integration-tests.yaml @@ -0,0 +1,20 @@ +name: Flakey integration tests + +on: workflow_dispatch + +jobs: + integration: + name: Integration + timeout-minutes: 150 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: charmed-kubernetes/actions-operator@main + with: + provider: lxd + juju-channel: 3.4/stable + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: astral-sh/setup-uv@v3 + - run: uvx tox -e integration-flakey diff --git a/tests/integration/test_application.py b/tests/integration/test_application.py index ecac8b7df..bbeb098b6 100644 --- a/tests/integration/test_application.py +++ b/tests/integration/test_application.py @@ -21,6 +21,7 @@ from ..utils import INTEGRATION_TEST_DIR +@pytest.mark.flakey @base.bootstrapped async def test_action(): async with base.CleanModel() as model: @@ -181,6 +182,7 @@ async def test_upgrade_charm_switch(): assert app.data['charm-url'] == 'ubuntu' +@pytest.mark.flakey @base.bootstrapped async def test_upgrade_local_charm(): async with base.CleanModel() as model: @@ -194,6 +196,7 @@ async def test_upgrade_local_charm(): assert app.data['charm-url'] == 'local:focal/ubuntu-0' +@pytest.mark.flakey @base.bootstrapped async def test_upgrade_local_charm_resource(): async with base.CleanModel() as model: @@ -313,6 +316,7 @@ async def test_trusted(): assert trusted is False +@pytest.mark.flakey @base.bootstrapped async def test_app_destroy(): async with base.CleanModel() as model: @@ -328,6 +332,7 @@ async def test_app_destroy(): assert a_name not in model.applications +@pytest.mark.flakey @base.bootstrapped async def test_app_remove_wait_flag(): async with base.CleanModel() as model: @@ -358,6 +363,7 @@ async def test_app_charm_name(): assert 'ubuntu' == app.charm_name +@pytest.mark.skip(reason="always fails") @base.bootstrapped async def test_app_relation_destroy_block_until_done(): async with base.CleanModel() as model: diff --git a/tests/integration/test_charmhub.py b/tests/integration/test_charmhub.py index bada3557e..90c4d9adf 100644 --- a/tests/integration/test_charmhub.py +++ b/tests/integration/test_charmhub.py @@ -118,6 +118,7 @@ async def test_subordinate_charm_zero_units(): assert len(app2.units) == 0 +@pytest.mark.flakey @base.bootstrapped async def test_subordinate_false_field_exists(): async with base.CleanModel() as model: diff --git a/tests/integration/test_machine.py b/tests/integration/test_machine.py index 1a6804152..635885a3f 100644 --- a/tests/integration/test_machine.py +++ b/tests/integration/test_machine.py @@ -39,6 +39,7 @@ async def test_status(): timeout=480) +@pytest.mark.flakey @base.bootstrapped async def test_machine_ssh(): async with base.CleanModel() as model: diff --git a/tests/integration/test_model.py b/tests/integration/test_model.py index 8d213553a..295543002 100644 --- a/tests/integration/test_model.py +++ b/tests/integration/test_model.py @@ -307,6 +307,7 @@ async def test_deploy_local_bundle_with_overlay_multi(): assert 'ghost' not in model.applications +@pytest.mark.skip(reason="always fails") @base.bootstrapped @pytest.mark.bundle async def test_deploy_bundle_with_overlay_as_argument(): @@ -340,6 +341,7 @@ async def test_deploy_bundle_with_multi_overlay_as_argument(): assert 'mysql' in model.applications +@pytest.mark.skip(reason="always fails") @base.bootstrapped @pytest.mark.bundle async def test_deploy_bundle_with_multiple_overlays_with_include_files(): @@ -430,6 +432,7 @@ async def test_deploy_from_ch_with_invalid_series(): pass +@pytest.mark.flakey @base.bootstrapped async def test_deploy_with_base(): async with base.CleanModel() as model: @@ -628,6 +631,7 @@ def wait_for_network(container, timeout=30): profile.delete() +@pytest.mark.flakey @base.bootstrapped async def test_add_manual_machine_ssh(): """Test manual machine provisioning with a non-root user @@ -637,6 +641,7 @@ async def test_add_manual_machine_ssh(): await add_manual_machine_ssh(is_root=False) +@pytest.mark.flakey @base.bootstrapped async def test_add_manual_machine_ssh_root(): """Test manual machine provisioning with the root user""" @@ -733,6 +738,7 @@ async def test_local_oci_image_resource_charm(): assert charm.units[0].workload_status == 'active' +@pytest.mark.flakey @base.bootstrapped async def test_local_file_resource_charm(): charm_path = INTEGRATION_TEST_DIR / 'file-resource-charm' @@ -749,6 +755,7 @@ async def test_local_file_resource_charm(): assert ress['file-res'] +@pytest.mark.flakey @base.bootstrapped async def test_attach_resource(): charm_path = TESTS_DIR / 'integration' / 'file-resource-charm' @@ -845,6 +852,7 @@ async def test_wait_for_idle_without_units(): await model.wait_for_idle(timeout=10) +@pytest.mark.flakey @base.bootstrapped @pytest.mark.wait_for_idle async def test_wait_for_idle_with_not_enough_units(): @@ -859,6 +867,7 @@ async def test_wait_for_idle_with_not_enough_units(): await model.wait_for_idle(timeout=5 * 60, wait_for_at_least_units=3) +@pytest.mark.flakey @base.bootstrapped @pytest.mark.wait_for_idle async def test_wait_for_idle_more_units_than_needed(): @@ -968,6 +977,7 @@ async def test_wait_for_idle_with_exact_units_scale_down_zero(): assert (end_time - start_time) > 0.001 +@pytest.mark.flakey @base.bootstrapped async def test_destroy_units(): async with base.CleanModel() as model: @@ -1091,6 +1101,7 @@ async def test_application_annotations(): assert annotations == expected +@pytest.mark.flakey @base.bootstrapped async def test_unit_annotations(): diff --git a/tests/integration/test_unit.py b/tests/integration/test_unit.py index d9928e36a..0ff8e0b67 100644 --- a/tests/integration/test_unit.py +++ b/tests/integration/test_unit.py @@ -62,6 +62,7 @@ async def test_unit_public_address(): assert addr is not None +@pytest.mark.flakey @base.bootstrapped async def test_run(): from juju.action import Action diff --git a/tox.ini b/tox.ini index a0d2aa367..940f96c36 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,8 @@ markers = serial: mark a test that must run by itself wait_for_idle: mark a test that waits for the model to be idle bundle: mark a test that uses a bundle + # https://github.com/juju/python-libjuju/issues/1108#issuecomment-2387154292 + flakey: exclude the test from the default set asyncio_mode = auto filterwarnings = ignore::DeprecationWarning:websockets @@ -63,7 +65,14 @@ envdir = {toxworkdir}/py3 commands = pip install urllib3<2 pip install pylxd - python -m pytest --tb native -ra -v -n auto -k 'integration' -m 'not serial' {posargs} + python -m pytest --tb native -ra -v -n auto -k 'integration' -m 'not serial and not flakey' {posargs} + +[testenv:integration-flakey] +envdir = {toxworkdir}/py3 +commands = + pip install urllib3<2 + pip install pylxd + python -m pytest --tb native -ra -v -n auto -k 'integration' -m 'not serial and flakey' {posargs} [testenv:unit] envdir = {toxworkdir}/py3