From 45950d098c270807cdb00019818a43b6f02bff31 Mon Sep 17 00:00:00 2001 From: Robert Gildein Date: Thu, 4 Apr 2024 10:33:51 +0200 Subject: [PATCH] Add base structure for testing of statuses from real cloud (#334) With these changes, we can simply add any result form real cloud to `tests/unit/sample_plans` directory in yaml file formatted like this: ```yaml plan: | Upgrade plan defined as string applications: keystone: can_upgrade_to: ussuri/stable charm: keystone channel: ussuri/stable config: openstack-origin: value: distro action-managed-upgrade: value: true origin: ch series: focal subordinate_to: [] workload_version: 17.0.1 units: keystone/0: name: keystone/0 machine: '0' workload_version: 17.0.1 os_version: ussuri machines: '0': id: '0' apps: !!python/tuple ['keystone-ldap'] az: az-0 ``` My only two concerns are: ~~- usage of `pytest-subtests`, which result could be strange if test failed, as you can see in current CI~~ After discussion I choose to got with only fixture returning dictionary with file as key, and tuple value with model and exp_plan. The model.get_applications returns applications defined in file. ~~- these test should be more part of functional tests instead of unit tests, or somehow exclude them from coverage report (e.g. 100% coverage is achieved without these tests)~~ After discussions we agreed that new tox env should be created, so I created `mocked-plans`. I also add new jobs to run this environment to our GitHub workflow. --------- Co-authored-by: TQ X --- .github/workflows/check.yaml | 64 +++++++-- cou/utils/nova_compute.py | 2 +- pyproject.toml | 4 - tests/mocked_plans/conftest.py | 37 +++++ tests/mocked_plans/sample_plans/base.yaml | 157 ++++++++++++++++++++++ tests/mocked_plans/test_base_plan.py | 34 +++++ tests/mocked_plans/utils.py | 77 +++++++++++ tests/unit/conftest.py | 23 +--- tests/unit/utils.py | 17 +++ tox.ini | 6 +- 10 files changed, 383 insertions(+), 38 deletions(-) create mode 100644 tests/mocked_plans/conftest.py create mode 100644 tests/mocked_plans/sample_plans/base.yaml create mode 100644 tests/mocked_plans/test_base_plan.py create mode 100644 tests/mocked_plans/utils.py diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 4dbace8e..4058a89f 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -17,15 +17,53 @@ concurrency: jobs: lint-unit: - name: Lint checkers and unit tests - uses: canonical/bootstack-actions/.github/workflows/lint-unit.yaml@v2 - strategy: - fail-fast: false - matrix: - python-version: ['3.10'] - with: - python-version: ${{ matrix.python-version }} - tox-version: '<4' + name: Lint checkers and Unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + - name: Run lint checkers + run: tox -e lint + - name: Run unit tests + run: tox -e unit + - name: Save PR number to file + run: echo ${{ github.event.number }} > PR_NUMBER.txt + - name: Archive PR number + uses: actions/upload-artifact@v4 + with: + name: PR_NUMBER + path: PR_NUMBER.txt + - name: Archive code coverage results + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: ./tests/unit/report/coverage.xml + + mocked-plans: + name: Mocked plans tests + needs: lint-unit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox + - name: Run mocked-plans tests + run: tox -e mocked-plans snap-build: name: Build snap package @@ -37,7 +75,7 @@ jobs: with: submodules: true - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install tox @@ -53,7 +91,7 @@ jobs: - name: Build snap run: make build - name: Upload the built snap as an artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SNAP_FILE path: charmed-openstack-upgrader.snap @@ -68,7 +106,7 @@ jobs: with: submodules: true - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Setup Juju 2.9/stable environment @@ -83,7 +121,7 @@ jobs: python -m pip install --upgrade pip python -m pip install tox - name: Download snap file artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SNAP_FILE - name: Run func tests diff --git a/cou/utils/nova_compute.py b/cou/utils/nova_compute.py index abf0b7f8..faf433a4 100644 --- a/cou/utils/nova_compute.py +++ b/cou/utils/nova_compute.py @@ -26,7 +26,7 @@ async def get_empty_hypervisors(units: list[Unit], model: Model) -> list[Machine]: """Get the empty hypervisors in the model. - :param units: all nova-compute units. + :param units: All nova-compute units. :type units: list[Unit] :param model: Model object :type model: Model diff --git a/pyproject.toml b/pyproject.toml index c556d392..2c446d4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ -# This is a template `pyproject.toml` file for snaps -# This file is managed by bootstack-charms-spec and should not be modified -# within individual snap repos. https://launchpad.net/bootstack-charms-spec - [tool.flake8] ignore = ["C901", "D100", "D101", "D102", "D103", "W503", "W504"] exclude = ['.eggs', '.git', '.tox', '.venv', '.build', 'build', 'report', 'docs'] diff --git a/tests/mocked_plans/conftest.py b/tests/mocked_plans/conftest.py new file mode 100644 index 00000000..b8708d40 --- /dev/null +++ b/tests/mocked_plans/conftest.py @@ -0,0 +1,37 @@ +# Copyright 2023 Canonical Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import pytest + +from cou.utils.juju_utils import Model +from tests.mocked_plans.utils import get_sample_plan + + +@pytest.fixture(scope="session") +def sample_plans() -> dict[str, tuple[Model, str]]: + """Fixture that returns all sample plans in a directory. + + This fixture returns a dictionary with the filename as the key and + a tuple consisting of a cou.utils.juju_utils.Model object and the + expected plan in string format as the value. The get_applications + function of this Model object returns the applications read from a + YAML file, from which the expected plan is also parsed. + """ + directory = Path(__file__).parent / "sample_plans" + + yield { + sample_file.name: get_sample_plan(sample_file) for sample_file in directory.glob("*.yaml") + } diff --git a/tests/mocked_plans/sample_plans/base.yaml b/tests/mocked_plans/sample_plans/base.yaml new file mode 100644 index 00000000..9ed3d199 --- /dev/null +++ b/tests/mocked_plans/sample_plans/base.yaml @@ -0,0 +1,157 @@ +plan: | + Upgrade cloud from 'ussuri' to 'victoria' + Verify that all OpenStack applications are in idle state + Back up MySQL databases + Control Plane principal(s) upgrade plan + Upgrade plan for 'keystone' to 'victoria' + Upgrade software packages of 'keystone' from the current APT repositories + Upgrade software packages on unit 'keystone/0' + Refresh 'keystone' to the latest revision of 'ussuri/stable' + Change charm config of 'keystone' 'action-managed-upgrade' to 'False' + Upgrade 'keystone' to the new channel: 'victoria/stable' + Change charm config of 'keystone' 'openstack-origin' to 'cloud:focal-victoria' + Wait for up to 1800s for model 'base' to reach the idle state + Verify that the workload of 'keystone' has been upgraded on units: keystone/0 + Control Plane subordinate(s) upgrade plan + Upgrade plan for 'keystone-ldap' to 'victoria' + Refresh 'keystone-ldap' to the latest revision of 'ussuri/stable' + Upgrade 'keystone-ldap' to the new channel: 'victoria/stable' + Upgrading all applications deployed on machines with hypervisor. + Upgrade plan for 'az-0' to 'victoria' + Disable nova-compute scheduler from unit: 'nova-compute/0' + Upgrade software packages of 'nova-compute' from the current APT repositories + Upgrade software packages on unit 'nova-compute/0' + Refresh 'nova-compute' to the latest revision of 'ussuri/stable' + Change charm config of 'nova-compute' 'action-managed-upgrade' to 'True' + Upgrade 'nova-compute' to the new channel: 'victoria/stable' + Change charm config of 'nova-compute' 'source' to 'cloud:focal-victoria' + Upgrade plan for units: nova-compute/0 + Upgrade plan for unit 'nova-compute/0' + Verify that unit 'nova-compute/0' has no VMs running + ├── Pause the unit: 'nova-compute/0' + ├── Upgrade the unit: 'nova-compute/0' + ├── Resume the unit: 'nova-compute/0' + Enable nova-compute scheduler from unit: 'nova-compute/0' + Wait for up to 1800s for model 'base' to reach the idle state + Verify that the workload of 'nova-compute' has been upgraded on units: nova-compute/0 + Remaining Data Plane principal(s) upgrade plan + Upgrade plan for 'ceph-osd' to 'victoria' + Verify that all 'nova-compute' units has been upgraded + Upgrade software packages of 'ceph-osd' from the current APT repositories + Upgrade software packages on unit 'ceph-osd/0' + Change charm config of 'ceph-osd' 'source' to 'cloud:focal-victoria' + Wait for up to 300s for app 'ceph-osd' to reach the idle state + Verify that the workload of 'ceph-osd' has been upgraded on units: ceph-osd/0 + Data Plane subordinate(s) upgrade plan + Upgrade plan for 'ovn-chassis' to 'victoria' + Refresh 'ovn-chassis' to the latest revision of '22.03/stable' + +applications: + keystone: + can_upgrade_to: ussuri/stable + charm: keystone + channel: ussuri/stable + config: + openstack-origin: + value: distro + action-managed-upgrade: + value: true + origin: ch + series: focal + subordinate_to: [] + workload_version: 17.0.1 + units: + keystone/0: + name: keystone/0 + machine: '0' + workload_version: 17.0.1 + os_version: ussuri + machines: + '0': + id: '0' + apps: !!python/tuple ['keystone', 'keystone-ldap'] + az: az-0 + + keystone-ldap: + can_upgrade_to: ussuri/stable + charm: keystone-ldap + channel: ussuri/stable + config: {} + origin: ch + series: focal + subordinate_to: + - keystone + workload_version: 17.0.1 + units: {} + machines: + '0': + id: '0' + apps: !!python/tuple ['keystone', 'keystone-ldap'] + az: az-0 + + ceph-osd: + can_upgrade_to: octopus/stable + charm: ceph-osd + channel: octopus/stable + config: + source: + value: distro + origin: ch + series: focal + subordinate_to: [] + workload_version: 17.0.1 + units: + ceph-osd/0: + name: ceph-osd/0 + machine: '2' + workload_version: 17.0.1 + os_version: xena + machines: + '2': + id: '2' + apps: !!python/tuple ['ceph-osd'] + az: az-0 + + nova-compute: + can_upgrade_to: ussuri/stable + charm: nova-compute + channel: ussuri/stable + config: + source: + value: distro + action-managed-upgrade: + value: false + origin: ch + series: focal + subordinate_to: [] + workload_version: 21.0.0 + units: + nova-compute/0: + name: nova-compute/0 + machine: '1' + workload_version: 21.0.0 + os_version: ussuri + machines: + '1': + id: '1' + apps: !!python/tuple ['nova-compute', 'ovn-chassis'] + az: az-0 + + ovn-chassis: + can_upgrade_to: 22.03/stable + charm: ovn-chassis + channel: 22.03/stable + config: + enable-version-pinning: + value: false + origin: ch + series: focal + subordinate_to: + - nova-compute + workload_version: '22.3' + units: {} + machines: + '1': + id: '1' + apps: !!python/tuple ['nova-compute', 'ovn-chassis'] + az: az-0 diff --git a/tests/mocked_plans/test_base_plan.py b/tests/mocked_plans/test_base_plan.py new file mode 100644 index 00000000..3614665b --- /dev/null +++ b/tests/mocked_plans/test_base_plan.py @@ -0,0 +1,34 @@ +# Copyright 2023 Canonical Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test all sample plans.""" +from unittest.mock import patch + +import pytest + +from cou.commands import CLIargs +from cou.steps.analyze import Analysis +from cou.steps.plan import generate_plan + + +@pytest.mark.asyncio +@patch("cou.utils.nova_compute.get_instance_count", return_value=0) +async def test_base_plan(_, sample_plans): + """Testing the base plans.""" + args = CLIargs("plan", auto_approve=True) + model, exp_plan = sample_plans["base.yaml"] + + analysis_results = await Analysis.create(model) + plan = await generate_plan(analysis_results, args) + + assert str(plan) == exp_plan diff --git a/tests/mocked_plans/utils.py b/tests/mocked_plans/utils.py new file mode 100644 index 00000000..d0516bc5 --- /dev/null +++ b/tests/mocked_plans/utils.py @@ -0,0 +1,77 @@ +# Copyright 2023 Canonical Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module to provide helper functions for writing mock upgrade tests.""" +from pathlib import Path +from unittest.mock import AsyncMock, PropertyMock + +import yaml + +from cou.utils.juju_utils import Application, Machine, Model, Unit +from tests.unit.utils import dedent_plan + + +def get_sample_plan(source: Path) -> tuple[Model, str]: + """Help function to get dict of Applications and expected upgrade plan from file. + + This function can load applications from yaml format, where each app is string representation + of OpenStackApplication (str(OpenStackApplication), see OpenStackApplication.__str__). + + applications: + : + model_name: ... + can_upgrade_to: ... + ... + : + model_name: ... + can_upgrade_to: ... + ... + plan: | + ... + """ + with open(source, "r") as file: + data = yaml.load(file, Loader=yaml.Loader) + + model = AsyncMock(spec_set=Model) + + # Note(rgildein): We need to get machines first, since they are used in Unit object. + machines = { + machine_id: Machine(machine["id"], machine["apps"], machine["az"]) + for app_data in data["applications"].values() + for machine_id, machine in app_data["machines"].items() + } + applications = { + name: Application( + name=name, + can_upgrade_to=app_data["can_upgrade_to"], + charm=app_data["charm"], + channel=app_data["channel"], + config=app_data["config"], + machines={machine_id: machines[machine_id] for machine_id in app_data["machines"]}, + model=model, + origin=app_data["origin"], + series=app_data["series"], + subordinate_to=app_data["subordinate_to"], + units={ + name: Unit(name, machines[unit["machine"]], unit["workload_version"]) + for name, unit in app_data["units"].items() + }, + workload_version=app_data["workload_version"], + ) + for name, app_data in data["applications"].items() + } + + type(model).name = PropertyMock(return_value=source.stem) + model.get_applications = AsyncMock(return_value=applications) + + return model, dedent_plan(data["plan"]) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9f81902d..45251830 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,36 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pytest -from juju.client.client import FullStatus from cou.commands import CLIargs - - -def get_status(): - """Help function to load Juju status from json file.""" - current_path = Path(__file__).parent.resolve() - with open(current_path / "jujustatus.json", "r") as file: - status = file.read().rstrip() - - return FullStatus.from_json(status) - - -async def get_charm_name(value: str): - """Help function to get charm name.""" - return value +from cou.utils.juju_utils import Model +from tests.unit.utils import get_charm_name, get_status @pytest.fixture -def model(): +def model() -> AsyncMock: """Define test Model object.""" model_name = "test_model" - from cou.utils import juju_utils - model = AsyncMock(spec_set=juju_utils.Model) + model = AsyncMock(spec_set=Model) type(model).name = PropertyMock(return_value=model_name) model.run_on_unit = AsyncMock() model.run_action = AsyncMock() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 9513c91e..f86746fe 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """Module to provide helper for writing unit tests.""" +from pathlib import Path from textwrap import dedent from unittest.mock import MagicMock +from juju.client.client import FullStatus + from cou.steps import BaseStep from cou.utils.juju_utils import Machine @@ -38,3 +41,17 @@ def dedent_plan(plan: str) -> str: result = dedent(plan) result = result.replace(" ", "\t") # replace 4 spaces with tap return result + + +def get_status(): + """Help function to load Juju status from json file.""" + current_path = Path(__file__).parent.resolve() + with open(current_path / "jujustatus.json", "r") as file: + status = file.read().rstrip() + + return FullStatus.from_json(status) + + +async def get_charm_name(value: str): + """Help function to get charm name.""" + return value diff --git a/tox.ini b/tox.ini index 5dce29d0..77106a36 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ [tox] skipsdist=True -envlist = lint, unit, func +envlist = lint, unit, mocked-plans, func skip_missing_interpreters = True [testenv] @@ -53,6 +53,10 @@ deps = .[unittests] commands = pytest {toxinidir}/tests/unit \ {posargs:-v --cov --cov-report=term-missing --cov-report=html --cov-report=xml} +[testenv:mocked-plans] +deps = .[unittests] +commands = pytest {toxinidir}/tests/mocked_plans/ {posargs:-s -vv} + [testenv:func] changedir = {toxinidir}/tests/functional deps = .[functests]