Feature/bseng 1937/upgrade unit by unit

cou/apps/base.py

@@ -33,6 +33,7 @@
     ApplicationUpgradePlan,
     PostUpgradeStep,
     PreUpgradeStep,
+    UnitUpgradeStep,
     UpgradeStep,
 )
 from cou.utils.app_utils import upgrade_packages
@@ -517,12 +518,21 @@ def _get_upgrade_current_release_packages_step(self) -> PreUpgradeStep:
         :return: Step for upgrading software packages to the latest of the current release.
         :rtype: PreUpgradeStep
         """
-        return PreUpgradeStep(
+        step = PreUpgradeStep(
             description=(
                 f"Upgrade software packages of '{self.name}' from the current APT repositories"
             ),
-            coro=upgrade_packages(self.status.units.keys(), self.model, self.packages_to_hold),
+            parallel=True,
         )
+        for unit in self.units:
+            step.add_step(
+                UnitUpgradeStep(
+                    description=f"Upgrade software packages on unit {unit.name}",
+                    coro=upgrade_packages(unit.name, self.model, self.packages_to_hold),
+                )
+            )
+
+        return step
 
     def _get_refresh_charm_step(self, target: OpenStackRelease) -> PreUpgradeStep:
         """Get step for refreshing the current channel.

cou/steps/__init__.py

@@ -21,7 +21,7 @@
 import logging
 import os
 import warnings
-from typing import Any, Coroutine, List, Optional
+from typing import Any, Coroutine, Iterable, List, Optional
 
 from cou.exceptions import CanceledStep
 
@@ -90,7 +90,7 @@ def __init__(
         self._coro: Optional[Coroutine] = coro
         self.parallel = parallel
         self.description = description
-        self.sub_steps: List[BaseStep] = []
+        self._sub_steps: List[BaseStep] = []
         self._canceled: bool = False
         self._task: Optional[asyncio.Task] = None
 
@@ -195,13 +195,36 @@ def done(self) -> bool:
 
         return self._task.done()
 
+    @property
+    def sub_steps(self) -> List[BaseStep]:
+        """Return list of sub-steps.
+
+        :return: List of BaseStep.
+        :rtype: List[BaseStep]
+        """
+        return self._sub_steps
+
+    @sub_steps.setter
+    def sub_steps(self, steps: Iterable[BaseStep]) -> None:
+        """Set a list of sub-steps.
+
+        :param steps: Iterable object containing all steps.
+        :type steps: Iterable
+        """
+        for step in steps:
+            self.add_step(step)
+
     def add_step(self, step: BaseStep) -> None:
         """Add a single step.
 
         :param step: BaseStep to be added as sub step.
         :type step: BaseStep
+        :raises TypeError: If step is not based on BaseStep.
         """
-        self.sub_steps.append(step)
+        if not isinstance(step, BaseStep):
+            raise TypeError("Cannot add an upgrade step that is not derived from BaseStep")
+
+        self._sub_steps.append(step)
 
     def cancel(self, safe: bool = True) -> None:
         """Cancel step and all its sub steps.
@@ -286,6 +309,10 @@ class UpgradeStep(BaseStep):
     prompt: bool = False
 
 
+class UnitUpgradeStep(UpgradeStep):
+    """Represents the upgrade step for an individual unit."""
+
+
 class PreUpgradeStep(UpgradeStep):
     """Represents the pre-upgrade step."""
 

cou/utils/app_utils.py

@@ -15,7 +15,6 @@
 """Application utilities."""
 import json
 import logging
-from collections.abc import Iterable
 from typing import Optional
 
 from packaging.version import Version
@@ -27,13 +26,11 @@
 logger = logging.getLogger(__name__)
 
 
-async def upgrade_packages(
-    units: Iterable[str], model: COUModel, packages_to_hold: Optional[list]
-) -> None:
+async def upgrade_packages(unit: str, model: COUModel, packages_to_hold: Optional[list]) -> None:
     """Run package updates and upgrades on each unit of an Application.
 
-    :param units: The list of unit names where the package upgrade runs on.
-    :type units: Iterable[str]
+    :param unit: Unit name where the package upgrade runs on.
+    :type unit: str
     :param model: COUModel object
     :type model: COUModel
     :param packages_to_hold: A list of packages to put on hold during package upgrade.
@@ -46,8 +43,7 @@ async def upgrade_packages(
         packages = " ".join(packages_to_hold)
         command = f"apt-mark hold {packages} && {command} ; apt-mark unhold {packages}"
 
-    for unit in units:
-        await model.run_on_unit(unit_name=unit, command=command, timeout=600)
+    await model.run_on_unit(unit_name=unit, command=command, timeout=600)
 
 
 async def set_require_osd_release_option(unit: str, model: COUModel) -> None:

docs/how-to/interruption.rst

@@ -14,7 +14,7 @@ the process at any prompt.
 
 Usage example:
 
-.. terminal:: 
+.. terminal::
     :input: cou upgrade
 
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
@@ -28,6 +28,9 @@ Usage example:
     Backup mysql databases ✔
     Upgrade plan for 'keystone' to victoria
         Upgrade software packages of 'keystone' from the current APT repositories
+            Upgrade software packages on unit keystone/0
+            Upgrade software packages on unit keystone/1
+            Upgrade software packages on unit keystone/2
         Upgrade 'keystone' to the new channel: 'victoria/stable'
         Change charm config of 'keystone' 'openstack-origin' to 'cloud:focal-victoria'
         Wait 1800s for model test-model to reach the idle state.
@@ -49,7 +52,7 @@ the cloud may be left in an inconsistent state.
 
 Exiting before running upgrade plan:
 
-.. terminal:: 
+.. terminal::
     :input: cou upgrade - # ctrl+c is pressed while connecting to the controller
 
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
@@ -63,7 +66,7 @@ Exiting before running upgrade plan:
 
 Safe cancel:
 
-.. terminal:: 
+.. terminal::
     :input: cou upgrade # ctrl+c is pressed once during the upgrade
 
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
@@ -73,11 +76,11 @@ Safe cancel:
     ...
     Running cloud upgrade...
     Canceling upgrade... (Press ctrl+c again to stop immediately) ✔
-    charmed-openstack-upgrader has been stopped safely 
+    charmed-openstack-upgrader has been stopped safely
 
 Unsafe cancel:
 
-.. terminal:: 
+.. terminal::
     :input: cou upgrade # ctrl+c is pressed twice during the upgrade
 
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'

docs/how-to/no-backup.rst

@@ -10,7 +10,7 @@ Usage examples
 
 Plan:
 
-.. terminal:: 
+.. terminal::
     :input: cou plan --no-backup
 
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
@@ -23,11 +23,14 @@ Plan:
         Control Plane principal(s) upgrade plan
         Upgrade plan for 'rabbitmq-server' to victoria
             Upgrade software packages of 'rabbitmq-server' from the current APT repositories
+                Upgrade software packages on unit rabbitmq-server/0
+                Upgrade software packages on unit rabbitmq-server/1
+                Upgrade software packages on unit rabbitmq-server/2
         ...
 
 Upgrade:
 
-.. terminal:: 
+.. terminal::
     :input: cou upgrade --no-backup
 
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
@@ -41,13 +44,16 @@ Upgrade:
 
     Upgrade plan for 'rabbitmq-server' to victoria
         Upgrade software packages of 'rabbitmq-server' from the current APT repositories
+            Upgrade software packages on unit rabbitmq-server/0
+            Upgrade software packages on unit rabbitmq-server/1
+            Upgrade software packages on unit rabbitmq-server/2
         Upgrade 'rabbitmq-server' to the new channel: '3.9/stable'
         Change charm config of 'rabbitmq-server' 'source' to 'cloud:focal-victoria'
         Wait 1800s for model test-model to reach the idle state.
         Check if the workload of 'rabbitmq-server' has been upgraded
 
     Continue (y/n): y
     Upgrade plan for 'rabbitmq-server' to victoria ✔
-    
+
     ... # apply steps
     Upgrade completed.

docs/how-to/plan-upgrade.rst

@@ -12,9 +12,9 @@ printed on STDOUT.
 Usage example
 -------------
 
-.. terminal:: 
+.. terminal::
     :input: cou plan
-    
+
     Full execution log: '/home/ubuntu/.local/share/cou/log/cou-20231215211717.log'
     Connected to 'test-model' ✔
     Analyzing cloud... ✔
@@ -25,54 +25,67 @@ Usage example
         Control Plane principal(s) upgrade plan
         Upgrade plan for 'rabbitmq-server' to victoria
             Upgrade software packages of 'rabbitmq-server' from the current APT repositories
+                Upgrade software packages on unit rabbitmq-server/0
+                Upgrade software packages on unit rabbitmq-server/1
+                Upgrade software packages on unit rabbitmq-server/2
             Upgrade 'rabbitmq-server' to the new channel: '3.9/stable'
             Change charm config of 'rabbitmq-server' 'source' to 'cloud:focal-victoria'
             Wait 1800s for model test-model to reach the idle state.
             Check if the workload of 'rabbitmq-server' has been upgraded
         Upgrade plan for 'keystone' to victoria
             Upgrade software packages of 'keystone' from the current APT repositories
+                Upgrade software packages on unit keystone/0
+                Upgrade software packages on unit keystone/1
+                Upgrade software packages on unit keystone/2
             Upgrade 'keystone' to the new channel: 'victoria/stable'
             Change charm config of 'keystone' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 1800s for model test-model to reach the idle state.
             Check if the workload of 'keystone' has been upgraded
         Upgrade plan for 'cinder' to victoria
             Upgrade software packages of 'cinder' from the current APT repositories
+                Upgrade software packages on unit cinder/0
             Upgrade 'cinder' to the new channel: 'victoria/stable'
             Change charm config of 'cinder' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 300s for app cinder to reach the idle state.
             Check if the workload of 'cinder' has been upgraded
         Upgrade plan for 'glance' to victoria
             Upgrade software packages of 'glance' from the current APT repositories
+                Upgrade software packages on unit glance/0
             Upgrade 'glance' to the new channel: 'victoria/stable'
             Change charm config of 'glance' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 300s for app glance to reach the idle state.
             Check if the workload of 'glance' has been upgraded
         Upgrade plan for 'neutron-api' to victoria
             Upgrade software packages of 'neutron-api' from the current APT repositories
+                Upgrade software packages on unit neutron-api/0
             Upgrade 'neutron-api' to the new channel: 'victoria/stable'
             Change charm config of 'neutron-api' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 300s for app neutron-api to reach the idle state.
             Check if the workload of 'neutron-api' has been upgraded
         Upgrade plan for 'neutron-gateway' to victoria
             Upgrade software packages of 'neutron-gateway' from the current APT repositories
+                Upgrade software packages on unit neutron-gateway/0
             Upgrade 'neutron-gateway' to the new channel: 'victoria/stable'
             Change charm config of 'neutron-gateway' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 300s for app neutron-gateway to reach the idle state.
             Check if the workload of 'neutron-gateway' has been upgraded
         Upgrade plan for 'placement' to victoria
             Upgrade software packages of 'placement' from the current APT repositories
+                Upgrade software packages on unit placement/0
             Upgrade 'placement' to the new channel: 'victoria/stable'
             Change charm config of 'placement' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 300s for app placement to reach the idle state.
             Check if the workload of 'placement' has been upgraded
         Upgrade plan for 'nova-cloud-controller' to victoria
             Upgrade software packages of 'nova-cloud-controller' from the current APT repositories
+                Upgrade software packages on unit nova-cloud-controller/0
             Upgrade 'nova-cloud-controller' to the new channel: 'victoria/stable'
             Change charm config of 'nova-cloud-controller' 'openstack-origin' to 'cloud:focal-victoria'
             Wait 300s for app nova-cloud-controller to reach the idle state.
             Check if the workload of 'nova-cloud-controller' has been upgraded
         Upgrade plan for 'mysql' to victoria
             Upgrade software packages of 'mysql' from the current APT repositories
+                Upgrade software packages on unit mysql/0
             Change charm config of 'mysql' 'source' to 'cloud:focal-victoria'
             Wait 1800s for app mysql to reach the idle state.
             Check if the workload of 'mysql' has been upgraded