From 140aa59d64eeec424619f372fa7380df40fd35d7 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Thu, 8 Feb 2024 13:53:56 +0100 Subject: [PATCH] Ceph: workloads adoption with cinder volume Enable back the cinder volume commands on the source cloud, and resume testing of the ceph-backed volume attached to the test VM. Extend volume/backup/snapshot/attachment commands to wait for the previous step results. Follow the EDPM Post Ceph steps of HCI VA to prepare adopted workloads for using Ceph backend on EDPM. Add Nova discover host command (step 5 of the HCI VA). Add Nova Ceph custom configs to properly configure ceph vms pool for libvirt. Combine nova-ceph related configurations and nova FFU related ones into a single nova-compute-extraconfig service (by design, having two dataplane services for Nova in the same node set is not supported). Note about available choises for libvirt storage backends for Nova Add nova_libvirt_backend to control either to deploy with the local or ceph storage EDPM backends Signed-off-by: Bohdan Dobrelia --- .../assemblies/development_environment.adoc | 26 ++--- ...ng-compute-services-to-the-data-plane.adoc | 106 +++++++++++++++++- ...t-forward-upgrade-on-compute-services.adoc | 6 + tests/playbooks/test_rollback_with_ceph.yaml | 1 + tests/playbooks/test_with_ceph.yaml | 1 + tests/playbooks/test_with_ironic.yaml | 1 + .../dataplane_adoption/defaults/main.yaml | 2 + .../roles/dataplane_adoption/tasks/main.yaml | 100 ++++++++++++++++- .../dataplane_adoption/tasks/nova_ffu.yaml | 8 +- .../files/pre_launch.bash | 45 ++++++-- .../development_environment/tasks/main.yaml | 1 + tests/roles/nova_adoption/defaults/main.yaml | 2 + 12 files changed, 263 insertions(+), 36 deletions(-) diff --git a/docs_dev/assemblies/development_environment.adoc b/docs_dev/assemblies/development_environment.adoc index 2b0c39221..a949fdb54 100644 --- a/docs_dev/assemblies/development_environment.adoc +++ b/docs_dev/assemblies/development_environment.adoc @@ -108,6 +108,7 @@ export EDPM_COMPUTE_ADDITIONAL_NETWORKS=$(jq -c . /tmp/addtional_nets.json) export STANDALONE_COMPUTE_DRIVER=ironic export NTP_SERVER=pool.ntp.org # Only necessary if not on the RedHat network ... export EDPM_COMPUTE_CEPH_ENABLED=false # Optional +export EDPM_COMPUTE_CEPH_NOVA=false # Optional export EDPM_COMPUTE_SRIOV_ENABLED=false # Without this the standalone deploy fails when compute driver is ironic. ---- @@ -115,6 +116,8 @@ export EDPM_COMPUTE_SRIOV_ENABLED=false # Without this the standalone deploy fai === If `EDPM_COMPUTE_CEPH_ENABLED=false` is set, TripleO configures `Glance` with `Swift` as a backend. +If `EDPM_COMPUTE_CEPH_NOVA=false` is set, TripleO configures `Nova/Libvirt` with +a local storage backend. === ''' @@ -301,19 +304,20 @@ ping -c 4 $(openstack server show baremetal-test -f json -c addresses | jq -r .a ==== Virtual Machine Steps +Create a test VM instance with a test volume attachement: + [,bash] ---- cd ~/data-plane-adoption bash tests/roles/development_environment/files/pre_launch.bash ---- +This also creates a test Cinder volume, a backup from it, and a snapshot of it. + ''' ==== Ceph Storage Steps -Make sure a cinder-volume backend is properly configured, or skip below steps -to create a test workload without volume attachments. - Confirm the image UUID can be seen in Ceph's images pool. [,bash] @@ -326,22 +330,6 @@ Create a Barbican secret openstack secret store --name testSecret --payload 'TestPayload' ``` -Create a Cinder volume, a backup from it, and snapshot it. - -[,bash] ----- -openstack volume create --image cirros --bootable --size 1 disk -openstack volume backup create --name backup disk -openstack volume snapshot create --volume disk snapshot ----- - -Add volume to the test VM - -[,bash] ----- -openstack server add volume test disk ----- - == Performing the Data Plane Adoption The development environment is now set up, you can go to the https://openstack-k8s-operators.github.io/data-plane-adoption/[Adoption diff --git a/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc b/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc index 0b7883b8d..3c07bbc7d 100644 --- a/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc +++ b/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc @@ -7,8 +7,9 @@ .Prerequisites * Remaining source cloud xref:stopping-infrastructure-management-and-compute-services_{context}[Stopping infrastructure management and Compute services] on Compute hosts. -* Make sure the IPAM is configured: -+ +* Ceph backend for Nova/Libvirt is configured xref:configuring-a-ceph-backend_{context}[Configuring a Ceph backend]. +* Make sure the IPAM is configured + ---- oc apply -f - <-compute-config` is auto-generated for each `cell`. You must specify `nova-cell-compute-config` and `nova-migration-ssh-key` for each custom `OpenStackDataPlaneService` related to the Compute service. +That service removes pre-FFU workarounds and configures Nova compute +services for local storage backend. + +* Or, create a `nova-compute-extra-config` service (with Ceph backend for libvirt): ++ +[source,yaml] +---- +oc apply -f - <` with the value of the bridge mappings in your configuration, for example, `"datacentre:br-ctlplane"`. . Ensure that the `ovn-controller` settings that are configured in the `OpenStackDataPlaneNodeSet` CR are the same as were set in the Compute nodes before adoption. This configuration is stored in the `external_ids`` column in the `Open_vSwitch` table in the Open vSwitch database: @@ -419,6 +513,10 @@ ovs-vsctl list Open . external_ids : {hostname=standalone.localdomain, ovn-bridge=br-int, ovn-bridge-mappings=, ovn-chassis-mac-mappings="datacentre:1e:0a:bb:e6:7c:ad", ovn-encap-ip="172.19.0.100", ovn-encap-tos="0", ovn-encap-type=geneve, ovn-match-northd-version=False, ovn-monitor-all=True, ovn-ofctrl-wait-before-clear="8000", ovn-openflow-probe-interval="60", ovn-remote="tcp:ovsdbserver-sb.openstack.svc:6642", ovn-remote-probe-interval="60000", rundir="/var/run/openvswitch", system-id="2eec68e6-aa21-4c95-a868-31aeafc11736"} ... ---- + +Note that you should retain the original `OpenStackDataPlaneNodeSet` services +composition, except the inserted `ceph-client` service. + + * Replace `` with the value of the bridge mappings in your configuration, for example, `"datacentre:br-ctlplane"`. diff --git a/docs_user/modules/proc_performing-a-fast-forward-upgrade-on-compute-services.adoc b/docs_user/modules/proc_performing-a-fast-forward-upgrade-on-compute-services.adoc index f29e71001..7a95eb780 100644 --- a/docs_user/modules/proc_performing-a-fast-forward-upgrade-on-compute-services.adoc +++ b/docs_user/modules/proc_performing-a-fast-forward-upgrade-on-compute-services.adoc @@ -109,6 +109,12 @@ oc exec -it nova-cell0-conductor-0 -- nova-manage db online_data_migrations oc exec -it nova-cell1-conductor-0 -- nova-manage db online_data_migrations ---- +. Discover Nova compute hosts in the cell: ++ +---- +oc rsh nova-cell0-conductor-0 nova-manage cell_v2 discover_hosts --verbose +---- + . Verify if Compute services can stop the existing test VM instance: + ---- diff --git a/tests/playbooks/test_rollback_with_ceph.yaml b/tests/playbooks/test_rollback_with_ceph.yaml index c5998cf2a..ec9d3c42b 100644 --- a/tests/playbooks/test_rollback_with_ceph.yaml +++ b/tests/playbooks/test_rollback_with_ceph.yaml @@ -9,6 +9,7 @@ manila_backend: cephfs cinder_volume_backend: ceph cinder_backup_backend: ceph + nova_libvirt_backend: ceph module_defaults: ansible.builtin.shell: executable: /bin/bash diff --git a/tests/playbooks/test_with_ceph.yaml b/tests/playbooks/test_with_ceph.yaml index 2282fa771..86967b0da 100644 --- a/tests/playbooks/test_with_ceph.yaml +++ b/tests/playbooks/test_with_ceph.yaml @@ -9,6 +9,7 @@ manila_backend: cephfs cinder_volume_backend: ceph cinder_backup_backend: ceph + nova_libvirt_backend: ceph module_defaults: ansible.builtin.shell: executable: /bin/bash diff --git a/tests/playbooks/test_with_ironic.yaml b/tests/playbooks/test_with_ironic.yaml index ee356f99c..35cf5bd81 100644 --- a/tests/playbooks/test_with_ironic.yaml +++ b/tests/playbooks/test_with_ironic.yaml @@ -25,6 +25,7 @@ executable: /bin/bash vars: ironic_adoption: true + nova_libvirt_backend: local roles: - role: development_environment - role: backend_services diff --git a/tests/roles/dataplane_adoption/defaults/main.yaml b/tests/roles/dataplane_adoption/defaults/main.yaml index 48c62e13c..b6bc54a1d 100644 --- a/tests/roles/dataplane_adoption/defaults/main.yaml +++ b/tests/roles/dataplane_adoption/defaults/main.yaml @@ -123,3 +123,5 @@ neutron_public_interface_name: "{{ dataplane_public_iface | default('eth0') }}" edpm_sshd_allowed_ranges: "{{ ['192.168.122.0/24'] if dataplane_os_net_config_set_route|default(true)|bool else ['0.0.0.0/0'] }}" edpm_neutron_sriov_agent_enabled: true edpm_neutron_dhcp_agent_enabled: true + +nova_libvirt_backend: local diff --git a/tests/roles/dataplane_adoption/tasks/main.yaml b/tests/roles/dataplane_adoption/tasks/main.yaml index a926040d6..8c139324c 100644 --- a/tests/roles/dataplane_adoption/tasks/main.yaml +++ b/tests/roles/dataplane_adoption/tasks/main.yaml @@ -1,3 +1,11 @@ +- name: set shell vars for edpm adoption + no_log: "{{ use_no_log }}" + when: + - ('ceph' in [nova_libvirt_backend]) + ansible.builtin.set_fact: + ceph_backend_configuration_fsid_shell_vars: | + CEPH_FSID=$(oc get secret ceph-conf-files -o json | jq -r '.data."ceph.conf"' | base64 -d | grep fsid | sed -e 's/fsid = //') + - name: Save ansibleee-operator pod name to be able to wait for the rollout of the new pod no_log: "{{ use_no_log }}" ansible.builtin.shell: | @@ -125,7 +133,9 @@ rm -f id* cd - -- name: create a Nova Compute Extra Config service +- name: create a Nova Compute Extra Config service (no ceph backend in use) + when: + - ('ceph' not in [nova_libvirt_backend]) no_log: "{{ use_no_log }}" ansible.builtin.shell: | {{ shell_header }} @@ -166,6 +176,58 @@ {% endif %} EOF +- name: create a Nova Compute Extra Config service (with ceph backend) + when: + - ('ceph' in [nova_libvirt_backend]) + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ ceph_backend_configuration_fsid_shell_vars }} + oc apply -f - <antelope FFU +- name: run Nova DB online migrations to complete FFU ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} @@ -124,3 +124,9 @@ until: nova_exec_result is success retries: 10 delay: 6 + +- name: discover Nova compute hosts in the cell + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc rsh nova-cell0-conductor-0 nova-manage cell_v2 discover_hosts --verbose diff --git a/tests/roles/development_environment/files/pre_launch.bash b/tests/roles/development_environment/files/pre_launch.bash index 9238bf364..d40859e52 100755 --- a/tests/roles/development_environment/files/pre_launch.bash +++ b/tests/roles/development_environment/files/pre_launch.bash @@ -2,6 +2,21 @@ set -e alias openstack="ssh -i ~/install_yamls/out/edpm/ansibleee-ssh-key-id_rsa root@192.168.122.100 OS_CLOUD=standalone openstack" +function wait_for_status() { + local time=0 + local msg="Waiting for $2" + local status="${3:-available}" + local result + while [ $time -le 30 ] ; do + result=$(${BASH_ALIASES[openstack]} $1 -f json) + echo $result | jq -r ".status" | grep -q $status && break + echo "result=$result" + echo "$msg" + time=$(( time + 5 )) + sleep 5 + done +} + # Create Image IMG=cirros-0.5.2-x86_64-disk.img URL=http://download.cirros-cloud.net/0.5.2/$IMG @@ -52,15 +67,23 @@ export FIP=192.168.122.20 # check connectivity via FIP ping -c4 ${FIP} -# FIXME: Invalid volume: Volume xxx status must be available, but current status is: backing-up -# Create a Cinder volume, a backup from it, and snapshot it. -#${BASH_ALIASES[openstack]} volume show disk || \ -# ${BASH_ALIASES[openstack]} volume create --image cirros --bootable --size 1 disk -#${BASH_ALIASES[openstack]} volume backup show backup || \ -# ${BASH_ALIASES[openstack]} volume backup create --name backup disk -#${BASH_ALIASES[openstack]} volume snapshot show snapshot || \ -# ${BASH_ALIASES[openstack]} volume snapshot create --volume disk snapshot +if ! ${BASH_ALIASES[openstack]} volume show disk ; then + ${BASH_ALIASES[openstack]} volume create --image cirros --bootable --size 1 disk + wait_for_status "volume show disk" "test volume 'disk' creation" +fi + +if ! ${BASH_ALIASES[openstack]} volume backup show backup; then + ${BASH_ALIASES[openstack]} volume backup create --name backup disk + wait_for_status "volume backup show backup" "test volume 'disk' backup completion" +fi + +if ! ${BASH_ALIASES[openstack]} volume snapshot show snapshot ; then + ${BASH_ALIASES[openstack]} volume snapshot create --volume disk snapshot + wait_for_status "volume snapshot show snapshot" "test volume 'disk' backup snapshot availability" +fi + +# Add volume to the test VM +if ${BASH_ALIASES[openstack]} volume show disk -f json | jq -r '.status' | grep -q available ; then + ${BASH_ALIASES[openstack]} server add volume test disk +fi -# TODO: Add volume to the test VM, after tripleo wallaby (osp 17) isolnet network adoption implemented for storage networks -#${BASH_ALIASES[openstack]} volume show disk -f json | jq -r '.status' | grep -q available && \ -# ${BASH_ALIASES[openstack]} server add volume test disk diff --git a/tests/roles/development_environment/tasks/main.yaml b/tests/roles/development_environment/tasks/main.yaml index b226c218c..4b2f47a8a 100644 --- a/tests/roles/development_environment/tasks/main.yaml +++ b/tests/roles/development_environment/tasks/main.yaml @@ -6,5 +6,6 @@ EDPM_PRIVATEKEY_PATH: "{{ edpm_privatekey_path }}" ansible.builtin.shell: cmd: | + {{ shell_header }} alias openstack="ssh -i {{ edpm_privatekey_path }} root@{{ edpm_node_ip }} OS_CLOUD=standalone openstack" {{ lookup('ansible.builtin.file', prelaunch_test_instance_script) }} diff --git a/tests/roles/nova_adoption/defaults/main.yaml b/tests/roles/nova_adoption/defaults/main.yaml index 4f5d459c7..0206c677f 100644 --- a/tests/roles/nova_adoption/defaults/main.yaml +++ b/tests/roles/nova_adoption/defaults/main.yaml @@ -1 +1,3 @@ ironic_adoption: false +# Libvirt backends: ceph or local +nova_libvirt_backend: local