diff --git a/docs/openstack/backend_services_deployment.md b/docs/openstack/backend_services_deployment.md index b79396218..056b48107 100644 --- a/docs/openstack/backend_services_deployment.md +++ b/docs/openstack/backend_services_deployment.md @@ -167,6 +167,8 @@ podified OpenStack control plane services. templates: openstack: storageRequest: 500M + openstack-cell1: + storageRequest: 500M memcached: enabled: true diff --git a/docs/openstack/mariadb_copy.md b/docs/openstack/mariadb_copy.md index 4d8ce8931..c33da6601 100644 --- a/docs/openstack/mariadb_copy.md +++ b/docs/openstack/mariadb_copy.md @@ -35,14 +35,22 @@ Define the shell variables used in the steps below. The values are just illustrative, use values that are correct for your environment: ``` -PODIFIED_MARIADB_IP=$(oc get -o yaml pod mariadb-openstack | grep podIP: | awk '{ print $2; }') MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified -SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') +PODIFIED_MARIADB_IP=$(oc get -o yaml pod mariadb-openstack | grep podIP: | awk '{ print $2; }') +PODIFIED_CELL1_MARIADB_IP=$(oc get -o yaml pod mariadb-openstack-cell1 | grep podIP: | awk '{ print $2; }') PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d) # Replace with your environment's MariaDB IP: SOURCE_MARIADB_IP=127.17.0.100 +SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }') + +# The CHARACTER_SET and collation should match the source DB +# if the do not then it will break foreign key relationships +# for any tables that are created in the future as part of db sync +CHARACTER_SET=utf8 +COLLATION=utf8_general_ci + ``` ## Pre-checks @@ -61,11 +69,13 @@ SOURCE_MARIADB_IP=127.17.0.100 mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK ``` -* Test connection to podified DB (show databases): +* Test connection to podified DBs (show databases): ``` oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' + oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -h "$PODIFIED_CELL1_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' ``` ## Procedure - data copy @@ -83,31 +93,60 @@ SOURCE_MARIADB_IP=127.17.0.100 ``` podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash < "\$dbname".sql + "\${dbname}" > "\${dbname}".sql done EOF ``` -* Before restoring the databases from .sql files, we need to ensure that the neutron -database name is "neutron" and not "ovs_neutron" by running the command: - - ``` - sed -i '/^CREATE DATABASE/s/ovs_neutron/neutron/g;/^USE/s/ovs_neutron/neutron/g' ovs_neutron.sql - ``` - * Restore the databases from .sql files into the podified MariaDB: ``` - for dbname in cinder glance keystone nova_api nova_cell0 nova ovs_neutron placement; do - echo "Importing $dbname" - oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" < "$dbname.sql" + # db schemas to rename on import + declare -A db_name_map + db_name_map["nova"]="nova_cell1" + db_name_map["ovs_neutron"]="neutron" + + # db servers to import into + declare -A db_server_map + db_server_map["default"]=${PODIFIED_MARIADB_IP} + db_server_map["nova_cell1"]=${PODIFIED_CELL1_MARIADB_IP} + + # db server root password map + declare -A db_server_password_map + db_server_password_map["default"]=${PODIFIED_DB_ROOT_PASSWORD} + db_server_password_map["nova_cell1"]=${PODIFIED_DB_ROOT_PASSWORD} + + all_db_files=$(ls *.sql) + for db_file in ${all_db_files}; do + db_name=$(echo ${db_file} | awk -F'.' '{ print $1; }') + if [[ -v "db_name_map[${db_name}]" ]]; then + echo "renaming ${db_name} to ${db_name_map[${db_name}]}" + db_name=${db_name_map[${db_name}]} + fi + db_server=${db_server_map["default"]} + if [[ -v "db_server_map[${db_name}]" ]]; then + db_server=${db_server_map[${db_name}]} + fi + db_password=${db_server_password_map["default"]} + if [[ -v "db_server_password_map[${db_name}]" ]]; then + db_password=${db_server_password_map[${db_name}]} + fi + echo "creating ${db_name} in ${db_server}" + container_name=$(echo "mariadb-client-${db_name}-create" | sed 's/_/-/g') + oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -h "${db_server}" -uroot "-p${db_password}" << EOF + CREATE DATABASE IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION}; + EOF + echo "importing ${db_name} into ${db_server}" + container_name=$(echo "mariadb-client-${db_name}-restore" | sed 's/_/-/g') + oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -h "${db_server}" -uroot "-p${db_password}" "${db_name}" < "${db_file}" done ``` @@ -117,7 +156,16 @@ database name is "neutron" and not "ovs_neutron" by running the command: ``` oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' + mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ + | grep keystone + # ensure neutron db is renamed from ovs_neutron + oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ + | grep neutron + # ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 + oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ + | grep nova_cell1 ``` * During the pre/post checks the pod `mariadb-client` might have returned a pod security warning diff --git a/tests/config/base/openstack_control_plane.yaml b/tests/config/base/openstack_control_plane.yaml index 7fdec586c..57d832c49 100644 --- a/tests/config/base/openstack_control_plane.yaml +++ b/tests/config/base/openstack_control_plane.yaml @@ -58,6 +58,8 @@ spec: templates: openstack: storageRequest: 500M + openstack-cell1: + storageRequest: 500M memcached: enabled: true diff --git a/tests/config/periodic_ci/container_image_overrides.yaml b/tests/config/periodic_ci/container_image_overrides.yaml index 18c4b9b2a..c5a319189 100644 --- a/tests/config/periodic_ci/container_image_overrides.yaml +++ b/tests/config/periodic_ci/container_image_overrides.yaml @@ -54,6 +54,8 @@ spec: templates: openstack: storageRequest: 500M + openstack-cell1: + storageRequest: 500M memcached: enabled: true diff --git a/tests/roles/backend_services/tasks/main.yaml b/tests/roles/backend_services/tasks/main.yaml index fc516cbbb..1c6a16b07 100644 --- a/tests/roles/backend_services/tasks/main.yaml +++ b/tests/roles/backend_services/tasks/main.yaml @@ -82,6 +82,17 @@ retries: 60 delay: 2 +- name: wait for cell1 mariadb to start up + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc get pod mariadb-openstack-cell1 -o jsonpath='{.status.phase}{"\n"}' | grep Running + register: mariadb_running_result + until: mariadb_running_result is success + retries: 60 + delay: 2 + + - name: Patch rabbitmq resources for lower resource consumption changed_when: false ansible.builtin.shell: | diff --git a/tests/roles/backend_services/templates/container_overrides.j2 b/tests/roles/backend_services/templates/container_overrides.j2 index 54567969d..6e2494867 100644 --- a/tests/roles/backend_services/templates/container_overrides.j2 +++ b/tests/roles/backend_services/templates/container_overrides.j2 @@ -48,6 +48,8 @@ spec: templates: openstack: containerImage: {{ container_registry }}/{{ container_namespace }}/openstack-mariadb:{{ container_tag }} + openstack-cell1: + containerImage: {{ container_registry }}/{{ container_namespace }}/openstack-mariadb:{{ container_tag }} memcached: templates: diff --git a/tests/roles/mariadb_copy/tasks/main.yaml b/tests/roles/mariadb_copy/tasks/main.yaml index 3bde8f374..e7422bea7 100644 --- a/tests/roles/mariadb_copy/tasks/main.yaml +++ b/tests/roles/mariadb_copy/tasks/main.yaml @@ -5,102 +5,50 @@ oc get -o yaml pod mariadb-openstack | grep podIP: | awk '{ print $2; }' register: podified_mariadb_ip_result +- name: get podified cell1 MariaDB IP + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc get -o yaml pod mariadb-openstack-cell1 | grep podIP: | awk '{ print $2; }' + register: podified_cell1_mariadb_ip_result + - name: set MariaDB copy shell vars no_log: "{{ use_no_log }}" ansible.builtin.set_fact: mariadb_copy_shell_vars: | - PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified + PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }} + PODIFIED_CELL1_MARIADB_IP={{podified_cell1_mariadb_ip_result.stdout}} + PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" + # TODO: remove the default(external_...) when CI is transitioned to use 'source_...' SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }} SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}" - PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}" - CONTROLLER1_SSH="{{ controller1_ssh }}" - CONTROLLER2_SSH="{{ controller2_ssh }}" - CONTROLLER3_SSH="{{ controller3_ssh }}" + # The CHARACTER_SET and collation should match the source DB + # if the do not then it will break foreign key relationships + # for any tables that are created in the future as part of db sync + CHARACTER_SET=utf8 + COLLATION=utf8_general_ci - name: MariaDB copy pre checks no_log: "{{ use_no_log }}" - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - {{ mariadb_copy_shell_vars }} - - podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ - mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;' - podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ - mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" - oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' + ansible.builtin.shell: + cmd: "{{ lookup('ansible.builtin.template', 'pre_checks.bash') }}" - name: dump databases no_log: "{{ use_no_log }}" - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - {{ mariadb_copy_shell_vars }} - - mkdir -p {{ mariadb_copy_tmp_dir }} - cd {{ mariadb_copy_tmp_dir }} - - podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash < "\$dbname".sql - done - - EOF - -- name: Replace neutron database name - ansible.builtin.shell: | - cd {{ mariadb_copy_tmp_dir }} - sed -i '/^CREATE DATABASE/s/ovs_neutron/neutron/g;/^USE/s/ovs_neutron/neutron/g' ovs_neutron.sql - -- name: Check name on creation - ansible.builtin.lineinfile: - path: "{{ mariadb_copy_tmp_dir }}/ovs_neutron.sql" - regexp: (^CREATE DATABASE.*`)(neutron)(`.*) - state: absent - check_mode: yes - register: create_database - failed_when: create_database.found == 0 - -- name: Check name on usage - ansible.builtin.lineinfile: - path: "{{ mariadb_copy_tmp_dir }}/ovs_neutron.sql" - regexp: (^USE.*`)(neutron)(`.*) - state: absent - check_mode: yes - register: use_database - failed_when: use_database.found == 0 + ansible.builtin.shell: + cmd: "{{ lookup('ansible.builtin.template', 'dump_dbs.bash') }}" - name: restore databases no_log: "{{ use_no_log }}" - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - {{ mariadb_copy_shell_vars }} - cd {{ mariadb_copy_tmp_dir }} - - for dbname in cinder glance keystone nova_api nova_cell0 nova ovs_neutron placement; do - echo "Restoring $dbname" - oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" < "$dbname.sql" - done - + ansible.builtin.shell: + cmd: "{{ lookup('ansible.builtin.template', 'restore_dbs.bash') }}" + - name: MariaDB post-checks no_log: "{{ use_no_log }}" - ansible.builtin.shell: | - {{ shell_header }} - {{ oc_header }} - {{ mariadb_copy_shell_vars }} - - oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ - mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' \ - | grep keystone + ansible.builtin.shell: + cmd: "{{ lookup('ansible.builtin.template', 'post_checks.bash') }}" + \ No newline at end of file diff --git a/tests/roles/mariadb_copy/templates/dump_dbs.bash b/tests/roles/mariadb_copy/templates/dump_dbs.bash new file mode 100644 index 000000000..3c73ed20d --- /dev/null +++ b/tests/roles/mariadb_copy/templates/dump_dbs.bash @@ -0,0 +1,19 @@ +#!/bin/bash +{{ shell_header }} +{{ oc_header }} +{{ mariadb_copy_shell_vars }} + +mkdir -p {{ mariadb_copy_tmp_dir }} +cd {{ mariadb_copy_tmp_dir }} + +podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash < "\${dbname}".sql +done + +EOF \ No newline at end of file diff --git a/tests/roles/mariadb_copy/templates/post_checks.bash b/tests/roles/mariadb_copy/templates/post_checks.bash new file mode 100644 index 000000000..dbe92c043 --- /dev/null +++ b/tests/roles/mariadb_copy/templates/post_checks.bash @@ -0,0 +1,14 @@ +{{ shell_header }} +{{ oc_header }} +{{ mariadb_copy_shell_vars }} +oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ +mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ + | grep keystone +# ensure neutron db is renamed from ovs_neutron +oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ +mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ + | grep neutron +# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1 +oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ +mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \ + | grep nova_cell1 \ No newline at end of file diff --git a/tests/roles/mariadb_copy/templates/pre_checks.bash b/tests/roles/mariadb_copy/templates/pre_checks.bash new file mode 100644 index 000000000..9f61b135c --- /dev/null +++ b/tests/roles/mariadb_copy/templates/pre_checks.bash @@ -0,0 +1,16 @@ +#!/bin/bash +{{ shell_header }} +{{ oc_header }} +{{ mariadb_copy_shell_vars }} + +# Test connection to the original DB (show databases) +podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ + mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;' +# Run mysqlcheck on the original DB to look for things that are not OK +podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \ + mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" +# Test connection to podified DBs (show databases) +oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' +oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \ + mysql -h "$PODIFIED_CELL1_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' \ No newline at end of file diff --git a/tests/roles/mariadb_copy/templates/restore_dbs.bash b/tests/roles/mariadb_copy/templates/restore_dbs.bash new file mode 100755 index 000000000..5cea23223 --- /dev/null +++ b/tests/roles/mariadb_copy/templates/restore_dbs.bash @@ -0,0 +1,47 @@ +#!/bin/bash +{{ shell_header }} +{{ oc_header }} +{{ mariadb_copy_shell_vars }} +cd {{ mariadb_copy_tmp_dir }} + +# db schemas to rename on import +declare -A db_name_map +db_name_map["nova"]="nova_cell1" +db_name_map["ovs_neutron"]="neutron" + +# db servers to import into +declare -A db_server_map +db_server_map["default"]=${PODIFIED_MARIADB_IP} +db_server_map["nova_cell1"]=${PODIFIED_CELL1_MARIADB_IP} + +# db server root password map +declare -A db_server_password_map +db_server_password_map["default"]=${PODIFIED_DB_ROOT_PASSWORD} +db_server_password_map["nova_cell1"]=${PODIFIED_DB_ROOT_PASSWORD} + +all_db_files=$(ls *.sql) +for db_file in ${all_db_files}; do + db_name=$(echo ${db_file} | awk -F'.' '{ print $1; }') + if [[ -v "db_name_map[${db_name}]" ]]; then + echo "renaming ${db_name} to ${db_name_map[${db_name}]}" + db_name=${db_name_map[${db_name}]} + fi + db_server=${db_server_map["default"]} + if [[ -v "db_server_map[${db_name}]" ]]; then + db_server=${db_server_map[${db_name}]} + fi + db_password=${db_server_password_map["default"]} + if [[ -v "db_server_password_map[${db_name}]" ]]; then + db_password=${db_server_password_map[${db_name}]} + fi + echo "creating ${db_name} in ${db_server}" + container_name=$(echo "mariadb-client-${db_name}-create" | sed 's/_/-/g') + oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -h "${db_server}" -uroot "-p${db_password}" << EOF +CREATE DATABASE IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION}; +EOF + echo "importing ${db_name} into ${db_server}" + container_name=$(echo "mariadb-client-${db_name}-restore" | sed 's/_/-/g') + oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \ + mysql -h "${db_server}" -uroot "-p${db_password}" "${db_name}" < "${db_file}" +done \ No newline at end of file