Merge pull request #102 from SeanMooney/nova-db-split

extend mariadb_copy to split out nova cell1 into a seperate db
openstack-k8s-operators · Sep 13, 2023 · 38336ce · 38336ce
2 parents 9cf5bc5 + 7ace228
commit 38336ce
Show file tree

Hide file tree

Showing 11 changed files with 208 additions and 97 deletions.
diff --git a/docs/openstack/backend_services_deployment.md b/docs/openstack/backend_services_deployment.md
@@ -167,6 +167,8 @@ podified OpenStack control plane services.
       templates:
         openstack:
           storageRequest: 500M
+        openstack-cell1:
+          storageRequest: 500M
 
     memcached:
       enabled: true

diff --git a/docs/openstack/mariadb_copy.md b/docs/openstack/mariadb_copy.md
@@ -35,14 +35,22 @@ Define the shell variables used in the steps below. The values are
 just illustrative, use values that are correct for your environment:
 
 ```
-PODIFIED_MARIADB_IP=$(oc get -o yaml pod mariadb-openstack | grep podIP: | awk '{ print $2; }')
 MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified
 
-SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }')
+PODIFIED_MARIADB_IP=$(oc get -o yaml pod mariadb-openstack | grep podIP: | awk '{ print $2; }')
+PODIFIED_CELL1_MARIADB_IP=$(oc get -o yaml pod  mariadb-openstack-cell1  | grep podIP: | awk '{ print $2; }')
 PODIFIED_DB_ROOT_PASSWORD=$(oc get -o json secret/osp-secret | jq -r .data.DbRootPassword | base64 -d)
 
 # Replace with your environment's MariaDB IP:
 SOURCE_MARIADB_IP=127.17.0.100
+SOURCE_DB_ROOT_PASSWORD=$(cat ~/tripleo-standalone-passwords.yaml | grep ' MysqlRootPassword:' | awk -F ': ' '{ print $2; }')
+
+# The CHARACTER_SET and collation should match the source DB
+# if the do not then it will break foreign key relationships
+# for any tables that are created in the future as part of db sync
+CHARACTER_SET=utf8
+COLLATION=utf8_general_ci
+
 ```
 
 ## Pre-checks
@@ -61,11 +69,13 @@ SOURCE_MARIADB_IP=127.17.0.100
       mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" | grep -v OK
   ```
 
-* Test connection to podified DB (show databases):
+* Test connection to podified DBs (show databases):
 
   ```
   oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
       mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
+  oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+      mysql -h "$PODIFIED_CELL1_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
   ```
 
 ## Procedure - data copy
@@ -83,31 +93,60 @@ SOURCE_MARIADB_IP=127.17.0.100
   ```
   podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash <<EOF
 
-  mysql -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" -N -e 'show databases' | while read dbname; do
-      echo "Exporting \$dbname"
+ # Note we do not want to dump the information and performance schema tables so we filter them
+  mysql -h ${SOURCE_MARIADB_IP} -u root "-p${SOURCE_DB_ROOT_PASSWORD}" -N -e 'show databases' | grep -E -v 'schema|mysql' | while read dbname; do
+      echo "Dumping \${dbname}"
       mysqldump -h $SOURCE_MARIADB_IP -uroot "-p$SOURCE_DB_ROOT_PASSWORD" \
           --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \
-          --databases "\$dbname" \
-          > "\$dbname".sql
+          "\${dbname}" > "\${dbname}".sql
   done
 
   EOF
   ```
 
-* Before restoring the databases from .sql files, we need to ensure that the neutron
-database name is "neutron" and not "ovs_neutron" by running the command:
-
-  ```
-  sed -i '/^CREATE DATABASE/s/ovs_neutron/neutron/g;/^USE/s/ovs_neutron/neutron/g' ovs_neutron.sql
-  ```
-
 * Restore the databases from .sql files into the podified MariaDB:
 
   ```
-  for dbname in cinder glance keystone nova_api nova_cell0 nova ovs_neutron placement; do
-      echo "Importing $dbname"
-      oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
-         mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" < "$dbname.sql"
+  # db schemas to rename on import
+  declare -A db_name_map
+  db_name_map["nova"]="nova_cell1"
+  db_name_map["ovs_neutron"]="neutron"
+
+  # db servers to import into
+  declare -A db_server_map
+  db_server_map["default"]=${PODIFIED_MARIADB_IP}
+  db_server_map["nova_cell1"]=${PODIFIED_CELL1_MARIADB_IP}
+
+  # db server root password map
+  declare -A db_server_password_map
+  db_server_password_map["default"]=${PODIFIED_DB_ROOT_PASSWORD}
+  db_server_password_map["nova_cell1"]=${PODIFIED_DB_ROOT_PASSWORD}
+
+  all_db_files=$(ls *.sql)
+  for db_file in ${all_db_files}; do
+      db_name=$(echo ${db_file} | awk -F'.' '{ print $1; }')
+      if [[ -v "db_name_map[${db_name}]" ]]; then
+          echo "renaming ${db_name} to ${db_name_map[${db_name}]}"
+          db_name=${db_name_map[${db_name}]}
+      fi
+      db_server=${db_server_map["default"]}
+      if [[ -v "db_server_map[${db_name}]" ]]; then
+          db_server=${db_server_map[${db_name}]}
+      fi
+      db_password=${db_server_password_map["default"]}
+      if [[ -v "db_server_password_map[${db_name}]" ]]; then
+          db_password=${db_server_password_map[${db_name}]}
+      fi
+      echo "creating ${db_name} in ${db_server}"
+      container_name=$(echo "mariadb-client-${db_name}-create" | sed 's/_/-/g')
+      oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
+          mysql -h "${db_server}" -uroot "-p${db_password}" << EOF
+  CREATE DATABASE IF NOT EXISTS ${db_name} DEFAULT CHARACTER SET ${CHARACTER_SET} DEFAULT COLLATE ${COLLATION};
+  EOF
+      echo "importing ${db_name} into ${db_server}"
+      container_name=$(echo "mariadb-client-${db_name}-restore" | sed 's/_/-/g')
+      oc run ${container_name} --image ${MARIADB_IMAGE} -i --rm --restart=Never -- \
+          mysql -h "${db_server}" -uroot "-p${db_password}" "${db_name}" < "${db_file}"
   done
   ```
 
@@ -117,7 +156,16 @@ database name is "neutron" and not "ovs_neutron" by running the command:
 
   ```
   oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
-     mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
+  mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \
+      | grep keystone
+  # ensure neutron db is renamed from ovs_neutron
+  oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+  mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \
+      | grep neutron
+  # ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1
+  oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+  mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \
+      | grep nova_cell1
   ```
 
 * During the pre/post checks the pod `mariadb-client` might have returned a pod security warning

diff --git a/tests/config/base/openstack_control_plane.yaml b/tests/config/base/openstack_control_plane.yaml
@@ -58,6 +58,8 @@ spec:
     templates:
       openstack:
         storageRequest: 500M
+      openstack-cell1:
+        storageRequest: 500M
 
   memcached:
     enabled: true

diff --git a/tests/config/periodic_ci/container_image_overrides.yaml b/tests/config/periodic_ci/container_image_overrides.yaml
@@ -54,6 +54,8 @@ spec:
     templates:
       openstack:
         storageRequest: 500M
+      openstack-cell1:
+        storageRequest: 500M
 
   memcached:
     enabled: true

diff --git a/tests/roles/backend_services/tasks/main.yaml b/tests/roles/backend_services/tasks/main.yaml
@@ -82,6 +82,17 @@
   retries: 60
   delay: 2
 
+- name: wait for cell1 mariadb to start up
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc get pod mariadb-openstack-cell1 -o jsonpath='{.status.phase}{"\n"}' | grep Running
+  register: mariadb_running_result
+  until: mariadb_running_result is success
+  retries: 60
+  delay: 2
+
+
 - name: Patch rabbitmq resources for lower resource consumption
   changed_when: false
   ansible.builtin.shell: |

diff --git a/tests/roles/backend_services/templates/container_overrides.j2 b/tests/roles/backend_services/templates/container_overrides.j2
@@ -48,6 +48,8 @@ spec:
     templates:
       openstack:
         containerImage: {{ container_registry }}/{{ container_namespace }}/openstack-mariadb:{{ container_tag }}
+      openstack-cell1:
+        containerImage: {{ container_registry }}/{{ container_namespace }}/openstack-mariadb:{{ container_tag }}
 
   memcached:
     templates:

diff --git a/tests/roles/mariadb_copy/tasks/main.yaml b/tests/roles/mariadb_copy/tasks/main.yaml
@@ -5,102 +5,50 @@
     oc get -o yaml pod mariadb-openstack | grep podIP: | awk '{ print $2; }'
   register: podified_mariadb_ip_result
 
+- name: get podified cell1 MariaDB IP
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+    oc get -o yaml pod mariadb-openstack-cell1 | grep podIP: | awk '{ print $2; }'
+  register: podified_cell1_mariadb_ip_result
+
 - name: set MariaDB copy shell vars
   no_log: "{{ use_no_log }}"
   ansible.builtin.set_fact:
     mariadb_copy_shell_vars: |
-      PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }}
       MARIADB_IMAGE=quay.io/podified-antelope-centos9/openstack-mariadb:current-podified
 
+      PODIFIED_MARIADB_IP={{ podified_mariadb_ip_result.stdout }}
+      PODIFIED_CELL1_MARIADB_IP={{podified_cell1_mariadb_ip_result.stdout}}
+      PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}"
+
       # TODO: remove the default(external_...) when CI is transitioned to use 'source_...'
       SOURCE_MARIADB_IP={{ source_mariadb_ip|default(external_mariadb_ip) }}
       SOURCE_DB_ROOT_PASSWORD="{{ source_db_root_password|default(external_db_root_password) }}"
-      PODIFIED_DB_ROOT_PASSWORD="{{ podified_db_root_password }}"
 
-      CONTROLLER1_SSH="{{ controller1_ssh }}"
-      CONTROLLER2_SSH="{{ controller2_ssh }}"
-      CONTROLLER3_SSH="{{ controller3_ssh }}"
+      # The CHARACTER_SET and collation should match the source DB
+      # if the do not then it will break foreign key relationships
+      # for any tables that are created in the future as part of db sync
+      CHARACTER_SET=utf8
+      COLLATION=utf8_general_ci
 
 - name: MariaDB copy pre checks
   no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ mariadb_copy_shell_vars }}
-
-    podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \
-        mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;'
-    podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \
-        mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD"
-    oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
-        mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
+  ansible.builtin.shell:
+    cmd: "{{ lookup('ansible.builtin.template', 'pre_checks.bash') }}"
 
 - name: dump databases
   no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ mariadb_copy_shell_vars }}
-
-    mkdir -p {{ mariadb_copy_tmp_dir }}
-    cd {{ mariadb_copy_tmp_dir }}
-
-    podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash <<EOF
-
-    mysql -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD" -N -e 'show databases' | while read dbname; do
-        echo "Dumping \$dbname"
-        mysqldump -h $SOURCE_MARIADB_IP -uroot "-p$SOURCE_DB_ROOT_PASSWORD" \
-            --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \
-            --databases "\$dbname" \
-            > "\$dbname".sql
-    done
-
-    EOF
-
-- name:  Replace neutron database name
-  ansible.builtin.shell: |
-    cd {{ mariadb_copy_tmp_dir }}
-    sed -i '/^CREATE DATABASE/s/ovs_neutron/neutron/g;/^USE/s/ovs_neutron/neutron/g' ovs_neutron.sql
-
-- name: Check name on creation
-  ansible.builtin.lineinfile:
-    path: "{{ mariadb_copy_tmp_dir }}/ovs_neutron.sql"
-    regexp: (^CREATE DATABASE.*`)(neutron)(`.*)
-    state: absent
-  check_mode: yes
-  register: create_database
-  failed_when: create_database.found == 0
-
-- name: Check name on usage
-  ansible.builtin.lineinfile:
-    path: "{{ mariadb_copy_tmp_dir }}/ovs_neutron.sql"
-    regexp: (^USE.*`)(neutron)(`.*)
-    state: absent
-  check_mode: yes
-  register: use_database
-  failed_when: use_database.found == 0
+  ansible.builtin.shell:
+    cmd: "{{ lookup('ansible.builtin.template', 'dump_dbs.bash') }}"
 
 - name: restore databases
   no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ mariadb_copy_shell_vars }}
-    cd {{ mariadb_copy_tmp_dir }}
-
-    for dbname in cinder glance keystone nova_api nova_cell0 nova ovs_neutron placement; do
-        echo "Restoring $dbname"
-        oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
-           mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" < "$dbname.sql"
-    done
-
+  ansible.builtin.shell:
+    cmd: "{{ lookup('ansible.builtin.template', 'restore_dbs.bash') }}"
+
 - name: MariaDB post-checks
   no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ mariadb_copy_shell_vars }}
-
-    oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
-        mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;' \
-        | grep keystone
+  ansible.builtin.shell:
+    cmd: "{{ lookup('ansible.builtin.template', 'post_checks.bash') }}"
+
diff --git a/tests/roles/mariadb_copy/templates/dump_dbs.bash b/tests/roles/mariadb_copy/templates/dump_dbs.bash
@@ -0,0 +1,19 @@
+#!/bin/bash
+{{ shell_header }}
+{{ oc_header }}
+{{ mariadb_copy_shell_vars }}
+
+mkdir -p {{ mariadb_copy_tmp_dir }}
+cd {{ mariadb_copy_tmp_dir }}
+
+podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE bash <<EOF
+
+# Note we do not want to dump the information and performance schema tables so we filter them
+mysql -h ${SOURCE_MARIADB_IP} -u root "-p${SOURCE_DB_ROOT_PASSWORD}" -N -e 'show databases' | grep -E -v 'schema|mysql' | while read dbname; do
+    echo "Dumping \${dbname}"
+    mysqldump -h $SOURCE_MARIADB_IP -uroot "-p$SOURCE_DB_ROOT_PASSWORD" \
+        --single-transaction --complete-insert --skip-lock-tables --lock-tables=0 \
+        "\${dbname}" > "\${dbname}".sql
+done
+
+EOF
diff --git a/tests/roles/mariadb_copy/templates/post_checks.bash b/tests/roles/mariadb_copy/templates/post_checks.bash
@@ -0,0 +1,14 @@
+{{ shell_header }}
+{{ oc_header }}
+{{ mariadb_copy_shell_vars }}
+oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \
+    | grep keystone
+# ensure neutron db is renamed from ovs_neutron
+oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+mysql -h "${PODIFIED_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \
+    | grep neutron
+# ensure nova cell1 db is extracted to a separate db server and renamed from nova to nova_cell1
+oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+mysql -h "${PODIFIED_CELL1_MARIADB_IP}" -uroot "-p${PODIFIED_DB_ROOT_PASSWORD}" -e 'SHOW databases;' \
+    | grep nova_cell1
diff --git a/tests/roles/mariadb_copy/templates/pre_checks.bash b/tests/roles/mariadb_copy/templates/pre_checks.bash
@@ -0,0 +1,16 @@
+#!/bin/bash
+{{ shell_header }}
+{{ oc_header }}
+{{ mariadb_copy_shell_vars }}
+
+# Test connection to the original DB (show databases)
+podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \
+    mysql -h "$SOURCE_MARIADB_IP" -uroot "-p$SOURCE_DB_ROOT_PASSWORD" -e 'SHOW databases;'
+# Run mysqlcheck on the original DB to look for things that are not OK
+podman run -i --rm --userns=keep-id -u $UID -v $PWD:$PWD:z,rw -w $PWD $MARIADB_IMAGE \
+    mysqlcheck --all-databases -h $SOURCE_MARIADB_IP -u root "-p$SOURCE_DB_ROOT_PASSWORD"
+# Test connection to podified DBs (show databases)
+oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+    mysql -h "$PODIFIED_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'
+oc run mariadb-client --image $MARIADB_IMAGE -i --rm --restart=Never -- \
+    mysql -h "$PODIFIED_CELL1_MARIADB_IP" -uroot "-p$PODIFIED_DB_ROOT_PASSWORD" -e 'SHOW databases;'