Wait for DB writes to propagate (causality checks)

Because we deploy the database in multi-master mode we can have cases
where reading data from the DB fails after the data was written to
another galera master node.

We saw this happening during Flavor creation where nova first creates
the flavor in the DB then the notification sending code tries to read
back the flavor projects from the DB and the query fails as the flavor
is not found.

In this patch we change the default value on the DB engine to wait for
previous write to sync before read.

This will have a performance impact.

We use `mysql_wsrep_sync_wait` from oslo.db [1] setting it to 1 as per
the documented values of this parameter in the DBMS [2][3].

[1]: https://opendev.org/openstack/oslo.db/commit/009d23df45969036c70e4cf59eb4019aaace9a55
[2]: https://mariadb.com/docs/server/ref/mdb/system-variables/wsrep_sync_wait/
[3]: https://galeracluster.com/library/documentation/mysql-wsrep-options.html

Fixes: https://issues.redhat.com/browse/OSPRH-7198

templates/nova.conf

@@ -211,12 +211,22 @@ cpu_power_management=true
 {{if (index . "cell_db_address")}}
 [database]
 connection = mysql+pymysql://{{ .cell_db_user }}:{{ .cell_db_password}}@{{ .cell_db_address }}/{{ .cell_db_name }}?read_default_file=/etc/my.cnf
+
+# Wait for writes to complete when doing a read.
+# Relevant for multi-master galera deployments
+# https://mariadb.com/docs/server/ref/mdb/system-variables/wsrep_sync_wait/
+mysql_wsrep_sync_wait = 1
 {{end}}
 
 
 {{if (index . "api_db_address")}}
 [api_database]
 connection = mysql+pymysql://{{ .api_db_user }}:{{ .api_db_password }}@{{ .api_db_address }}/{{ .api_db_name }}?read_default_file=/etc/my.cnf
+
+# Wait for writes to complete when doing a read.
+# Relevant for multi-master galera deployments
+# https://mariadb.com/docs/server/ref/mdb/system-variables/wsrep_sync_wait/
+mysql_wsrep_sync_wait = 1
 {{end}}
 
 [keystone_authtoken]

test/functional/nova_controller_test.go

@@ -319,14 +319,16 @@ var _ = Describe("Nova controller", func() {
 					cell0Account.Spec.UserName, cell0Secret.Data[mariadbv1.DatabasePasswordSelector],
 					novaNames.Namespace)),
 			)
-
 			apiAccount := mariadb.GetMariaDBAccount(novaNames.APIMariaDBDatabaseAccount)
 			apiSecret := th.GetSecret(types.NamespacedName{Name: apiAccount.Spec.Secret, Namespace: novaNames.APIMariaDBDatabaseAccount.Namespace})
 
 			Expect(configData).To(
 				ContainSubstring(fmt.Sprintf("[api_database]\nconnection = mysql+pymysql://%s:%s@hostname-for-openstack.%s.svc/nova_api?read_default_file=/etc/my.cnf",
 					apiAccount.Spec.UserName, apiSecret.Data[mariadbv1.DatabasePasswordSelector], novaNames.Namespace)),
 			)
+
+			Expect(configData).To(ContainSubstring("mysql_wsrep_sync_wait = 1"))
+
 			// NOTE(gibi): cell mapping for cell0 should not have transport_url
 			// configured. As the nova-manage command used to create the mapping
 			// uses the transport_url from the nova.conf provided to the job