From a5b0bf43b21f495db52eac58c44e52261744605c Mon Sep 17 00:00:00 2001 From: Balazs Gibizer Date: Wed, 29 May 2024 17:47:28 +0200 Subject: [PATCH] Wait for DB writes to propagate (causality checks) Because we deploy the database in multi-master mode we can have cases where reading data from the DB fails after the data was written to another galera master node. We only saw issues happening in nova but we need strict consistency during writing resource allocations to placement. In this patch we change the default value on the DB engine to wait for previous write to sync before read. This will have a performance impact. We use `mysql_wsrep_sync_wait` from oslo.db [1] setting it to 1 as per the documented values of this parameter in the DBMS [2][3]. [1]: https://opendev.org/openstack/oslo.db/commit/009d23df45969036c70e4cf59eb4019aaace9a55 [2]: https://mariadb.com/docs/server/ref/mdb/system-variables/wsrep_sync_wait/ [3]: https://galeracluster.com/library/documentation/mysql-wsrep-options.html Related: https://issues.redhat.com/browse/OSPRH-7198 --- templates/placementapi/config/placement.conf | 5 +++++ tests/functional/placementapi_controller_test.go | 1 + 2 files changed, 6 insertions(+) diff --git a/templates/placementapi/config/placement.conf b/templates/placementapi/config/placement.conf index 2cdbd2f8..d0323428 100644 --- a/templates/placementapi/config/placement.conf +++ b/templates/placementapi/config/placement.conf @@ -11,6 +11,11 @@ debug = true [placement_database] connection = {{ .DatabaseConnection }} +# Wait for writes to complete when doing a read. +# Relevant for multi-master galera deployments +# https://mariadb.com/docs/server/ref/mdb/system-variables/wsrep_sync_wait/ +mysql_wsrep_sync_wait = 1 + [api] auth_strategy = keystone diff --git a/tests/functional/placementapi_controller_test.go b/tests/functional/placementapi_controller_test.go index 4d2040e4..eaf2167f 100644 --- a/tests/functional/placementapi_controller_test.go +++ b/tests/functional/placementapi_controller_test.go @@ -333,6 +333,7 @@ var _ = Describe("PlacementAPI controller", func() { Expect(string(conf)).Should( ContainSubstring(fmt.Sprintf("connection = mysql+pymysql://%s:%s@hostname-for-openstack.%s.svc/placement?read_default_file=/etc/my.cnf", mariadbAccount.Spec.UserName, mariadbSecret.Data[mariadbv1.DatabasePasswordSelector], namespace))) + Expect(string(conf)).Should(ContainSubstring("mysql_wsrep_sync_wait = 1")) custom := cm.Data["custom.conf"] Expect(custom).Should(ContainSubstring("foo = bar"))