WIP: Added a functional test

test/functional/nova_multicell_test.go

@@ -56,6 +56,11 @@ var _ = Describe("Nova multi cell", func() {
 		DeferCleanup(k8sClient.Delete, ctx, cell2Account)
 		DeferCleanup(k8sClient.Delete, ctx, cell2Secret)
 
+		cell3Account, cell3Secret := mariadb.CreateMariaDBAccountAndSecret(
+			cell3.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{})
+		DeferCleanup(k8sClient.Delete, ctx, cell3Account)
+		DeferCleanup(k8sClient.Delete, ctx, cell3Secret)
+
 	})
 	When("Nova CR instance is created with 3 cells", func() {
 		BeforeEach(func() {
@@ -64,12 +69,14 @@ var _ = Describe("Nova multi cell", func() {
 			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell0))
 			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell1))
 			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell2))
+			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell3))
 
 			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
 			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(novaNames.APIMariaDBDatabaseName.Namespace, novaNames.APIMariaDBDatabaseName.Name, serviceSpec))
 			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell0.MariaDBDatabaseName.Namespace, cell0.MariaDBDatabaseName.Name, serviceSpec))
 			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell1.MariaDBDatabaseName.Namespace, cell1.MariaDBDatabaseName.Name, serviceSpec))
 			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell2.MariaDBDatabaseName.Namespace, cell2.MariaDBDatabaseName.Name, serviceSpec))
+			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell3.MariaDBDatabaseName.Namespace, cell3.MariaDBDatabaseName.Name, serviceSpec))
 
 			spec := GetDefaultNovaSpec()
 			cell0Template := GetDefaultNovaCellTemplate()
@@ -98,11 +105,22 @@ var _ = Describe("Nova multi cell", func() {
 				"database": "NovaCell2DatabasePassword",
 			}
 
+			cell3Template := GetDefaultNovaCellTemplate()
+			cell3Template["cellDatabaseInstance"] = cell3.MariaDBDatabaseName.Name
+			cell3Template["cellDatabaseAccount"] = cell3.MariaDBAccountName.Name
+			cell3Template["cellMessageBusInstance"] = cell3.TransportURLName.Name
+			cell3Template["hasAPIAccess"] = false
+			cell3Template["passwordSelectors"] = map[string]interface{}{
+				"database": "NovaCell3DatabasePassword",
+			}
+
 			spec["cellTemplates"] = map[string]interface{}{
 				"cell0": cell0Template,
 				"cell1": cell1Template,
 				"cell2": cell2Template,
+				"cell3": cell3Template,
 			}
+
 			spec["apiDatabaseInstance"] = novaNames.APIMariaDBDatabaseName.Name
 			spec["apiMessageBusInstance"] = cell0.TransportURLName.Name
 
@@ -131,6 +149,44 @@ var _ = Describe("Nova multi cell", func() {
 			keystone.SimulateKeystoneServiceReady(novaNames.KeystoneServiceName)
 		})
 
+		It("deletes cell3 and verifies error for cell2 because its DB deleted already", func() {
+			// Create/Simulate DB for cell2 and cell3
+			mariadb.SimulateMariaDBDatabaseCompleted(cell2.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(cell2.MariaDBAccountName)
+			infra.SimulateTransportURLReady(cell2.TransportURLName)
+
+			mariadb.SimulateMariaDBDatabaseCompleted(cell3.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(cell3.MariaDBAccountName)
+			infra.SimulateTransportURLReady(cell3.TransportURLName)
+
+			// manually delete DB for cell2, to reproduce the error in cell2 deletion
+			Expect(k8sClient.Delete(ctx, mariadb.GetMariaDBDatabase(cell2.MariaDBDatabaseName))).To(Succeed())
+
+			// delete cell2 and then cell3,
+			// the overall call should succeed
+			Eventually(func(g Gomega) {
+				nova := GetNova(novaNames.NovaName)
+				delete(nova.Spec.CellTemplates, "cell2")
+				delete(nova.Spec.CellTemplates, "cell3")
+				g.Expect(k8sClient.Update(ctx, nova)).To(Succeed())
+			}, timeout, interval).Should(Succeed())
+
+			Eventually(func(g Gomega) {
+				nova := GetNova(novaNames.NovaName)
+				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell3.CellCRName.Name))
+			}, timeout, interval).Should(Succeed())
+
+			NovaCellNotExists(cell3.CellCRName)
+
+			// cell2 deletion should have failed
+			Eventually(func(g Gomega) {
+				mappingJob := th.GetJob(cell2.CellDeleteJobName)
+				newJobInputHash := GetEnvVarValue(
+					mappingJob.Spec.Template.Spec.Containers[0].Env, "INPUT_HASH", "")
+				g.Expect(newJobInputHash).To(BeEmpty())
+			}, timeout, interval).Should(Succeed())
+		})
+
 		It("creates cell0 NovaCell", func() {
 			mariadb.SimulateMariaDBDatabaseCompleted(novaNames.APIMariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(novaNames.APIMariaDBDatabaseAccount)

test/functional/suite_test.go

@@ -83,6 +83,7 @@ var (
 	cell0     CellNames
 	cell1     CellNames
 	cell2     CellNames
+	cell3     CellNames
 )
 
 func TestAPIs(t *testing.T) {