From 92daefb8e32282735b148dc4d711a7ce69319c5b Mon Sep 17 00:00:00 2001 From: Nir Magnezi Date: Mon, 24 May 2021 15:46:27 +0300 Subject: [PATCH] MGMT-4261 Agent CR and ACI cleanup when deleting CD (#1606) clusterdeployments_controller: When a CD CR gets deleted, the controller will place a finalizer to trigger a pre-deletion cleanup. The cleanup includes ACI deletion, which will place a finalizer on ACI. ACI finalizer includes backend-cluster deregister and deletion of the cluster agent resources. As for subsystem tests: 1. Tests will now assert for agent deletion per ACI removal. 2. Tests will now assert for agent deletion and ACI per CD removal. 3. Each test will now use a unique CD and ACI CR name - to provide better test isolation. This was also done because deletion via finalizers takes an additional reconcile to be fully deleted. 4. Per 2, and to avoid any cleanup bugs, the cleanup done for subsystem tests (AfterEach) is now followed up by verifyCleanUP P.S. Note that this PR does not cover host deregister upon Agent CR deletion, which was added in https://github.com/openshift/assisted-service/pull/1642 --- config/rbac/role.yaml | 6 + ...ervice-operator.clusterserviceversion.yaml | 6 + .../clusterdeployments_controller.go | 185 ++++++++- .../clusterdeployments_controller_test.go | 83 ++-- subsystem/kubeapi_test.go | 368 ++++++++++++------ 5 files changed, 486 insertions(+), 162 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6440805e784..c1f8f0659f9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -183,6 +183,12 @@ rules: - patch - update - watch +- apiGroups: + - extensions.hive.openshift.io + resources: + - agentclusterinstalls/finalizers + verbs: + - update - apiGroups: - extensions.hive.openshift.io resources: diff --git a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml index e74550c7c8f..069a1f321e6 100644 --- a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml @@ -347,6 +347,12 @@ spec: - patch - update - watch + - apiGroups: + - extensions.hive.openshift.io + resources: + - agentclusterinstalls/finalizers + verbs: + - update - apiGroups: - extensions.hive.openshift.io resources: diff --git a/internal/controller/controllers/clusterdeployments_controller.go b/internal/controller/controllers/clusterdeployments_controller.go index f301c4b0588..bfddc6a13d5 100644 --- a/internal/controller/controllers/clusterdeployments_controller.go +++ b/internal/controller/controllers/clusterdeployments_controller.go @@ -33,7 +33,7 @@ import ( "github.com/openshift/assisted-service/internal/cluster" "github.com/openshift/assisted-service/internal/common" hiveext "github.com/openshift/assisted-service/internal/controller/api/hiveextension/v1beta1" - "github.com/openshift/assisted-service/internal/controller/api/v1beta1" + aiv1beta1 "github.com/openshift/assisted-service/internal/controller/api/v1beta1" "github.com/openshift/assisted-service/internal/gencrypto" "github.com/openshift/assisted-service/internal/host" "github.com/openshift/assisted-service/internal/manifests" @@ -44,6 +44,7 @@ import ( hivev1 "github.com/openshift/hive/apis/hive/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/thoas/go-funk" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,7 +63,9 @@ import ( const ( adminPasswordSecretStringTemplate = "%s-admin-password" adminKubeConfigStringTemplate = "%s-admin-kubeconfig" - InstallConfigOverrides = v1beta1.Group + "/install-config-overrides" + InstallConfigOverrides = aiv1beta1.Group + "/install-config-overrides" + ClusterDeploymentFinalizerName = "clusterdeployments." + aiv1beta1.Group + "/ai-deprovision" + AgentClusterInstallFinalizerName = "agentclusterinstall." + aiv1beta1.Group + "/ai-deprovision" ) const HighAvailabilityModeNone = "None" @@ -90,18 +93,17 @@ type ClusterDeploymentsReconciler struct { // +kubebuilder:rbac:groups=hive.openshift.io,resources=clusterimagesets,verbs=get;list;watch // +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls/finalizers,verbs=update func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log.Infof("Reconcile has been called for ClusterDeployment name=%s namespace=%s", req.Name, req.Namespace) clusterDeployment := &hivev1.ClusterDeployment{} - err := r.Get(ctx, req.NamespacedName, clusterDeployment) - if err != nil { - if k8serrors.IsNotFound(err) { - return r.deregisterClusterIfNeeded(ctx, req.NamespacedName) - } - r.Log.WithError(err).Errorf("Failed to get resource %s", req.NamespacedName) - return ctrl.Result{Requeue: true}, nil + clusterInstallDeleted := false + + if err := r.Get(ctx, req.NamespacedName, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to get ClusterDeployment name=%s namespace=%s", req.Name, req.Namespace) + return ctrl.Result{}, client.IgnoreNotFound(err) } // ignore unsupported platforms @@ -116,7 +118,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R } aciName := clusterDeployment.Spec.ClusterInstallRef.Name - err = r.Get(ctx, + err := r.Get(ctx, types.NamespacedName{ Namespace: clusterDeployment.Namespace, Name: aciName, @@ -124,11 +126,29 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R clusterInstall) if err != nil { if k8serrors.IsNotFound(err) { + // mark that clusterInstall was already deleted so we skip it if needed. + clusterInstallDeleted = true r.Log.WithField("AgentClusterInstall", aciName).Infof("AgentClusterInstall does not exist for ClusterDeployment %s", clusterDeployment.Name) - return ctrl.Result{}, nil + if clusterDeployment.ObjectMeta.DeletionTimestamp.IsZero() { + // we have no agentClusterInstall and clusterDeployment is not being deleted. stop reconciliation. + return ctrl.Result{}, nil + } + } else { + r.Log.WithError(err).Errorf("failed to get AgentClusterInstall name=%s namespace=%s", aciName, clusterDeployment.Namespace) + return ctrl.Result{Requeue: true}, err + } + } + + if !clusterInstallDeleted { + aciReply, aciErr := r.agentClusterInstallFinalizer(ctx, req, clusterInstall) + if aciReply != nil { + return *aciReply, aciErr } - r.Log.WithError(err).Errorf("Failed to get AgentClusterInstall %s", aciName) - return ctrl.Result{Requeue: true}, err + } + + cdReplay, cdErr := r.clusterDeploymentFinalizer(ctx, clusterDeployment) + if cdReplay != nil { + return *cdReplay, cdErr } err = r.ensureOwnerRef(ctx, clusterDeployment, clusterInstall) @@ -144,7 +164,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R if !r.isSNO(clusterInstall) { return r.createNewDay2Cluster(ctx, req.NamespacedName, clusterDeployment, clusterInstall) } - // cluster is installed and SNO. Clear EventsURL + // cluster is installed and SNO. Clear EventsURL clusterInstall.Status.DebugInfo.EventsURL = "" if updateErr := r.Status().Update(ctx, clusterInstall); updateErr != nil { r.Log.WithError(updateErr).Error("failed to update ClusterDeployment Status") @@ -176,9 +196,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R return r.updateStatus(ctx, clusterInstall, cluster, err) } else { // Delete Day1 Cluster - err = r.Installer.DeregisterClusterInternal(ctx, installer.DeregisterClusterParams{ - ClusterID: *cluster.ID, - }) + _, err = r.deregisterClusterIfNeeded(ctx, req.NamespacedName) if err != nil { return r.updateStatus(ctx, clusterInstall, cluster, err) } @@ -202,6 +220,78 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R return r.updateStatus(ctx, clusterInstall, cluster, nil) } +func (r *ClusterDeploymentsReconciler) agentClusterInstallFinalizer(ctx context.Context, req ctrl.Request, + clusterInstall *hiveext.AgentClusterInstall) (*ctrl.Result, error) { + if clusterInstall.ObjectMeta.DeletionTimestamp.IsZero() { // clusterInstall not being deleted + // Register a finalizer if it is absent. + if !funk.ContainsString(clusterInstall.GetFinalizers(), AgentClusterInstallFinalizerName) { + controllerutil.AddFinalizer(clusterInstall, AgentClusterInstallFinalizerName) + if err := r.Update(ctx, clusterInstall); err != nil { + r.Log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + } else { // clusterInstall is being deleted + if funk.ContainsString(clusterInstall.GetFinalizers(), AgentClusterInstallFinalizerName) { + // deletion finalizer found, deregister the backend cluster and delete agents + reply, cleanUpErr := r.deregisterClusterIfNeeded(ctx, req.NamespacedName) + if cleanUpErr != nil { + r.Log.WithError(cleanUpErr).Errorf("failed to run pre-deletion cleanup for finalizer %s on resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &reply, cleanUpErr + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(clusterInstall, AgentClusterInstallFinalizerName) + if err := r.Update(ctx, clusterInstall); err != nil { + r.Log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + // Stop reconciliation as the item is being deleted + return &ctrl.Result{}, nil + } + return nil, nil +} + +func (r *ClusterDeploymentsReconciler) clusterDeploymentFinalizer(ctx context.Context, clusterDeployment *hivev1.ClusterDeployment) (*ctrl.Result, error) { + if clusterDeployment.ObjectMeta.DeletionTimestamp.IsZero() { // clusterDeployment not being deleted + // Register a finalizer if it is absent. + if !funk.ContainsString(clusterDeployment.GetFinalizers(), ClusterDeploymentFinalizerName) { + controllerutil.AddFinalizer(clusterDeployment, ClusterDeploymentFinalizerName) + if err := r.Update(ctx, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", + ClusterDeploymentFinalizerName, clusterDeployment.Name, clusterDeployment.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + } else { // clusterDeployment is being deleted + if funk.ContainsString(clusterDeployment.GetFinalizers(), ClusterDeploymentFinalizerName) { + reply, cleanUpErr := r.deleteClusterInstall(ctx, clusterDeployment) + if cleanUpErr != nil { + r.Log.WithError(cleanUpErr).Errorf( + "clusterDeployment %s %s is still waiting for clusterInstall %s to be deleted", + clusterDeployment.Name, clusterDeployment.Namespace, clusterDeployment.Spec.ClusterInstallRef.Name, + ) + return &reply, cleanUpErr + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(clusterDeployment, ClusterDeploymentFinalizerName) + if err := r.Update(ctx, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", + ClusterDeploymentFinalizerName, clusterDeployment.Name, clusterDeployment.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + // Stop reconciliation as the item is being deleted + return &ctrl.Result{}, nil + } + return nil, nil +} + func isInstalled(clusterDeployment *hivev1.ClusterDeployment, clusterInstall *hiveext.AgentClusterInstall) bool { if clusterDeployment.Spec.Installed { return true @@ -444,7 +534,7 @@ func (r *ClusterDeploymentsReconciler) updateIfNeeded(ctx context.Context, update := false notifyInfraEnv := false - var infraEnv *v1beta1.InfraEnv + var infraEnv *aiv1beta1.InfraEnv params := &models.ClusterUpdateParams{} @@ -783,12 +873,71 @@ func (r *ClusterDeploymentsReconciler) deregisterClusterIfNeeded(ctx context.Con }); err != nil { return buildReply(err) } + // Delete agents because their backend cluster got deregistered. + if err = r.DeleteClusterDeploymentAgents(ctx, key); err != nil { + return buildReply(err) + } r.Log.Infof("Cluster resource deleted, Unregistered cluster: %s", c.ID.String()) return buildReply(nil) } +func (r *ClusterDeploymentsReconciler) deleteClusterInstall(ctx context.Context, clusterDeployment *hivev1.ClusterDeployment) (ctrl.Result, error) { + + buildReply := func(err error) (ctrl.Result, error) { + reply := ctrl.Result{} + if err == nil { + return reply, nil + } + reply.RequeueAfter = defaultRequeueAfterOnError + err = errors.Wrapf(err, "clusterInstall: %s not deleted", clusterDeployment.Spec.ClusterInstallRef.Name) + r.Log.Error(err) + return reply, err + } + + clusterInstall := &hiveext.AgentClusterInstall{} + err := r.Get(ctx, + types.NamespacedName{ + Name: clusterDeployment.Spec.ClusterInstallRef.Name, + Namespace: clusterDeployment.Namespace, + }, + clusterInstall) + + if err != nil { + if k8serrors.IsNotFound(err) { + return buildReply(nil) + } + return buildReply(err) + } + + if err = r.Delete(ctx, clusterInstall); err != nil { + return buildReply(err) + } + // place this err so we requeue and verify deletion + err = errors.Errorf("could not confirm clusterInstall %s deletion was successfuly completed", clusterDeployment.Spec.ClusterInstallRef.Name) + return buildReply(err) +} + +func (r *ClusterDeploymentsReconciler) DeleteClusterDeploymentAgents(ctx context.Context, clusterDeployment types.NamespacedName) error { + agents := &aiv1beta1.AgentList{} + r.Log = r.Log.WithFields(logrus.Fields{"clusterDeployment": clusterDeployment.Name, "namespace": clusterDeployment.Namespace}) + if err := r.List(ctx, agents); err != nil { + return err + } + for i, clusterAgent := range agents.Items { + if clusterAgent.Spec.ClusterDeploymentName.Name == clusterDeployment.Name && + clusterAgent.Spec.ClusterDeploymentName.Namespace == clusterDeployment.Namespace { + r.Log.Infof("delete agent %s namespace %s", clusterAgent.Name, clusterAgent.Namespace) + if err := r.Client.Delete(ctx, &agents.Items[i]); err != nil { + r.Log.WithError(err).Errorf("Failed to delete resource %s %s", clusterAgent.Name, clusterAgent.Namespace) + return err + } + } + } + return nil +} + func (r *ClusterDeploymentsReconciler) SetupWithManager(mgr ctrl.Manager) error { mapSecretToClusterDeployment := func(a client.Object) []reconcile.Request { clusterDeployments := &hivev1.ClusterDeploymentList{} diff --git a/internal/controller/controllers/clusterdeployments_controller_test.go b/internal/controller/controllers/clusterdeployments_controller_test.go index 8d34aad1c8d..bf240106f0e 100644 --- a/internal/controller/controllers/clusterdeployments_controller_test.go +++ b/internal/controller/controllers/clusterdeployments_controller_test.go @@ -138,6 +138,18 @@ func getDefaultClusterDeploymentSpec(clusterName, aciName, pullSecretName string } } +func kubeTimeNow() *metav1.Time { + t := metav1.NewTime(time.Now()) + return &t +} + +func simulateACIDeletionWithFinalizer(ctx context.Context, c client.Client, aci *hiveext.AgentClusterInstall) { + // simulate ACI deletion with finalizer + aci.ObjectMeta.Finalizers = []string{AgentClusterInstallFinalizerName} + aci.ObjectMeta.DeletionTimestamp = kubeTimeNow() + Expect(c.Update(ctx, aci)).Should(BeNil()) +} + var _ = Describe("cluster reconcile", func() { var ( c client.Client @@ -471,23 +483,45 @@ var _ = Describe("cluster reconcile", func() { Context("cluster deletion", func() { var ( - sId strfmt.UUID - cluster *hivev1.ClusterDeployment + sId strfmt.UUID + cd *hivev1.ClusterDeployment + aci *hiveext.AgentClusterInstall ) BeforeEach(func() { - cluster = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) + defaultClusterSpec = getDefaultClusterDeploymentSpec(clusterName, agentClusterInstallName, pullSecretName) + cd = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) + cd.Status = hivev1.ClusterDeploymentStatus{} + defaultAgentClusterInstallSpec = getDefaultAgentClusterInstallSpec(clusterName) + aci = newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cd) id := uuid.New() sId = strfmt.UUID(id.String()) - cluster.Status = hivev1.ClusterDeploymentStatus{} - Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) + c = fakeclient.NewClientBuilder().WithScheme(scheme.Scheme).Build() + mockCtrl = gomock.NewController(GinkgoT()) + mockInstallerInternal = bminventory.NewMockInstallerInternals(mockCtrl) + mockClusterApi = cluster.NewMockAPI(mockCtrl) + mockHostApi = host.NewMockAPI(mockCtrl) + mockCRDEventsHandler = NewMockCRDEventsHandler(mockCtrl) + mockManifestsApi = manifests.NewMockClusterManifestsInternals(mockCtrl) + cr = &ClusterDeploymentsReconciler{ + Client: c, + Scheme: scheme.Scheme, + Log: common.GetTestLog(), + Installer: mockInstallerInternal, + ClusterApi: mockClusterApi, + HostApi: mockHostApi, + CRDEventsHandler: mockCRDEventsHandler, + Manifests: mockManifestsApi, + } + Expect(c.Create(ctx, cd)).ShouldNot(HaveOccurred()) + Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSet := getDefaultTestImageSet(imageSetName, releaseImage) Expect(c.Create(ctx, imageSet)).To(BeNil()) }) - It("cluster resource deleted - verify call to deregister cluster", func() { + It("agentClusterInstall resource deleted - verify call to deregister cluster", func() { backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -496,8 +530,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).ShouldNot(HaveOccurred()) Expect(result).Should(Equal(ctrl.Result{})) @@ -509,20 +543,21 @@ var _ = Describe("cluster reconcile", func() { ID: &sId, }, } - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(errors.New("internal error")) - expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: internal error", cluster.Name) + expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: internal error", cd.Name) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + Expect(c.Update(ctx, aci)).Should(BeNil()) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).Should(HaveOccurred()) Expect(err.Error()).Should(Equal(expectedErrMsg)) Expect(result).Should(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) }) - It("cluster resource deleted and created again", func() { + It("agentClusterInstall resource deleted and created again", func() { backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -532,8 +567,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil) mockInstallerInternal.EXPECT().AddOpenshiftVersion(gomock.Any(), gomock.Any(), gomock.Any()).Return(openshiftVersion, nil) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).ShouldNot(HaveOccurred()) Expect(result).Should(Equal(ctrl.Result{})) @@ -541,12 +576,11 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(nil, gorm.ErrRecordNotFound) mockInstallerInternal.EXPECT().RegisterClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()).Return(backEndCluster, nil) - cluster = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) - Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) - aci := newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cluster) + Expect(c.Delete(ctx, aci)).ShouldNot(HaveOccurred()) + aci = newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cd) Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) - request = newClusterDeploymentRequest(cluster) + request = newClusterDeploymentRequest(cd) result, err = cr.Reconcile(ctx, request) Expect(err).To(BeNil()) Expect(result).To(Equal(ctrl.Result{})) @@ -636,7 +670,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(3) password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -682,7 +716,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.StatusInfo = swag.String("Done") backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(3) password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -728,8 +762,9 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) expectedError := errors.New("internal error") + expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: %s", cluster.Name, expectedError) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(expectedError).Times(1) setClusterCondition(&aci.Status.Conditions, hivev1.ClusterInstallCondition{ Type: ClusterCompletedCondition, @@ -744,7 +779,7 @@ var _ = Describe("cluster reconcile", func() { Expect(result).To(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) aci = getTestClusterInstall() - expectedState := fmt.Sprintf("%s %s", BackendErrorMsg, expectedError) + expectedState := fmt.Sprintf("%s %s", BackendErrorMsg, expectedErrMsg) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Reason).To(Equal(BackendErrorReason)) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Status).To(Equal(corev1.ConditionFalse)) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Message).To(Equal(expectedState)) @@ -757,7 +792,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) expectedErr := "internal error" mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil).Times(1) mockInstallerInternal.EXPECT().RegisterAddHostsClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(expectedErr)) diff --git a/subsystem/kubeapi_test.go b/subsystem/kubeapi_test.go index 0e159d6e072..c079c53c0ff 100644 --- a/subsystem/kubeapi_test.go +++ b/subsystem/kubeapi_test.go @@ -11,6 +11,7 @@ import ( "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" + "github.com/google/uuid" "github.com/jinzhu/gorm" bmhv1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" . "github.com/onsi/ginkgo" @@ -36,10 +37,10 @@ import ( ) const ( - fakeIgnitionConfigOverride = `{"ignition": {"version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}` - badIgnitionConfigOverride = `bad ignition config` - clusterDeploymentName = "test-cluster" - clusterAgentClusterInstallName = "test-agent-cluster-install" + fakeIgnitionConfigOverride = `{"ignition": {"version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}` + badIgnitionConfigOverride = `bad ignition config` + clusterDeploymentNamePrefix = "test-cluster" + clusterAgentClusterInstallNamePrefix = "test-agent-cluster-install" ) var ( @@ -117,7 +118,8 @@ func updateAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, }, "30s", "10s").Should(BeNil()) } -func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, spec *hiveext.AgentClusterInstallSpec) { +func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, spec *hiveext.AgentClusterInstallSpec, + clusterAgentClusterInstallName string) { deployClusterImageSetCRD(ctx, client, spec.ImageSetRef) err := client.Create(ctx, &hiveext.AgentClusterInstall{ TypeMeta: metav1.TypeMeta{ @@ -353,7 +355,7 @@ func checkInfraEnvCondition(ctx context.Context, key types.NamespacedName, condi func getDefaultClusterDeploymentSpec(secretRef *corev1.LocalObjectReference) *hivev1.ClusterDeploymentSpec { return &hivev1.ClusterDeploymentSpec{ - ClusterName: clusterDeploymentName, + ClusterName: clusterDeploymentNamePrefix + randomNameSuffix(), BaseDomain: "hive.example.com", Platform: hivev1.Platform{ AgentBareMetal: &agentv1.BareMetalPlatform{}, @@ -363,12 +365,12 @@ func getDefaultClusterDeploymentSpec(secretRef *corev1.LocalObjectReference) *hi Group: hiveext.Group, Version: hiveext.Version, Kind: "AgentClusterInstall", - Name: clusterAgentClusterInstallName, + Name: clusterAgentClusterInstallNamePrefix + randomNameSuffix(), }, } } -func getDefaultAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { +func getDefaultAgentClusterInstallSpec(clusterDeploymentName string) *hiveext.AgentClusterInstallSpec { return &hiveext.AgentClusterInstallSpec{ Networking: hiveext.Networking{ MachineNetwork: []hiveext.MachineNetworkEntry{}, @@ -390,7 +392,7 @@ func getDefaultAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { } } -func getDefaultSNOAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { +func getDefaultSNOAgentClusterInstallSpec(clusterDeploymentName string) *hiveext.AgentClusterInstallSpec { return &hiveext.AgentClusterInstallSpec{ Networking: hiveext.Networking{ MachineNetwork: []hiveext.MachineNetworkEntry{{CIDR: "1.2.3.0/24"}}, @@ -448,12 +450,15 @@ func getAgentMac(ctx context.Context, client k8sclient.Client, key types.Namespa return mac } +func randomNameSuffix() string { + return fmt.Sprintf("-%s", strings.Split(uuid.New().String(), "-")[0]) +} + func cleanUP(ctx context.Context, client k8sclient.Client) { - Expect(client.DeleteAllOf(ctx, &hivev1.ClusterDeployment{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) + Expect(client.DeleteAllOf(ctx, &hivev1.ClusterDeployment{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) // Should also delete all agents Expect(client.DeleteAllOf(ctx, &hivev1.ClusterImageSet{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &v1beta1.InfraEnv{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &v1beta1.NMStateConfig{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) - Expect(client.DeleteAllOf(ctx, &v1beta1.Agent{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &bmhv1alpha1.BareMetalHost{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) ps := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -470,6 +475,14 @@ func cleanUP(ctx context.Context, client k8sclient.Client) { } func verifyCleanUP(ctx context.Context, client k8sclient.Client) { + By("Verify ClusterDeployment Cleanup") + Eventually(func() int { + clusterDeploymentList := &hivev1.ClusterDeploymentList{} + err := client.List(ctx, clusterDeploymentList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(clusterDeploymentList.Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify AgentClusterInstall Cleanup") Eventually(func() int { aciList := &hiveext.AgentClusterInstallList{} @@ -477,6 +490,46 @@ func verifyCleanUP(ctx context.Context, client k8sclient.Client) { Expect(err).To(BeNil()) return len(aciList.Items) }, "2m", "2s").Should(Equal(0)) + + By("Verify ClusterImageSet Cleanup") + Eventually(func() int { + clusterImageSetList := &hivev1.ClusterImageSetList{} + err := client.List(ctx, clusterImageSetList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(clusterImageSetList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify InfraEnv Cleanup") + Eventually(func() int { + infraEnvList := &v1beta1.InfraEnvList{} + err := client.List(ctx, infraEnvList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(infraEnvList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify NMStateConfig Cleanup") + Eventually(func() int { + nmStateConfigList := &v1beta1.NMStateConfigList{} + err := client.List(ctx, nmStateConfigList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(nmStateConfigList.Items) + }, "2m", "10s").Should(Equal(0)) + + By("Verify Agent Cleanup") + Eventually(func() int { + agentList := &v1beta1.AgentList{} + err := client.List(ctx, agentList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(agentList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify BareMetalHost Cleanup") + Eventually(func() int { + bareMetalHostList := &bmhv1alpha1.BareMetalHostList{} + err := client.List(ctx, bareMetalHostList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(bareMetalHostList.Items) + }, "2m", "2s").Should(Equal(0)) } func setupNewHost(ctx context.Context, hostname string, clusterID strfmt.UUID) *models.Host { @@ -502,17 +555,17 @@ var _ = Describe("[kube-api]cluster installation", func() { clearDB() }) - It("deploy clusterDeployment with agents and wait for ready", func() { + It("deploy CD with ACI and agents - wait for ready, delete CD and verify ACI and agents deletion", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) - key := types.NamespacedName{ + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } - cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) hosts := make([]*models.Host, 0) for i := 0; i < 3; i++ { @@ -524,6 +577,7 @@ var _ = Describe("[kube-api]cluster installation", func() { checkAgentCondition(ctx, host.ID.String(), controllers.ValidatedCondition, controllers.ValidationsFailingReason) } generateFullMeshConnectivity(ctx, "1.2.3.10", hosts...) + By("Approve Agents") for _, host := range hosts { hostkey := types.NamespacedName{ Namespace: Options.Namespace, @@ -537,20 +591,83 @@ var _ = Describe("[kube-api]cluster installation", func() { } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } + By("Verify ClusterDeployment ReadyForInstallation") checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterAlreadyInstallingReason) + By("Delete ClusterDeployment") + err := kubeClient.Delete(ctx, getClusterDeploymentCRD(ctx, kubeClient, clusterKey)) + Expect(err).To(BeNil()) + By("Verify AgentClusterInstall was deleted") + Eventually(func() bool { + aci := &hiveext.AgentClusterInstall{} + err := kubeClient.Get(ctx, installkey, aci) + return apierrors.IsNotFound(err) + }, "30s", "10s").Should(Equal(true)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + }) + + It("deploy CD with ACI and agents - wait for ready, delete ACI only and verify agents deletion", func() { + secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterName, + } + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) + configureLocalAgentClient(cluster.ID.String()) + hosts := make([]*models.Host, 0) + for i := 0; i < 3; i++ { + hostname := fmt.Sprintf("h%d", i) + host := setupNewHost(ctx, hostname, *cluster.ID) + hosts = append(hosts, host) + } + for _, host := range hosts { + checkAgentCondition(ctx, host.ID.String(), controllers.ValidatedCondition, controllers.ValidationsFailingReason) + } + generateFullMeshConnectivity(ctx, "1.2.3.10", hosts...) + By("Approve Agents") + for _, host := range hosts { + hostkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: host.ID.String(), + } + Eventually(func() error { + agent := getAgentCRD(ctx, kubeClient, hostkey) + agent.Spec.Approved = true + return kubeClient.Update(ctx, agent) + }, "30s", "10s").Should(BeNil()) + } + installkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, + } + By("Verify ClusterDeployment ReadyForInstallation") + checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterAlreadyInstallingReason) + By("Delete AgentClusterInstall") + err := kubeClient.Delete(ctx, getAgentClusterInstallCRD(ctx, kubeClient, installkey)) + Expect(err).To(BeNil()) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) }) It("deploy clusterDeployment with agent and update agent", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -587,13 +704,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent,bmh and ignition config override", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -655,13 +772,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and invalid ignition config", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -696,13 +813,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and update installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -761,13 +878,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and invalid installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -822,13 +939,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent,bmh and installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -905,15 +1022,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -956,15 +1073,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) @@ -999,15 +1116,15 @@ var _ = Describe("[kube-api]cluster installation", func() { Name: infraEnvName, } deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1023,15 +1140,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1063,7 +1180,7 @@ var _ = Describe("[kube-api]cluster installation", func() { } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } verifyHyperthreadingSetup := func(mode string) { Eventually(func() string { @@ -1073,19 +1190,19 @@ var _ = Describe("[kube-api]cluster installation", func() { } By("new deployment with hyperthreading disabled") - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.ControlPlane = &hiveext.AgentMachinePool{ Hyperthreading: hiveext.HyperthreadingDisabled, } aciSpec.Compute = []hiveext.AgentMachinePool{ {Hyperthreading: hiveext.HyperthreadingDisabled}, } - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) verifyHyperthreadingSetup(models.ClusterHyperthreadingNone) By("update deployment with hyperthreading enabled on master only") - aciSpec = getDefaultAgentClusterInstallSpec() + aciSpec = getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.ControlPlane = &hiveext.AgentMachinePool{ Hyperthreading: hiveext.HyperthreadingEnabled, } @@ -1094,7 +1211,7 @@ var _ = Describe("[kube-api]cluster installation", func() { verifyHyperthreadingSetup(models.ClusterHyperthreadingMasters) By("update deployment with hyperthreading enabled on workers only") - aciSpec = getDefaultAgentClusterInstallSpec() + aciSpec = getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Compute = []hiveext.AgentMachinePool{ {Hyperthreading: hiveext.HyperthreadingEnabled}, } @@ -1107,15 +1224,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1137,15 +1254,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1182,15 +1299,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) infraEnvSpec := getDefaultInfraEnvSpec(secretRef, clusterDeploymentSpec) @@ -1223,15 +1340,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) infraEnvSpec := getDefaultInfraEnvSpec(secretRef, clusterDeploymentSpec) @@ -1250,16 +1367,16 @@ var _ = Describe("[kube-api]cluster installation", func() { It("SNO deploy clusterDeployment full install and validate MetaData", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) // Add space suffix to SSHPublicKey to validate proper install sshPublicKeySuffixSpace := fmt.Sprintf("%s ", aciSpec.SSHPublicKey) aciSpec.SSHPublicKey = sshPublicKeySuffixSpace - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1270,7 +1387,7 @@ var _ = Describe("[kube-api]cluster installation", func() { } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } By("Check Event URL exists") Eventually(func() string { @@ -1343,13 +1460,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("None SNO deploy clusterDeployment full install and validate MetaData", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1383,7 +1500,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) Eventually(func() bool { @@ -1420,6 +1537,11 @@ var _ = Describe("[kube-api]cluster installation", func() { cluster = getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) Expect(*cluster.Kind).Should(Equal(models.ClusterKindAddHostsCluster)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify Cluster Metadata") passwordSecretRef := getAgentClusterInstallCRD(ctx, kubeClient, installkey).Spec.ClusterMetadata.AdminPasswordSecretRef Expect(passwordSecretRef).NotTo(BeNil()) @@ -1443,13 +1565,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("None SNO deploy clusterDeployment full install and Day 2 new host", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1476,7 +1598,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) Eventually(func() bool { @@ -1509,6 +1631,11 @@ var _ = Describe("[kube-api]cluster installation", func() { cluster = getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) Expect(*cluster.Kind).Should(Equal(models.ClusterKindAddHostsCluster)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + By("Add Day 2 host and approve agent") configureLocalAgentClient(cluster.ID.String()) host := setupNewHost(ctx, "hostnameday2", *cluster.ID) @@ -1533,13 +1660,14 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with invalid machine cidr", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Networking.MachineNetwork = []hiveext.MachineNetworkEntry{{CIDR: "1.2.3.5/24"}} - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) }) @@ -1548,12 +1676,12 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Networking.MachineNetwork = []hiveext.MachineNetworkEntry{} - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) }) @@ -1562,22 +1690,22 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.ImageSetRef.Name = "invalid" - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) }) It("deploy clusterDeployment with missing clusterImageSet", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) // Create AgentClusterInstall without creating the clusterImageSet err := kubeClient.Create(ctx, &hiveext.AgentClusterInstall{ TypeMeta: metav1.TypeMeta{ @@ -1586,14 +1714,14 @@ var _ = Describe("[kube-api]cluster installation", func() { }, ObjectMeta: metav1.ObjectMeta{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, }, Spec: *aciSpec, }) Expect(err).To(BeNil()) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) @@ -1606,15 +1734,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } Eventually(func() bool { aci := getAgentClusterInstallCRD(ctx, kubeClient, installkey) @@ -1635,7 +1763,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) ref := &corev1.LocalObjectReference{Name: "cluster-install-config"} aciSpec.ManifestsConfigMapRef = ref content := `apiVersion: machineconfiguration.openshift.io/v1 @@ -1650,7 +1778,7 @@ spec: By("Start installation without config map") deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, @@ -1670,7 +1798,7 @@ spec: }, "30s", "10s").Should(BeNil()) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterReadyReason) checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) @@ -1700,15 +1828,15 @@ spec: secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout)