From 9e52e871b53e9f454d0e86b4971c9dc1ca909412 Mon Sep 17 00:00:00 2001 From: Nir Magnezi Date: Wed, 28 Apr 2021 15:30:36 +0300 Subject: [PATCH] MGMT-4261 Agent CR and ACI cleanup when deleting CD clusterdeployments_controller: When a CD CR gets deleted, the controller will place a finalizer to trigger a pre-deletion cleanup. The cleanup includes ACI deletion, which will place a finalizer on ACI. ACI finalizer includes backend-cluster deregister and deletion of the cluster agent resources. As for subsystem tests: 1. Tests will now assert for agent deletion per ACI removal. 2. Tests will now assert for agent deletion and ACI per CD removal. 3. Each test will now use a unique CD and ACI CR name - to provide better test isolation. This was also done because deletion via finalizers takes an additional reconcile to be fully deleted. 4. Per 2, and to avoid any cleanup bugs, the cleanup done for subsystem tests (AfterEach) is now followed up by verifyCleanUP P.S. Note that this PR does not cover host deregister upon Agent CR deletion, which was added in https://github.com/openshift/assisted-service/pull/1642 --- config/rbac/role.yaml | 6 + ...ervice-operator.clusterserviceversion.yaml | 6 + .../clusterdeployments_controller.go | 185 ++++++++- .../clusterdeployments_controller_test.go | 83 ++-- subsystem/kubeapi_test.go | 360 ++++++++++++------ 5 files changed, 482 insertions(+), 158 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6440805e784..c1f8f0659f9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -183,6 +183,12 @@ rules: - patch - update - watch +- apiGroups: + - extensions.hive.openshift.io + resources: + - agentclusterinstalls/finalizers + verbs: + - update - apiGroups: - extensions.hive.openshift.io resources: diff --git a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml index 625ca966d87..afdfd3c5f2b 100644 --- a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml @@ -347,6 +347,12 @@ spec: - patch - update - watch + - apiGroups: + - extensions.hive.openshift.io + resources: + - agentclusterinstalls/finalizers + verbs: + - update - apiGroups: - extensions.hive.openshift.io resources: diff --git a/internal/controller/controllers/clusterdeployments_controller.go b/internal/controller/controllers/clusterdeployments_controller.go index d47c7f1571e..8c96d394678 100644 --- a/internal/controller/controllers/clusterdeployments_controller.go +++ b/internal/controller/controllers/clusterdeployments_controller.go @@ -32,7 +32,7 @@ import ( "github.com/openshift/assisted-service/internal/cluster" "github.com/openshift/assisted-service/internal/common" hiveext "github.com/openshift/assisted-service/internal/controller/api/hiveextension/v1beta1" - "github.com/openshift/assisted-service/internal/controller/api/v1beta1" + aiv1beta1 "github.com/openshift/assisted-service/internal/controller/api/v1beta1" "github.com/openshift/assisted-service/internal/host" "github.com/openshift/assisted-service/internal/manifests" "github.com/openshift/assisted-service/models" @@ -41,6 +41,7 @@ import ( hivev1 "github.com/openshift/hive/apis/hive/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/thoas/go-funk" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -59,7 +60,9 @@ import ( const ( adminPasswordSecretStringTemplate = "%s-admin-password" adminKubeConfigStringTemplate = "%s-admin-kubeconfig" - InstallConfigOverrides = v1beta1.Group + "/install-config-overrides" + InstallConfigOverrides = aiv1beta1.Group + "/install-config-overrides" + ClusterDeploymentFinalizerName = "clusterdeployments." + aiv1beta1.Group + "/ai-deprovision" + AgentClusterInstallFinalizerName = "agentclusterinstall." + aiv1beta1.Group + "/ai-deprovision" ) const HighAvailabilityModeNone = "None" @@ -85,18 +88,17 @@ type ClusterDeploymentsReconciler struct { // +kubebuilder:rbac:groups=hive.openshift.io,resources=clusterimagesets,verbs=get;list;watch // +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls/finalizers,verbs=update func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log.Infof("Reconcile has been called for ClusterDeployment name=%s namespace=%s", req.Name, req.Namespace) clusterDeployment := &hivev1.ClusterDeployment{} - err := r.Get(ctx, req.NamespacedName, clusterDeployment) - if err != nil { - if k8serrors.IsNotFound(err) { - return r.deregisterClusterIfNeeded(ctx, req.NamespacedName) - } - r.Log.WithError(err).Errorf("Failed to get resource %s", req.NamespacedName) - return ctrl.Result{Requeue: true}, nil + clusterInstallDeleted := false + + if err := r.Get(ctx, req.NamespacedName, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to get ClusterDeployment name=%s namespace=%s", req.Name, req.Namespace) + return ctrl.Result{}, client.IgnoreNotFound(err) } // ignore unsupported platforms @@ -111,7 +113,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R } aciName := clusterDeployment.Spec.ClusterInstallRef.Name - err = r.Get(ctx, + err := r.Get(ctx, types.NamespacedName{ Namespace: clusterDeployment.Namespace, Name: aciName, @@ -119,11 +121,29 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R clusterInstall) if err != nil { if k8serrors.IsNotFound(err) { + // mark that clusterInstall was already deleted so we skip it if needed. + clusterInstallDeleted = true r.Log.WithField("AgentClusterInstall", aciName).Infof("AgentClusterInstall does not exist for ClusterDeployment %s", clusterDeployment.Name) - return ctrl.Result{}, nil + if clusterDeployment.ObjectMeta.DeletionTimestamp.IsZero() { + // we have no agentClusterInstall and clusterDeployment is not being deleted. stop reconciliation. + return ctrl.Result{}, nil + } + } else { + r.Log.WithError(err).Errorf("failed to get AgentClusterInstall name=%s namespace=%s", aciName, clusterDeployment.Namespace) + return ctrl.Result{Requeue: true}, err + } + } + + if !clusterInstallDeleted { + aciReply, aciErr := r.agentClusterInstallFinalizer(ctx, req, clusterInstall) + if aciReply != nil { + return *aciReply, aciErr } - r.Log.WithError(err).Errorf("Failed to get AgentClusterInstall %s", aciName) - return ctrl.Result{Requeue: true}, err + } + + cdReplay, cdErr := r.clusterDeploymentFinalizer(ctx, clusterDeployment) + if cdReplay != nil { + return *cdReplay, cdErr } err = r.ensureOwnerRef(ctx, clusterDeployment, clusterInstall) @@ -139,7 +159,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R if !r.isSNO(clusterInstall) { return r.createNewDay2Cluster(ctx, req.NamespacedName, clusterDeployment, clusterInstall) } - // cluster is installed and SNO nothing to do + // cluster is installed and SNO nothing to do return ctrl.Result{Requeue: false}, nil } if err != nil { @@ -166,9 +186,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R return r.updateStatus(ctx, clusterInstall, cluster, err) } else { // Delete Day1 Cluster - err = r.Installer.DeregisterClusterInternal(ctx, installer.DeregisterClusterParams{ - ClusterID: *cluster.ID, - }) + _, err = r.deregisterClusterIfNeeded(ctx, req.NamespacedName) if err != nil { return r.updateStatus(ctx, clusterInstall, cluster, err) } @@ -192,6 +210,78 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R return r.updateStatus(ctx, clusterInstall, cluster, nil) } +func (r *ClusterDeploymentsReconciler) agentClusterInstallFinalizer(ctx context.Context, req ctrl.Request, + clusterInstall *hiveext.AgentClusterInstall) (*ctrl.Result, error) { + if clusterInstall.ObjectMeta.DeletionTimestamp.IsZero() { // clusterInstall not being deleted + // Register a finalizer if it is absent. + if !funk.ContainsString(clusterInstall.GetFinalizers(), AgentClusterInstallFinalizerName) { + controllerutil.AddFinalizer(clusterInstall, AgentClusterInstallFinalizerName) + if err := r.Update(ctx, clusterInstall); err != nil { + r.Log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + } else { // clusterInstall is being deleted + if funk.ContainsString(clusterInstall.GetFinalizers(), AgentClusterInstallFinalizerName) { + // deletion finalizer found, deregister the backend cluster and delete agents + reply, cleanUpErr := r.deregisterClusterIfNeeded(ctx, req.NamespacedName) + if cleanUpErr != nil { + r.Log.WithError(cleanUpErr).Errorf("failed to run pre-deletion cleanup for finalizer %s on resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &reply, cleanUpErr + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(clusterInstall, AgentClusterInstallFinalizerName) + if err := r.Update(ctx, clusterInstall); err != nil { + r.Log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + // Stop reconciliation as the item is being deleted + return &ctrl.Result{}, nil + } + return nil, nil +} + +func (r *ClusterDeploymentsReconciler) clusterDeploymentFinalizer(ctx context.Context, clusterDeployment *hivev1.ClusterDeployment) (*ctrl.Result, error) { + if clusterDeployment.ObjectMeta.DeletionTimestamp.IsZero() { // clusterDeployment not being deleted + // Register a finalizer if it is absent. + if !funk.ContainsString(clusterDeployment.GetFinalizers(), ClusterDeploymentFinalizerName) { + controllerutil.AddFinalizer(clusterDeployment, ClusterDeploymentFinalizerName) + if err := r.Update(ctx, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", + ClusterDeploymentFinalizerName, clusterDeployment.Name, clusterDeployment.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + } else { // clusterDeployment is being deleted + if funk.ContainsString(clusterDeployment.GetFinalizers(), ClusterDeploymentFinalizerName) { + reply, cleanUpErr := r.deleteClusterInstall(ctx, clusterDeployment) + if cleanUpErr != nil { + r.Log.WithError(cleanUpErr).Errorf( + "clusterDeployment %s %s is still waiting for clusterInstall %s to be deleted", + clusterDeployment.Name, clusterDeployment.Namespace, clusterDeployment.Spec.ClusterInstallRef.Name, + ) + return &reply, cleanUpErr + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(clusterDeployment, ClusterDeploymentFinalizerName) + if err := r.Update(ctx, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", + ClusterDeploymentFinalizerName, clusterDeployment.Name, clusterDeployment.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + // Stop reconciliation as the item is being deleted + return &ctrl.Result{}, nil + } + return nil, nil +} + func isInstalled(clusterDeployment *hivev1.ClusterDeployment, clusterInstall *hiveext.AgentClusterInstall) bool { if clusterDeployment.Spec.Installed { return true @@ -388,7 +478,7 @@ func (r *ClusterDeploymentsReconciler) updateIfNeeded(ctx context.Context, update := false notifyInfraEnv := false - var infraEnv *v1beta1.InfraEnv + var infraEnv *aiv1beta1.InfraEnv params := &models.ClusterUpdateParams{} @@ -715,12 +805,71 @@ func (r *ClusterDeploymentsReconciler) deregisterClusterIfNeeded(ctx context.Con }); err != nil { return buildReply(err) } + // Delete agents because their backend cluster got deregistered. + if err = r.DeleteClusterDeploymentAgents(ctx, key); err != nil { + return buildReply(err) + } r.Log.Infof("Cluster resource deleted, Unregistered cluster: %s", c.ID.String()) return buildReply(nil) } +func (r *ClusterDeploymentsReconciler) deleteClusterInstall(ctx context.Context, clusterDeployment *hivev1.ClusterDeployment) (ctrl.Result, error) { + + buildReply := func(err error) (ctrl.Result, error) { + reply := ctrl.Result{} + if err == nil { + return reply, nil + } + reply.RequeueAfter = defaultRequeueAfterOnError + err = errors.Wrapf(err, "clusterInstall: %s not deleted", clusterDeployment.Spec.ClusterInstallRef.Name) + r.Log.Error(err) + return reply, err + } + + clusterInstall := &hiveext.AgentClusterInstall{} + err := r.Get(ctx, + types.NamespacedName{ + Name: clusterDeployment.Spec.ClusterInstallRef.Name, + Namespace: clusterDeployment.Namespace, + }, + clusterInstall) + + if err != nil { + if k8serrors.IsNotFound(err) { + return buildReply(nil) + } + return buildReply(err) + } + + if err = r.Delete(ctx, clusterInstall); err != nil { + return buildReply(err) + } + // place this err so we requeue and verify deletion + err = errors.Errorf("could not confirm clusterInstall %s deletion was successfuly completed", clusterDeployment.Spec.ClusterInstallRef.Name) + return buildReply(err) +} + +func (r *ClusterDeploymentsReconciler) DeleteClusterDeploymentAgents(ctx context.Context, clusterDeployment types.NamespacedName) error { + agents := &aiv1beta1.AgentList{} + r.Log = r.Log.WithFields(logrus.Fields{"clusterDeployment": clusterDeployment.Name, "namespace": clusterDeployment.Namespace}) + if err := r.List(ctx, agents); err != nil { + return err + } + for i, clusterAgent := range agents.Items { + if clusterAgent.Spec.ClusterDeploymentName.Name == clusterDeployment.Name && + clusterAgent.Spec.ClusterDeploymentName.Namespace == clusterDeployment.Namespace { + r.Log.Infof("delete agent %s namespace %s", clusterAgent.Name, clusterAgent.Namespace) + if err := r.Client.Delete(ctx, &agents.Items[i]); err != nil { + r.Log.WithError(err).Errorf("Failed to delete resource %s %s", clusterAgent.Name, clusterAgent.Namespace) + return err + } + } + } + return nil +} + func (r *ClusterDeploymentsReconciler) SetupWithManager(mgr ctrl.Manager) error { mapSecretToClusterDeployment := func(a client.Object) []reconcile.Request { clusterDeployments := &hivev1.ClusterDeploymentList{} diff --git a/internal/controller/controllers/clusterdeployments_controller_test.go b/internal/controller/controllers/clusterdeployments_controller_test.go index 7ded3138b1e..340178ac486 100644 --- a/internal/controller/controllers/clusterdeployments_controller_test.go +++ b/internal/controller/controllers/clusterdeployments_controller_test.go @@ -136,6 +136,18 @@ func getDefaultClusterDeploymentSpec(clusterName, aciName, pullSecretName string } } +func kubeTimeNow() *metav1.Time { + t := metav1.NewTime(time.Now()) + return &t +} + +func simulateACIDeletionWithFinalizer(ctx context.Context, c client.Client, aci *hiveext.AgentClusterInstall) { + // simulate ACI deletion with finalizer + aci.ObjectMeta.Finalizers = []string{AgentClusterInstallFinalizerName} + aci.ObjectMeta.DeletionTimestamp = kubeTimeNow() + Expect(c.Update(ctx, aci)).Should(BeNil()) +} + var _ = Describe("cluster reconcile", func() { var ( c client.Client @@ -435,23 +447,45 @@ var _ = Describe("cluster reconcile", func() { Context("cluster deletion", func() { var ( - sId strfmt.UUID - cluster *hivev1.ClusterDeployment + sId strfmt.UUID + cd *hivev1.ClusterDeployment + aci *hiveext.AgentClusterInstall ) BeforeEach(func() { - cluster = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) + defaultClusterSpec = getDefaultClusterDeploymentSpec(clusterName, agentClusterInstallName, pullSecretName) + cd = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) + cd.Status = hivev1.ClusterDeploymentStatus{} + defaultAgentClusterInstallSpec = getDefaultAgentClusterInstallSpec(clusterName) + aci = newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cd) id := uuid.New() sId = strfmt.UUID(id.String()) - cluster.Status = hivev1.ClusterDeploymentStatus{} - Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) + c = fakeclient.NewClientBuilder().WithScheme(scheme.Scheme).Build() + mockCtrl = gomock.NewController(GinkgoT()) + mockInstallerInternal = bminventory.NewMockInstallerInternals(mockCtrl) + mockClusterApi = cluster.NewMockAPI(mockCtrl) + mockHostApi = host.NewMockAPI(mockCtrl) + mockCRDEventsHandler = NewMockCRDEventsHandler(mockCtrl) + mockManifestsApi = manifests.NewMockClusterManifestsInternals(mockCtrl) + cr = &ClusterDeploymentsReconciler{ + Client: c, + Scheme: scheme.Scheme, + Log: common.GetTestLog(), + Installer: mockInstallerInternal, + ClusterApi: mockClusterApi, + HostApi: mockHostApi, + CRDEventsHandler: mockCRDEventsHandler, + Manifests: mockManifestsApi, + } + Expect(c.Create(ctx, cd)).ShouldNot(HaveOccurred()) + Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSet := getDefaultTestImageSet(imageSetName, releaseImage) Expect(c.Create(ctx, imageSet)).To(BeNil()) }) - It("cluster resource deleted - verify call to deregister cluster", func() { + It("agentClusterInstall resource deleted - verify call to deregister cluster", func() { backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -460,8 +494,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).ShouldNot(HaveOccurred()) Expect(result).Should(Equal(ctrl.Result{})) @@ -473,20 +507,21 @@ var _ = Describe("cluster reconcile", func() { ID: &sId, }, } - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(errors.New("internal error")) - expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: internal error", cluster.Name) + expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: internal error", cd.Name) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + Expect(c.Update(ctx, aci)).Should(BeNil()) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).Should(HaveOccurred()) Expect(err.Error()).Should(Equal(expectedErrMsg)) Expect(result).Should(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) }) - It("cluster resource deleted and created again", func() { + It("agentClusterInstall resource deleted and created again", func() { backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -496,8 +531,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil) mockInstallerInternal.EXPECT().AddOpenshiftVersion(gomock.Any(), gomock.Any(), gomock.Any()).Return(openshiftVersion, nil) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).ShouldNot(HaveOccurred()) Expect(result).Should(Equal(ctrl.Result{})) @@ -505,12 +540,11 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(nil, gorm.ErrRecordNotFound) mockInstallerInternal.EXPECT().RegisterClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()).Return(backEndCluster, nil) - cluster = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) - Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) - aci := newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cluster) + Expect(c.Delete(ctx, aci)).ShouldNot(HaveOccurred()) + aci = newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cd) Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) - request = newClusterDeploymentRequest(cluster) + request = newClusterDeploymentRequest(cd) result, err = cr.Reconcile(ctx, request) Expect(err).To(BeNil()) Expect(result).To(Equal(ctrl.Result{})) @@ -599,7 +633,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(3) password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -645,7 +679,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.StatusInfo = swag.String("Done") backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(3) password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -691,8 +725,9 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) expectedError := errors.New("internal error") + expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: %s", cluster.Name, expectedError) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(expectedError).Times(1) setClusterCondition(&aci.Status.Conditions, hivev1.ClusterInstallCondition{ Type: ClusterCompletedCondition, @@ -707,7 +742,7 @@ var _ = Describe("cluster reconcile", func() { Expect(result).To(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) aci = getTestClusterInstall() - expectedState := fmt.Sprintf("%s %s", BackendErrorMsg, expectedError) + expectedState := fmt.Sprintf("%s %s", BackendErrorMsg, expectedErrMsg) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Reason).To(Equal(BackendErrorReason)) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Status).To(Equal(corev1.ConditionFalse)) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Message).To(Equal(expectedState)) @@ -720,7 +755,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) expectedErr := "internal error" mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil).Times(1) mockInstallerInternal.EXPECT().RegisterAddHostsClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(expectedErr)) diff --git a/subsystem/kubeapi_test.go b/subsystem/kubeapi_test.go index 6cfe27d1f32..bdab96051bd 100644 --- a/subsystem/kubeapi_test.go +++ b/subsystem/kubeapi_test.go @@ -11,6 +11,7 @@ import ( "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" + "github.com/google/uuid" "github.com/jinzhu/gorm" bmhv1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" . "github.com/onsi/ginkgo" @@ -36,10 +37,10 @@ import ( ) const ( - fakeIgnitionConfigOverride = `{"ignition": {"version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}` - badIgnitionConfigOverride = `bad ignition config` - clusterDeploymentName = "test-cluster" - clusterAgentClusterInstallName = "test-agent-cluster-install" + fakeIgnitionConfigOverride = `{"ignition": {"version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}` + badIgnitionConfigOverride = `bad ignition config` + clusterDeploymentNamePrefix = "test-cluster" + clusterAgentCLusterInstallNamePrefix = "test-agent-cluster-install" ) var ( @@ -109,7 +110,8 @@ func deployPullSecretResource(ctx context.Context, client k8sclient.Client, name Expect(client.Create(ctx, s)).To(BeNil()) } -func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, spec *hiveext.AgentClusterInstallSpec) { +func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, spec *hiveext.AgentClusterInstallSpec, + clusterAgentCLusterInstallName string) { deployClusterImageSetCRD(ctx, client, spec.ImageSetRef) err := client.Create(ctx, &hiveext.AgentClusterInstall{ TypeMeta: metav1.TypeMeta{ @@ -118,7 +120,7 @@ func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, }, ObjectMeta: metav1.ObjectMeta{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterAgentCLusterInstallName, }, Spec: *spec, }) @@ -345,7 +347,7 @@ func checkInfraEnvCondition(ctx context.Context, key types.NamespacedName, condi func getDefaultClusterDeploymentSpec(secretRef *corev1.LocalObjectReference) *hivev1.ClusterDeploymentSpec { return &hivev1.ClusterDeploymentSpec{ - ClusterName: clusterDeploymentName, + ClusterName: clusterDeploymentNamePrefix + randomNameSuffix(), BaseDomain: "hive.example.com", Platform: hivev1.Platform{ AgentBareMetal: &agentv1.BareMetalPlatform{}, @@ -355,12 +357,12 @@ func getDefaultClusterDeploymentSpec(secretRef *corev1.LocalObjectReference) *hi Group: hiveext.Group, Version: hiveext.Version, Kind: "AgentClusterInstall", - Name: clusterAgentClusterInstallName, + Name: clusterAgentCLusterInstallNamePrefix + randomNameSuffix(), }, } } -func getDefaultAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { +func getDefaultAgentClusterInstallSpec(clusterDeploymentName string) *hiveext.AgentClusterInstallSpec { return &hiveext.AgentClusterInstallSpec{ Networking: hiveext.Networking{ MachineNetwork: []hiveext.MachineNetworkEntry{}, @@ -382,7 +384,7 @@ func getDefaultAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { } } -func getDefaultSNOAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { +func getDefaultSNOAgentClusterInstallSpec(clusterDeploymentName string) *hiveext.AgentClusterInstallSpec { return &hiveext.AgentClusterInstallSpec{ Networking: hiveext.Networking{ MachineNetwork: []hiveext.MachineNetworkEntry{{CIDR: "1.2.3.0/24"}}, @@ -440,12 +442,15 @@ func getAgentMac(ctx context.Context, client k8sclient.Client, key types.Namespa return mac } +func randomNameSuffix() string { + return fmt.Sprintf("-%s", strings.Split(uuid.New().String(), "-")[0]) +} + func cleanUP(ctx context.Context, client k8sclient.Client) { - Expect(client.DeleteAllOf(ctx, &hivev1.ClusterDeployment{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) + Expect(client.DeleteAllOf(ctx, &hivev1.ClusterDeployment{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) // Should also delete all agents Expect(client.DeleteAllOf(ctx, &hivev1.ClusterImageSet{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &v1beta1.InfraEnv{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &v1beta1.NMStateConfig{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) - Expect(client.DeleteAllOf(ctx, &v1beta1.Agent{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &bmhv1alpha1.BareMetalHost{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) ps := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -462,6 +467,14 @@ func cleanUP(ctx context.Context, client k8sclient.Client) { } func verifyCleanUP(ctx context.Context, client k8sclient.Client) { + By("Verify ClusterDeployment Cleanup") + Eventually(func() int { + clusterDeploymentList := &hivev1.ClusterDeploymentList{} + err := client.List(ctx, clusterDeploymentList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(clusterDeploymentList.Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify AgentClusterInstall Cleanup") Eventually(func() int { aciList := &hiveext.AgentClusterInstallList{} @@ -469,6 +482,46 @@ func verifyCleanUP(ctx context.Context, client k8sclient.Client) { Expect(err).To(BeNil()) return len(aciList.Items) }, "2m", "2s").Should(Equal(0)) + + By("Verify ClusterImageSet Cleanup") + Eventually(func() int { + clusterImageSetList := &hivev1.ClusterImageSetList{} + err := client.List(ctx, clusterImageSetList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(clusterImageSetList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify InfraEnv Cleanup") + Eventually(func() int { + infraEnvList := &v1beta1.InfraEnvList{} + err := client.List(ctx, infraEnvList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(infraEnvList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify NMStateConfig Cleanup") + Eventually(func() int { + nmStateConfigList := &v1beta1.NMStateConfigList{} + err := client.List(ctx, nmStateConfigList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(nmStateConfigList.Items) + }, "2m", "10s").Should(Equal(0)) + + By("Verify Agent Cleanup") + Eventually(func() int { + agentList := &v1beta1.AgentList{} + err := client.List(ctx, agentList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(agentList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify BareMetalHost Cleanup") + Eventually(func() int { + bareMetalHostList := &bmhv1alpha1.BareMetalHostList{} + err := client.List(ctx, bareMetalHostList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(bareMetalHostList.Items) + }, "2m", "2s").Should(Equal(0)) } func setupNewHost(ctx context.Context, hostname string, clusterID strfmt.UUID) *models.Host { @@ -494,17 +547,17 @@ var _ = Describe("[kube-api]cluster installation", func() { clearDB() }) - It("deploy clusterDeployment with agents and wait for ready", func() { + It("deploy CD with ACI and agents - wait for ready, delete CD and verify ACI and agents deletion", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) - key := types.NamespacedName{ + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } - cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) hosts := make([]*models.Host, 0) for i := 0; i < 3; i++ { @@ -516,6 +569,7 @@ var _ = Describe("[kube-api]cluster installation", func() { checkAgentCondition(ctx, host.ID.String(), controllers.ValidatedCondition, controllers.ValidationsFailingReason) } generateFullMeshConnectivity(ctx, "1.2.3.10", hosts...) + By("Approve Agents") for _, host := range hosts { hostkey := types.NamespacedName{ Namespace: Options.Namespace, @@ -529,20 +583,83 @@ var _ = Describe("[kube-api]cluster installation", func() { } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } + By("Verify ClusterDeployment ReadyForInstallation") checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterAlreadyInstallingReason) + By("Delete ClusterDeployment") + err := kubeClient.Delete(ctx, getClusterDeploymentCRD(ctx, kubeClient, clusterKey)) + Expect(err).To(BeNil()) + By("Verify AgentClusterInstall was deleted") + Eventually(func() bool { + aci := &hiveext.AgentClusterInstall{} + err := kubeClient.Get(ctx, installkey, aci) + return apierrors.IsNotFound(err) + }, "30s", "10s").Should(Equal(true)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + }) + + It("deploy CD with ACI and agents - wait for ready, delete ACI only and verify agents deletion", func() { + secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterName, + } + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) + configureLocalAgentClient(cluster.ID.String()) + hosts := make([]*models.Host, 0) + for i := 0; i < 3; i++ { + hostname := fmt.Sprintf("h%d", i) + host := setupNewHost(ctx, hostname, *cluster.ID) + hosts = append(hosts, host) + } + for _, host := range hosts { + checkAgentCondition(ctx, host.ID.String(), controllers.ValidatedCondition, controllers.ValidationsFailingReason) + } + generateFullMeshConnectivity(ctx, "1.2.3.10", hosts...) + By("Approve Agents") + for _, host := range hosts { + hostkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: host.ID.String(), + } + Eventually(func() error { + agent := getAgentCRD(ctx, kubeClient, hostkey) + agent.Spec.Approved = true + return kubeClient.Update(ctx, agent) + }, "30s", "10s").Should(BeNil()) + } + installkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, + } + By("Verify ClusterDeployment ReadyForInstallation") + checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterAlreadyInstallingReason) + By("Delete AgentClusterInstall") + err := kubeClient.Delete(ctx, getAgentClusterInstallCRD(ctx, kubeClient, installkey)) + Expect(err).To(BeNil()) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) }) It("deploy clusterDeployment with agent and update agent", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -579,13 +696,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent,bmh and ignition config override", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -647,13 +764,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and invalid ignition config", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -688,13 +805,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and update installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -753,13 +870,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and invalid installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -814,13 +931,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent,bmh and installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -897,15 +1014,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -947,15 +1064,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) @@ -990,15 +1107,15 @@ var _ = Describe("[kube-api]cluster installation", func() { Name: infraEnvName, } deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1014,15 +1131,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1048,15 +1165,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1078,15 +1195,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1123,15 +1240,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) infraEnvSpec := getDefaultInfraEnvSpec(secretRef, clusterDeploymentSpec) @@ -1164,15 +1281,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) infraEnvSpec := getDefaultInfraEnvSpec(secretRef, clusterDeploymentSpec) @@ -1191,16 +1308,16 @@ var _ = Describe("[kube-api]cluster installation", func() { It("SNO deploy clusterDeployment full install and validate MetaData", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) // Add space suffix to SSHPublicKey to validate proper install sshPublicKeySuffixSpace := fmt.Sprintf("%s ", aciSpec.SSHPublicKey) aciSpec.SSHPublicKey = sshPublicKeySuffixSpace - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1219,7 +1336,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) @@ -1274,13 +1391,13 @@ var _ = Describe("[kube-api]cluster installation", func() { Skip("MGMT-6025 day2 test") By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1314,7 +1431,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) Eventually(func() bool { @@ -1351,6 +1468,11 @@ var _ = Describe("[kube-api]cluster installation", func() { cluster = getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) Expect(*cluster.Kind).Should(Equal(models.ClusterKindAddHostsCluster)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify Cluster Metadata") Eventually(func() bool { return getClusterDeploymentCRD(ctx, kubeClient, clusterKey).Spec.Installed @@ -1378,13 +1500,13 @@ var _ = Describe("[kube-api]cluster installation", func() { Skip("MGMT-6025 day2 test") By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1411,7 +1533,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) Eventually(func() bool { @@ -1444,6 +1566,11 @@ var _ = Describe("[kube-api]cluster installation", func() { cluster = getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) Expect(*cluster.Kind).Should(Equal(models.ClusterKindAddHostsCluster)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + By("Add Day 2 host and approve agent") configureLocalAgentClient(cluster.ID.String()) host := setupNewHost(ctx, "hostnameday2", *cluster.ID) @@ -1468,13 +1595,14 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with invalid machine cidr", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Networking.MachineNetwork = []hiveext.MachineNetworkEntry{{CIDR: "1.2.3.5/24"}} - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) }) @@ -1483,12 +1611,12 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Networking.MachineNetwork = []hiveext.MachineNetworkEntry{} - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) }) @@ -1497,22 +1625,22 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.ImageSetRef.Name = "invalid" - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) }) It("deploy clusterDeployment with missing clusterImageSet", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) // Create AgentClusterInstall without creating the clusterImageSet err := kubeClient.Create(ctx, &hiveext.AgentClusterInstall{ TypeMeta: metav1.TypeMeta{ @@ -1521,14 +1649,14 @@ var _ = Describe("[kube-api]cluster installation", func() { }, ObjectMeta: metav1.ObjectMeta{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, }, Spec: *aciSpec, }) Expect(err).To(BeNil()) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) @@ -1541,15 +1669,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } Eventually(func() bool { aci := getAgentClusterInstallCRD(ctx, kubeClient, installkey) @@ -1570,7 +1698,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) ref := &corev1.LocalObjectReference{Name: "cluster-install-config"} aciSpec.ManifestsConfigMapRef = ref content := `apiVersion: machineconfiguration.openshift.io/v1 @@ -1585,7 +1713,7 @@ spec: By("Start installation without config map") deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, @@ -1605,7 +1733,7 @@ spec: }, "30s", "10s").Should(BeNil()) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterReadyReason) checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) @@ -1635,15 +1763,15 @@ spec: secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout)