diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6440805e784..c1f8f0659f9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -183,6 +183,12 @@ rules: - patch - update - watch +- apiGroups: + - extensions.hive.openshift.io + resources: + - agentclusterinstalls/finalizers + verbs: + - update - apiGroups: - extensions.hive.openshift.io resources: diff --git a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml index 625ca966d87..afdfd3c5f2b 100644 --- a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml @@ -347,6 +347,12 @@ spec: - patch - update - watch + - apiGroups: + - extensions.hive.openshift.io + resources: + - agentclusterinstalls/finalizers + verbs: + - update - apiGroups: - extensions.hive.openshift.io resources: diff --git a/internal/controller/controllers/clusterdeployments_controller.go b/internal/controller/controllers/clusterdeployments_controller.go index d47c7f1571e..eb86e032ddf 100644 --- a/internal/controller/controllers/clusterdeployments_controller.go +++ b/internal/controller/controllers/clusterdeployments_controller.go @@ -32,7 +32,7 @@ import ( "github.com/openshift/assisted-service/internal/cluster" "github.com/openshift/assisted-service/internal/common" hiveext "github.com/openshift/assisted-service/internal/controller/api/hiveextension/v1beta1" - "github.com/openshift/assisted-service/internal/controller/api/v1beta1" + aiv1beta1 "github.com/openshift/assisted-service/internal/controller/api/v1beta1" "github.com/openshift/assisted-service/internal/host" "github.com/openshift/assisted-service/internal/manifests" "github.com/openshift/assisted-service/models" @@ -41,6 +41,7 @@ import ( hivev1 "github.com/openshift/hive/apis/hive/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/thoas/go-funk" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -59,7 +60,9 @@ import ( const ( adminPasswordSecretStringTemplate = "%s-admin-password" adminKubeConfigStringTemplate = "%s-admin-kubeconfig" - InstallConfigOverrides = v1beta1.Group + "/install-config-overrides" + InstallConfigOverrides = aiv1beta1.Group + "/install-config-overrides" + ClusterDeploymentFinalizerName = "clusterdeployments." + aiv1beta1.Group + "/deprovision" + AgentClusterInstallFinalizerName = "agentclusterinstall." + aiv1beta1.Group + "/deprovision" ) const HighAvailabilityModeNone = "None" @@ -85,18 +88,17 @@ type ClusterDeploymentsReconciler struct { // +kubebuilder:rbac:groups=hive.openshift.io,resources=clusterimagesets,verbs=get;list;watch // +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=extensions.hive.openshift.io,resources=agentclusterinstalls/finalizers,verbs=update func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log.Infof("Reconcile has been called for ClusterDeployment name=%s namespace=%s", req.Name, req.Namespace) clusterDeployment := &hivev1.ClusterDeployment{} - err := r.Get(ctx, req.NamespacedName, clusterDeployment) - if err != nil { - if k8serrors.IsNotFound(err) { - return r.deregisterClusterIfNeeded(ctx, req.NamespacedName) - } - r.Log.WithError(err).Errorf("Failed to get resource %s", req.NamespacedName) - return ctrl.Result{Requeue: true}, nil + clusterInstallDeleted := false + + if err := r.Get(ctx, req.NamespacedName, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to get ClusterDeployment name=%s namespace=%s", req.Name, req.Namespace) + return ctrl.Result{}, client.IgnoreNotFound(err) } // ignore unsupported platforms @@ -111,7 +113,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R } aciName := clusterDeployment.Spec.ClusterInstallRef.Name - err = r.Get(ctx, + err := r.Get(ctx, types.NamespacedName{ Namespace: clusterDeployment.Namespace, Name: aciName, @@ -119,11 +121,29 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R clusterInstall) if err != nil { if k8serrors.IsNotFound(err) { + // mark that clusterInstall was already deleted so we skip it if needed. + clusterInstallDeleted = true r.Log.WithField("AgentClusterInstall", aciName).Infof("AgentClusterInstall does not exist for ClusterDeployment %s", clusterDeployment.Name) - return ctrl.Result{}, nil + if clusterDeployment.ObjectMeta.DeletionTimestamp.IsZero() { + // we have no agentClusterInstall and clusterDeployment is not being deleted. stop reconciliation. + return ctrl.Result{}, nil + } + } else { + r.Log.WithError(err).Errorf("failed to get AgentClusterInstall name=%s namespace=%s", aciName, clusterDeployment.Namespace) + return ctrl.Result{Requeue: true}, err + } + } + + if !clusterInstallDeleted { + aciReply, aciErr := r.agentClusterInstallFinalizer(ctx, req, clusterInstall) + if aciReply != nil { + return *aciReply, aciErr } - r.Log.WithError(err).Errorf("Failed to get AgentClusterInstall %s", aciName) - return ctrl.Result{Requeue: true}, err + } + + cdReplay, cdErr := r.clusterDeploymentFinalizer(ctx, clusterDeployment) + if cdReplay != nil { + return *cdReplay, cdErr } err = r.ensureOwnerRef(ctx, clusterDeployment, clusterInstall) @@ -139,7 +159,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R if !r.isSNO(clusterInstall) { return r.createNewDay2Cluster(ctx, req.NamespacedName, clusterDeployment, clusterInstall) } - // cluster is installed and SNO nothing to do + // cluster is installed and SNO nothing to do return ctrl.Result{Requeue: false}, nil } if err != nil { @@ -166,9 +186,7 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R return r.updateStatus(ctx, clusterInstall, cluster, err) } else { // Delete Day1 Cluster - err = r.Installer.DeregisterClusterInternal(ctx, installer.DeregisterClusterParams{ - ClusterID: *cluster.ID, - }) + _, err = r.deregisterClusterIfNeeded(ctx, req.NamespacedName) if err != nil { return r.updateStatus(ctx, clusterInstall, cluster, err) } @@ -192,6 +210,78 @@ func (r *ClusterDeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.R return r.updateStatus(ctx, clusterInstall, cluster, nil) } +func (r *ClusterDeploymentsReconciler) agentClusterInstallFinalizer(ctx context.Context, req ctrl.Request, + clusterInstall *hiveext.AgentClusterInstall) (*ctrl.Result, error) { + if clusterInstall.ObjectMeta.DeletionTimestamp.IsZero() { // clusterInstall not being deleted + // Register a finalizer if it is absent. + if !funk.ContainsString(clusterInstall.GetFinalizers(), AgentClusterInstallFinalizerName) { + controllerutil.AddFinalizer(clusterInstall, AgentClusterInstallFinalizerName) + if err := r.Update(ctx, clusterInstall); err != nil { + r.Log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + } else { // clusterInstall is being deleted + if funk.ContainsString(clusterInstall.GetFinalizers(), AgentClusterInstallFinalizerName) { + // deletion finalizer found, deregister the backend cluster and delete agents + reply, cleanUpErr := r.deregisterClusterIfNeeded(ctx, req.NamespacedName) + if cleanUpErr != nil { + r.Log.WithError(cleanUpErr).Errorf("failed to run pre-deletion cleanup for finalizer %s on resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &reply, cleanUpErr + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(clusterInstall, AgentClusterInstallFinalizerName) + if err := r.Update(ctx, clusterInstall); err != nil { + r.Log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", + AgentClusterInstallFinalizerName, clusterInstall.Name, clusterInstall.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + // Stop reconciliation as the item is being deleted + return &ctrl.Result{}, nil + } + return nil, nil +} + +func (r *ClusterDeploymentsReconciler) clusterDeploymentFinalizer(ctx context.Context, clusterDeployment *hivev1.ClusterDeployment) (*ctrl.Result, error) { + if clusterDeployment.ObjectMeta.DeletionTimestamp.IsZero() { // clusterDeployment not being deleted + // Register a finalizer if it is absent. + if !funk.ContainsString(clusterDeployment.GetFinalizers(), ClusterDeploymentFinalizerName) { + controllerutil.AddFinalizer(clusterDeployment, ClusterDeploymentFinalizerName) + if err := r.Update(ctx, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", + ClusterDeploymentFinalizerName, clusterDeployment.Name, clusterDeployment.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + } else { // clusterDeployment is being deleted + if funk.ContainsString(clusterDeployment.GetFinalizers(), ClusterDeploymentFinalizerName) { + reply, cleanUpErr := r.deleteClusterInstall(ctx, clusterDeployment) + if cleanUpErr != nil { + r.Log.WithError(cleanUpErr).Errorf( + "clusterDeployment %s %s is still waiting for clusterInstall %s to be deleted", + clusterDeployment.Name, clusterDeployment.Namespace, clusterDeployment.Spec.ClusterInstallRef.Name, + ) + return &reply, cleanUpErr + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(clusterDeployment, ClusterDeploymentFinalizerName) + if err := r.Update(ctx, clusterDeployment); err != nil { + r.Log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", + ClusterDeploymentFinalizerName, clusterDeployment.Name, clusterDeployment.Namespace) + return &ctrl.Result{Requeue: true}, err + } + } + // Stop reconciliation as the item is being deleted + return &ctrl.Result{}, nil + } + return nil, nil +} + func isInstalled(clusterDeployment *hivev1.ClusterDeployment, clusterInstall *hiveext.AgentClusterInstall) bool { if clusterDeployment.Spec.Installed { return true @@ -388,7 +478,7 @@ func (r *ClusterDeploymentsReconciler) updateIfNeeded(ctx context.Context, update := false notifyInfraEnv := false - var infraEnv *v1beta1.InfraEnv + var infraEnv *aiv1beta1.InfraEnv params := &models.ClusterUpdateParams{} @@ -715,12 +805,71 @@ func (r *ClusterDeploymentsReconciler) deregisterClusterIfNeeded(ctx context.Con }); err != nil { return buildReply(err) } + // Delete agents because their backend cluster got deregistered. + if err = r.DeleteClusterDeploymentAgents(ctx, key); err != nil { + return buildReply(err) + } r.Log.Infof("Cluster resource deleted, Unregistered cluster: %s", c.ID.String()) return buildReply(nil) } +func (r *ClusterDeploymentsReconciler) deleteClusterInstall(ctx context.Context, clusterDeployment *hivev1.ClusterDeployment) (ctrl.Result, error) { + + buildReply := func(err error) (ctrl.Result, error) { + reply := ctrl.Result{} + if err == nil { + return reply, nil + } + reply.RequeueAfter = defaultRequeueAfterOnError + err = errors.Wrapf(err, "clusterInstall: %s not deleted", clusterDeployment.Spec.ClusterInstallRef.Name) + r.Log.Error(err) + return reply, err + } + + clusterInstall := &hiveext.AgentClusterInstall{} + err := r.Get(ctx, + types.NamespacedName{ + Name: clusterDeployment.Spec.ClusterInstallRef.Name, + Namespace: clusterDeployment.Namespace, + }, + clusterInstall) + + if err != nil { + if k8serrors.IsNotFound(err) { + return buildReply(nil) + } + return buildReply(err) + } + + if err = r.Delete(ctx, clusterInstall); err != nil { + return buildReply(err) + } + // place this err so we requeue and verify deletion + err = errors.Errorf("could not confirm clusterInstall %s deletion was successfuly completed", clusterDeployment.Spec.ClusterInstallRef.Name) + return buildReply(err) +} + +func (r *ClusterDeploymentsReconciler) DeleteClusterDeploymentAgents(ctx context.Context, clusterDeployment types.NamespacedName) error { + agents := &aiv1beta1.AgentList{} + r.Log = r.Log.WithFields(logrus.Fields{"clusterDeployment": clusterDeployment.Name, "namespace": clusterDeployment.Namespace}) + if err := r.List(ctx, agents); err != nil { + return err + } + for i, clusterAgent := range agents.Items { + if clusterAgent.Spec.ClusterDeploymentName.Name == clusterDeployment.Name && + clusterAgent.Spec.ClusterDeploymentName.Namespace == clusterDeployment.Namespace { + r.Log.Infof("delete agent %s namespace %s", clusterAgent.Name, clusterAgent.Namespace) + if err := r.Client.Delete(ctx, &agents.Items[i]); err != nil { + r.Log.WithError(err).Errorf("Failed to delete resource %s %s", clusterAgent.Name, clusterAgent.Namespace) + return err + } + } + } + return nil +} + func (r *ClusterDeploymentsReconciler) SetupWithManager(mgr ctrl.Manager) error { mapSecretToClusterDeployment := func(a client.Object) []reconcile.Request { clusterDeployments := &hivev1.ClusterDeploymentList{} diff --git a/internal/controller/controllers/clusterdeployments_controller_test.go b/internal/controller/controllers/clusterdeployments_controller_test.go index 7ded3138b1e..340178ac486 100644 --- a/internal/controller/controllers/clusterdeployments_controller_test.go +++ b/internal/controller/controllers/clusterdeployments_controller_test.go @@ -136,6 +136,18 @@ func getDefaultClusterDeploymentSpec(clusterName, aciName, pullSecretName string } } +func kubeTimeNow() *metav1.Time { + t := metav1.NewTime(time.Now()) + return &t +} + +func simulateACIDeletionWithFinalizer(ctx context.Context, c client.Client, aci *hiveext.AgentClusterInstall) { + // simulate ACI deletion with finalizer + aci.ObjectMeta.Finalizers = []string{AgentClusterInstallFinalizerName} + aci.ObjectMeta.DeletionTimestamp = kubeTimeNow() + Expect(c.Update(ctx, aci)).Should(BeNil()) +} + var _ = Describe("cluster reconcile", func() { var ( c client.Client @@ -435,23 +447,45 @@ var _ = Describe("cluster reconcile", func() { Context("cluster deletion", func() { var ( - sId strfmt.UUID - cluster *hivev1.ClusterDeployment + sId strfmt.UUID + cd *hivev1.ClusterDeployment + aci *hiveext.AgentClusterInstall ) BeforeEach(func() { - cluster = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) + defaultClusterSpec = getDefaultClusterDeploymentSpec(clusterName, agentClusterInstallName, pullSecretName) + cd = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) + cd.Status = hivev1.ClusterDeploymentStatus{} + defaultAgentClusterInstallSpec = getDefaultAgentClusterInstallSpec(clusterName) + aci = newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cd) id := uuid.New() sId = strfmt.UUID(id.String()) - cluster.Status = hivev1.ClusterDeploymentStatus{} - Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) + c = fakeclient.NewClientBuilder().WithScheme(scheme.Scheme).Build() + mockCtrl = gomock.NewController(GinkgoT()) + mockInstallerInternal = bminventory.NewMockInstallerInternals(mockCtrl) + mockClusterApi = cluster.NewMockAPI(mockCtrl) + mockHostApi = host.NewMockAPI(mockCtrl) + mockCRDEventsHandler = NewMockCRDEventsHandler(mockCtrl) + mockManifestsApi = manifests.NewMockClusterManifestsInternals(mockCtrl) + cr = &ClusterDeploymentsReconciler{ + Client: c, + Scheme: scheme.Scheme, + Log: common.GetTestLog(), + Installer: mockInstallerInternal, + ClusterApi: mockClusterApi, + HostApi: mockHostApi, + CRDEventsHandler: mockCRDEventsHandler, + Manifests: mockManifestsApi, + } + Expect(c.Create(ctx, cd)).ShouldNot(HaveOccurred()) + Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) pullSecret := getDefaultTestPullSecret("pull-secret", testNamespace) Expect(c.Create(ctx, pullSecret)).To(BeNil()) imageSet := getDefaultTestImageSet(imageSetName, releaseImage) Expect(c.Create(ctx, imageSet)).To(BeNil()) }) - It("cluster resource deleted - verify call to deregister cluster", func() { + It("agentClusterInstall resource deleted - verify call to deregister cluster", func() { backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -460,8 +494,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).ShouldNot(HaveOccurred()) Expect(result).Should(Equal(ctrl.Result{})) @@ -473,20 +507,21 @@ var _ = Describe("cluster reconcile", func() { ID: &sId, }, } - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(errors.New("internal error")) - expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: internal error", cluster.Name) + expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: internal error", cd.Name) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + Expect(c.Update(ctx, aci)).Should(BeNil()) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).Should(HaveOccurred()) Expect(err.Error()).Should(Equal(expectedErrMsg)) Expect(result).Should(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) }) - It("cluster resource deleted and created again", func() { + It("agentClusterInstall resource deleted and created again", func() { backEndCluster := &common.Cluster{ Cluster: models.Cluster{ ID: &sId, @@ -496,8 +531,8 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil) mockInstallerInternal.EXPECT().AddOpenshiftVersion(gomock.Any(), gomock.Any(), gomock.Any()).Return(openshiftVersion, nil) - Expect(c.Delete(ctx, cluster)).ShouldNot(HaveOccurred()) - request := newClusterDeploymentRequest(cluster) + simulateACIDeletionWithFinalizer(ctx, c, aci) + request := newClusterDeploymentRequest(cd) result, err := cr.Reconcile(ctx, request) Expect(err).ShouldNot(HaveOccurred()) Expect(result).Should(Equal(ctrl.Result{})) @@ -505,12 +540,11 @@ var _ = Describe("cluster reconcile", func() { mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(nil, gorm.ErrRecordNotFound) mockInstallerInternal.EXPECT().RegisterClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()).Return(backEndCluster, nil) - cluster = newClusterDeployment(clusterName, testNamespace, defaultClusterSpec) - Expect(c.Create(ctx, cluster)).ShouldNot(HaveOccurred()) - aci := newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cluster) + Expect(c.Delete(ctx, aci)).ShouldNot(HaveOccurred()) + aci = newAgentClusterInstall(agentClusterInstallName, testNamespace, defaultAgentClusterInstallSpec, cd) Expect(c.Create(ctx, aci)).ShouldNot(HaveOccurred()) - request = newClusterDeploymentRequest(cluster) + request = newClusterDeploymentRequest(cd) result, err = cr.Reconcile(ctx, request) Expect(err).To(BeNil()) Expect(result).To(Equal(ctrl.Result{})) @@ -599,7 +633,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(3) password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -645,7 +679,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.StatusInfo = swag.String("Done") backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(3) password := "test" username := "admin" kubeconfig := "kubeconfig content" @@ -691,8 +725,9 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) expectedError := errors.New("internal error") + expectedErrMsg := fmt.Sprintf("failed to deregister cluster: %s: %s", cluster.Name, expectedError) mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(expectedError).Times(1) setClusterCondition(&aci.Status.Conditions, hivev1.ClusterInstallCondition{ Type: ClusterCompletedCondition, @@ -707,7 +742,7 @@ var _ = Describe("cluster reconcile", func() { Expect(result).To(Equal(ctrl.Result{RequeueAfter: defaultRequeueAfterOnError})) aci = getTestClusterInstall() - expectedState := fmt.Sprintf("%s %s", BackendErrorMsg, expectedError) + expectedState := fmt.Sprintf("%s %s", BackendErrorMsg, expectedErrMsg) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Reason).To(Equal(BackendErrorReason)) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Status).To(Equal(corev1.ConditionFalse)) Expect(FindStatusCondition(aci.Status.Conditions, ClusterSpecSyncedCondition).Message).To(Equal(expectedState)) @@ -720,7 +755,7 @@ var _ = Describe("cluster reconcile", func() { backEndCluster.Status = swag.String(models.ClusterStatusInstalled) backEndCluster.OpenshiftClusterID = openshiftID backEndCluster.Kind = swag.String(models.ClusterKindCluster) - mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(1) + mockInstallerInternal.EXPECT().GetClusterByKubeKey(gomock.Any()).Return(backEndCluster, nil).Times(2) expectedErr := "internal error" mockInstallerInternal.EXPECT().DeregisterClusterInternal(gomock.Any(), gomock.Any()).Return(nil).Times(1) mockInstallerInternal.EXPECT().RegisterAddHostsClusterInternal(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(expectedErr)) diff --git a/subsystem/kubeapi_test.go b/subsystem/kubeapi_test.go index 6cfe27d1f32..bdab96051bd 100644 --- a/subsystem/kubeapi_test.go +++ b/subsystem/kubeapi_test.go @@ -11,6 +11,7 @@ import ( "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" + "github.com/google/uuid" "github.com/jinzhu/gorm" bmhv1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" . "github.com/onsi/ginkgo" @@ -36,10 +37,10 @@ import ( ) const ( - fakeIgnitionConfigOverride = `{"ignition": {"version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}` - badIgnitionConfigOverride = `bad ignition config` - clusterDeploymentName = "test-cluster" - clusterAgentClusterInstallName = "test-agent-cluster-install" + fakeIgnitionConfigOverride = `{"ignition": {"version": "3.1.0"}, "storage": {"files": [{"path": "/tmp/example", "contents": {"source": "data:text/plain;base64,aGVscGltdHJhcHBlZGluYXN3YWdnZXJzcGVj"}}]}}` + badIgnitionConfigOverride = `bad ignition config` + clusterDeploymentNamePrefix = "test-cluster" + clusterAgentCLusterInstallNamePrefix = "test-agent-cluster-install" ) var ( @@ -109,7 +110,8 @@ func deployPullSecretResource(ctx context.Context, client k8sclient.Client, name Expect(client.Create(ctx, s)).To(BeNil()) } -func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, spec *hiveext.AgentClusterInstallSpec) { +func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, spec *hiveext.AgentClusterInstallSpec, + clusterAgentCLusterInstallName string) { deployClusterImageSetCRD(ctx, client, spec.ImageSetRef) err := client.Create(ctx, &hiveext.AgentClusterInstall{ TypeMeta: metav1.TypeMeta{ @@ -118,7 +120,7 @@ func deployAgentClusterInstallCRD(ctx context.Context, client k8sclient.Client, }, ObjectMeta: metav1.ObjectMeta{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterAgentCLusterInstallName, }, Spec: *spec, }) @@ -345,7 +347,7 @@ func checkInfraEnvCondition(ctx context.Context, key types.NamespacedName, condi func getDefaultClusterDeploymentSpec(secretRef *corev1.LocalObjectReference) *hivev1.ClusterDeploymentSpec { return &hivev1.ClusterDeploymentSpec{ - ClusterName: clusterDeploymentName, + ClusterName: clusterDeploymentNamePrefix + randomNameSuffix(), BaseDomain: "hive.example.com", Platform: hivev1.Platform{ AgentBareMetal: &agentv1.BareMetalPlatform{}, @@ -355,12 +357,12 @@ func getDefaultClusterDeploymentSpec(secretRef *corev1.LocalObjectReference) *hi Group: hiveext.Group, Version: hiveext.Version, Kind: "AgentClusterInstall", - Name: clusterAgentClusterInstallName, + Name: clusterAgentCLusterInstallNamePrefix + randomNameSuffix(), }, } } -func getDefaultAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { +func getDefaultAgentClusterInstallSpec(clusterDeploymentName string) *hiveext.AgentClusterInstallSpec { return &hiveext.AgentClusterInstallSpec{ Networking: hiveext.Networking{ MachineNetwork: []hiveext.MachineNetworkEntry{}, @@ -382,7 +384,7 @@ func getDefaultAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { } } -func getDefaultSNOAgentClusterInstallSpec() *hiveext.AgentClusterInstallSpec { +func getDefaultSNOAgentClusterInstallSpec(clusterDeploymentName string) *hiveext.AgentClusterInstallSpec { return &hiveext.AgentClusterInstallSpec{ Networking: hiveext.Networking{ MachineNetwork: []hiveext.MachineNetworkEntry{{CIDR: "1.2.3.0/24"}}, @@ -440,12 +442,15 @@ func getAgentMac(ctx context.Context, client k8sclient.Client, key types.Namespa return mac } +func randomNameSuffix() string { + return fmt.Sprintf("-%s", strings.Split(uuid.New().String(), "-")[0]) +} + func cleanUP(ctx context.Context, client k8sclient.Client) { - Expect(client.DeleteAllOf(ctx, &hivev1.ClusterDeployment{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) + Expect(client.DeleteAllOf(ctx, &hivev1.ClusterDeployment{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) // Should also delete all agents Expect(client.DeleteAllOf(ctx, &hivev1.ClusterImageSet{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &v1beta1.InfraEnv{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &v1beta1.NMStateConfig{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) - Expect(client.DeleteAllOf(ctx, &v1beta1.Agent{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) Expect(client.DeleteAllOf(ctx, &bmhv1alpha1.BareMetalHost{}, k8sclient.InNamespace(Options.Namespace))).To(BeNil()) ps := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -462,6 +467,14 @@ func cleanUP(ctx context.Context, client k8sclient.Client) { } func verifyCleanUP(ctx context.Context, client k8sclient.Client) { + By("Verify ClusterDeployment Cleanup") + Eventually(func() int { + clusterDeploymentList := &hivev1.ClusterDeploymentList{} + err := client.List(ctx, clusterDeploymentList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(clusterDeploymentList.Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify AgentClusterInstall Cleanup") Eventually(func() int { aciList := &hiveext.AgentClusterInstallList{} @@ -469,6 +482,46 @@ func verifyCleanUP(ctx context.Context, client k8sclient.Client) { Expect(err).To(BeNil()) return len(aciList.Items) }, "2m", "2s").Should(Equal(0)) + + By("Verify ClusterImageSet Cleanup") + Eventually(func() int { + clusterImageSetList := &hivev1.ClusterImageSetList{} + err := client.List(ctx, clusterImageSetList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(clusterImageSetList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify InfraEnv Cleanup") + Eventually(func() int { + infraEnvList := &v1beta1.InfraEnvList{} + err := client.List(ctx, infraEnvList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(infraEnvList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify NMStateConfig Cleanup") + Eventually(func() int { + nmStateConfigList := &v1beta1.NMStateConfigList{} + err := client.List(ctx, nmStateConfigList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(nmStateConfigList.Items) + }, "2m", "10s").Should(Equal(0)) + + By("Verify Agent Cleanup") + Eventually(func() int { + agentList := &v1beta1.AgentList{} + err := client.List(ctx, agentList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(agentList.Items) + }, "2m", "2s").Should(Equal(0)) + + By("Verify BareMetalHost Cleanup") + Eventually(func() int { + bareMetalHostList := &bmhv1alpha1.BareMetalHostList{} + err := client.List(ctx, bareMetalHostList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(bareMetalHostList.Items) + }, "2m", "2s").Should(Equal(0)) } func setupNewHost(ctx context.Context, hostname string, clusterID strfmt.UUID) *models.Host { @@ -494,17 +547,17 @@ var _ = Describe("[kube-api]cluster installation", func() { clearDB() }) - It("deploy clusterDeployment with agents and wait for ready", func() { + It("deploy CD with ACI and agents - wait for ready, delete CD and verify ACI and agents deletion", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) - key := types.NamespacedName{ + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } - cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) hosts := make([]*models.Host, 0) for i := 0; i < 3; i++ { @@ -516,6 +569,7 @@ var _ = Describe("[kube-api]cluster installation", func() { checkAgentCondition(ctx, host.ID.String(), controllers.ValidatedCondition, controllers.ValidationsFailingReason) } generateFullMeshConnectivity(ctx, "1.2.3.10", hosts...) + By("Approve Agents") for _, host := range hosts { hostkey := types.NamespacedName{ Namespace: Options.Namespace, @@ -529,20 +583,83 @@ var _ = Describe("[kube-api]cluster installation", func() { } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } + By("Verify ClusterDeployment ReadyForInstallation") checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterAlreadyInstallingReason) + By("Delete ClusterDeployment") + err := kubeClient.Delete(ctx, getClusterDeploymentCRD(ctx, kubeClient, clusterKey)) + Expect(err).To(BeNil()) + By("Verify AgentClusterInstall was deleted") + Eventually(func() bool { + aci := &hiveext.AgentClusterInstall{} + err := kubeClient.Get(ctx, installkey, aci) + return apierrors.IsNotFound(err) + }, "30s", "10s").Should(Equal(true)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + }) + + It("deploy CD with ACI and agents - wait for ready, delete ACI only and verify agents deletion", func() { + secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) + clusterKey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterName, + } + cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) + configureLocalAgentClient(cluster.ID.String()) + hosts := make([]*models.Host, 0) + for i := 0; i < 3; i++ { + hostname := fmt.Sprintf("h%d", i) + host := setupNewHost(ctx, hostname, *cluster.ID) + hosts = append(hosts, host) + } + for _, host := range hosts { + checkAgentCondition(ctx, host.ID.String(), controllers.ValidatedCondition, controllers.ValidationsFailingReason) + } + generateFullMeshConnectivity(ctx, "1.2.3.10", hosts...) + By("Approve Agents") + for _, host := range hosts { + hostkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: host.ID.String(), + } + Eventually(func() error { + agent := getAgentCRD(ctx, kubeClient, hostkey) + agent.Spec.Approved = true + return kubeClient.Update(ctx, agent) + }, "30s", "10s").Should(BeNil()) + } + installkey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, + } + By("Verify ClusterDeployment ReadyForInstallation") + checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterAlreadyInstallingReason) + By("Delete AgentClusterInstall") + err := kubeClient.Delete(ctx, getAgentClusterInstallCRD(ctx, kubeClient, installkey)) + Expect(err).To(BeNil()) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) }) It("deploy clusterDeployment with agent and update agent", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -579,13 +696,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent,bmh and ignition config override", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -647,13 +764,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and invalid ignition config", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -688,13 +805,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and update installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -753,13 +870,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent and invalid installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -814,13 +931,13 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with agent,bmh and installer args", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) key := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, key, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -897,15 +1014,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -947,15 +1064,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) @@ -990,15 +1107,15 @@ var _ = Describe("[kube-api]cluster installation", func() { Name: infraEnvName, } deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1014,15 +1131,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1048,15 +1165,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1078,15 +1195,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKubeName, waitForReconcileTimeout) @@ -1123,15 +1240,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) infraEnvSpec := getDefaultInfraEnvSpec(secretRef, clusterDeploymentSpec) @@ -1164,15 +1281,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKubeName := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) infraEnvSpec := getDefaultInfraEnvSpec(secretRef, clusterDeploymentSpec) @@ -1191,16 +1308,16 @@ var _ = Describe("[kube-api]cluster installation", func() { It("SNO deploy clusterDeployment full install and validate MetaData", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) // Add space suffix to SSHPublicKey to validate proper install sshPublicKeySuffixSpace := fmt.Sprintf("%s ", aciSpec.SSHPublicKey) aciSpec.SSHPublicKey = sshPublicKeySuffixSpace - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1219,7 +1336,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) @@ -1274,13 +1391,13 @@ var _ = Describe("[kube-api]cluster installation", func() { Skip("MGMT-6025 day2 test") By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1314,7 +1431,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) Eventually(func() bool { @@ -1351,6 +1468,11 @@ var _ = Describe("[kube-api]cluster installation", func() { cluster = getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) Expect(*cluster.Kind).Should(Equal(models.ClusterKindAddHostsCluster)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify Cluster Metadata") Eventually(func() bool { return getClusterDeploymentCRD(ctx, kubeClient, clusterKey).Spec.Installed @@ -1378,13 +1500,13 @@ var _ = Describe("[kube-api]cluster installation", func() { Skip("MGMT-6025 day2 test") By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) - aciSpec := getDefaultAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, - Name: spec.ClusterName, + Name: clusterDeploymentSpec.ClusterName, } cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) configureLocalAgentClient(cluster.ID.String()) @@ -1411,7 +1533,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Wait for installing") installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterCompletedCondition, controllers.InstallationInProgressReason) Eventually(func() bool { @@ -1444,6 +1566,11 @@ var _ = Describe("[kube-api]cluster installation", func() { cluster = getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout) Expect(*cluster.Kind).Should(Equal(models.ClusterKindAddHostsCluster)) + By("Verify ClusterDeployment Agents were deleted") + Eventually(func() int { + return len(getClusterDeploymentAgents(ctx, kubeClient, clusterKey).Items) + }, "2m", "2s").Should(Equal(0)) + By("Add Day 2 host and approve agent") configureLocalAgentClient(cluster.ID.String()) host := setupNewHost(ctx, "hostnameday2", *cluster.ID) @@ -1468,13 +1595,14 @@ var _ = Describe("[kube-api]cluster installation", func() { It("deploy clusterDeployment with invalid machine cidr", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Networking.MachineNetwork = []hiveext.MachineNetworkEntry{{CIDR: "1.2.3.5/24"}} - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) }) @@ -1483,12 +1611,12 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.Networking.MachineNetwork = []hiveext.MachineNetworkEntry{} - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) }) @@ -1497,22 +1625,22 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) aciSpec.ImageSetRef.Name = "invalid" - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) }) It("deploy clusterDeployment with missing clusterImageSet", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) - spec := getDefaultClusterDeploymentSpec(secretRef) - deployClusterDeploymentCRD(ctx, kubeClient, spec) + clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) + deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultAgentClusterInstallSpec() + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) // Create AgentClusterInstall without creating the clusterImageSet err := kubeClient.Create(ctx, &hiveext.AgentClusterInstall{ TypeMeta: metav1.TypeMeta{ @@ -1521,14 +1649,14 @@ var _ = Describe("[kube-api]cluster installation", func() { }, ObjectMeta: metav1.ObjectMeta{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, }, Spec: *aciSpec, }) Expect(err).To(BeNil()) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) @@ -1541,15 +1669,15 @@ var _ = Describe("[kube-api]cluster installation", func() { secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } Eventually(func() bool { aci := getAgentClusterInstallCRD(ctx, kubeClient, installkey) @@ -1570,7 +1698,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Create cluster") secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) - aciSpec := getDefaultSNOAgentClusterInstallSpec() + aciSpec := getDefaultSNOAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) ref := &corev1.LocalObjectReference{Name: "cluster-install-config"} aciSpec.ManifestsConfigMapRef = ref content := `apiVersion: machineconfiguration.openshift.io/v1 @@ -1585,7 +1713,7 @@ spec: By("Start installation without config map") deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, @@ -1605,7 +1733,7 @@ spec: }, "30s", "10s").Should(BeNil()) installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterReadyReason) checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterSpecSyncedCondition, controllers.BackendErrorReason) @@ -1635,15 +1763,15 @@ spec: secretRef := deployLocalObjectSecretIfNeeded(ctx, kubeClient) clusterDeploymentSpec := getDefaultClusterDeploymentSpec(secretRef) deployClusterDeploymentCRD(ctx, kubeClient, clusterDeploymentSpec) - aciSpec := getDefaultSNOAgentClusterInstallSpec() - deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec) + aciSpec := getDefaultAgentClusterInstallSpec(clusterDeploymentSpec.ClusterName) + deployAgentClusterInstallCRD(ctx, kubeClient, aciSpec, clusterDeploymentSpec.ClusterInstallRef.Name) clusterKey := types.NamespacedName{ Namespace: Options.Namespace, Name: clusterDeploymentSpec.ClusterName, } installkey := types.NamespacedName{ Namespace: Options.Namespace, - Name: clusterAgentClusterInstallName, + Name: clusterDeploymentSpec.ClusterInstallRef.Name, } checkAgentClusterInstallCondition(ctx, installkey, controllers.ClusterRequirementsMetCondition, controllers.ClusterNotReadyReason) cluster := getClusterFromDB(ctx, kubeClient, db, clusterKey, waitForReconcileTimeout)