diff --git a/docs/multicluster/architecture.md b/docs/multicluster/architecture.md index ed4726ebe75..dee62cea984 100644 --- a/docs/multicluster/architecture.md +++ b/docs/multicluster/architecture.md @@ -90,88 +90,16 @@ IPs from all member clusters. The new created Antrea Multi-cluster Service is ju Kubernetes Service, so Pods in a member cluster can access the multi-cluster Service as usual without any extra setting. -## Multi-cluster ClusterNetworkPolicy Replication (ACNP Copy-span) - -Antrea Multi-cluster admins can specify certain ClusterNetworkPolicies to be replicated across -the entire ClusterSet. This is especially useful for ClusterSet admins who want all clusters in the -ClusterSet to be applied with a consistent security posture (for example, all namespaces in all -clusters can only communicate with Pods in their own namespaces). For more information regarding -Antrea ClusterNetworkPolicy(ACNP), refer to [this document](../antrea-network-policy.md). - -To achieve such ACNP copy-span, admins can, in the acting leader cluster of a Multi-cluster deployment, -create a ResourceExport of kind `AntreaClusterNetworkPolicy` which contains the ClusterNetworkPolicy spec -they wish to be replicated. The ResourceExport should be created in the Namespace which implements the -Common Area of the ClusterSet. In future releases, some additional tooling may become available to -automate the creation of such ResourceExport and make ACNP replication across cluster eaiser. - -```yaml -apiVersion: multicluster.crd.antrea.io/v1alpha1 -kind: ResourceExport -metadata: - name: strict-namespace-isolation-for-test-clusterset - namespace: antrea-mcs-ns # Namespace that implements Common Area of test-clusterset -spec: - kind: AntreaClusterNetworkPolicy - name: strict-namespace-isolation # In each importing cluster, an ACNP of name antrea-mc-strict-namespace-isolation will be created with the spec below - clusternetworkpolicy: - priority: 1 - tier: securityops - appliedTo: - - namespaceSelector: {} # Selects all Namespaces in the member cluster - ingress: - - action: Pass - from: - - namespaces: - match: Self # Skip drop rule for traffic from Pods in the same Namespace - - podSelector: - matchLabels: - k8s-app: kube-dns # Skip drop rule for traffic from the core-dns components - - action: Drop - from: - - namespaceSelector: {} # Drop from Pods from all other Namespaces -``` - -The above sample spec will create an ACNP in each member cluster which implements strict namespace -isolation for that cluster. - -Note that because the Tier that an ACNP refers to must exist before the ACNP is applied, an importing -cluster may fail to create the ACNP to be replicated, if the tier in the ResourceExport spec cannot be -found in that particular cluster. The ACNP creation status of each member cluster will be reported back -to the Common Area as K8s Events, and can be checked by describing the ResourceImport of the original -ResourceExport: - -```text -kubectl describe resourceimport -A ---- -Name: strict-namespace-isolation-antreaclusternetworkpolicy -Namespace: antrea-mcs-ns -API Version: multicluster.crd.antrea.io/v1alpha1 -Kind: ResourceImport -Spec: - Clusternetworkpolicy: - Applied To: - Namespace Selector: - Ingress: - Action: Pass - Enable Logging: false - From: - Namespaces: - Match: Self - Pod Selector: - Match Labels: - k8s-app: kube-dns - Action: Drop - Enable Logging: false - From: - Namespace Selector: - Priority: 1 - Tier: random - Kind: AntreaClusterNetworkPolicy - Name: strict-namespace-isolation - ... -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal ACNPImportSucceeded 2m11s resourceimport-controller ACNP successfully created in the importing cluster test-cluster-east - Warning ACNPImportFailed 2m11s resourceimport-controller ACNP Tier does not exist in the importing cluster test-cluster-west -``` +## Antrea Multi-cluster policy enforcement + +At this moment, Antrea does not support Pod-level policy enforcement for cross-cluster traffic. Access +towards Multi-cluster Services can be regulated with Antrea ClusterNetworkPolicy `toService` rules. In +each member cluster, users can create an Antrea ClusterNetworkPolicy selecting Pods in that cluster, with +the imported Mutli-cluster Service name and Namespace in an egress `toService` rule, and the Action to +take for traffic matching this rule. For more information regarding Antrea ClusterNetworkPolicy (ACNP), +refer to [this document](../antrea-network-policy.md). + +Multi-cluster admins can also specify certain ClusterNetworkPolicies to be replicated across the entire +ClusterSet. The ACNP to be replicated should be created as a ResourceExport in the leader cluster, and +the resource export/import pipeline will ensure member clusters receive this ACNP spec to be replicated. +Each member cluster's Multi-cluster Controller will then create an ACNP in their respective clusters. diff --git a/docs/multicluster/getting-started.md b/docs/multicluster/getting-started.md index 8d15086462d..4f785073c37 100644 --- a/docs/multicluster/getting-started.md +++ b/docs/multicluster/getting-started.md @@ -412,6 +412,91 @@ ResourceExport into the corresponding ResourceImport until users correct it. due to forementioned mismatch issue, Antrea Multi-cluster Controller will also skip converging the corresponding Endpoints ResourceExport until users correct it. +## Multi-cluster ClusterNetworkPolicy Replication + +Since Antrea v1.6.0, Multi-cluster admins can specify certain ClusterNetworkPolicies to be replicated +across the entire ClusterSet. This is especially useful for ClusterSet admins who want all clusters in +the ClusterSet to be applied with a consistent security posture (for example, all Namespaces in all +clusters can only communicate with Pods in their own namespaces). For more information regarding +Antrea ClusterNetworkPolicy (ACNP), refer to [this document](../antrea-network-policy.md). + +To achieve such ACNP copy-span replication across clusters, admins can, in the acting leader cluster of +a Multi-cluster deployment, create a ResourceExport of kind `AntreaClusterNetworkPolicy` which contains +the ClusterNetworkPolicy spec they wish to be replicated. The ResourceExport should be created in the +Namespace which implements the Common Area of the ClusterSet. In future releases, some additional tooling +may become available to automate the creation of such ResourceExport and make ACNP replication easier. + +```yaml +apiVersion: multicluster.crd.antrea.io/v1alpha1 +kind: ResourceExport +metadata: + name: strict-namespace-isolation-for-test-clusterset + namespace: antrea-mcs-ns # Namespace that implements Common Area of test-clusterset +spec: + kind: AntreaClusterNetworkPolicy + name: strict-namespace-isolation # In each importing cluster, an ACNP of name antrea-mc-strict-namespace-isolation will be created with the spec below + clusternetworkpolicy: + priority: 1 + tier: securityops + appliedTo: + - namespaceSelector: {} # Selects all Namespaces in the member cluster + ingress: + - action: Pass + from: + - namespaces: + match: Self # Skip drop rule for traffic from Pods in the same Namespace + - podSelector: + matchLabels: + k8s-app: kube-dns # Skip drop rule for traffic from the core-dns components + - action: Drop + from: + - namespaceSelector: {} # Drop from Pods from all other Namespaces +``` + +The above sample spec will create an ACNP in each member cluster which implements strict namespace +isolation for that cluster. + +Note that because the Tier that an ACNP refers to must exist before the ACNP is applied, an importing +cluster may fail to create the ACNP to be replicated, if the Tier in the ResourceExport spec cannot be +found in that particular cluster. If there are such failures, the ACNP creation status of failed member +clusters will be reported back to the Common Area as K8s Events, and can be checked by describing the +ResourceImport of the original ResourceExport: + +```text +kubectl describe resourceimport -A +--- +Name: strict-namespace-isolation-antreaclusternetworkpolicy +Namespace: antrea-mcs-ns +API Version: multicluster.crd.antrea.io/v1alpha1 +Kind: ResourceImport +Spec: + Clusternetworkpolicy: + Applied To: + Namespace Selector: + Ingress: + Action: Pass + Enable Logging: false + From: + Namespaces: + Match: Self + Pod Selector: + Match Labels: + k8s-app: kube-dns + Action: Drop + Enable Logging: false + From: + Namespace Selector: + Priority: 1 + Tier: random + Kind: AntreaClusterNetworkPolicy + Name: strict-namespace-isolation + ... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning ACNPImportFailed 2m11s resourceimport-controller ACNP Tier random does not exist in the importing cluster test-cluster-west +``` + ## Known Issue We recommend user to reinstall or update Antrea Multi-cluster controllers through `kubectl apply`. diff --git a/multicluster/Makefile b/multicluster/Makefile index 632ac3f9525..fc7c06d0743 100644 --- a/multicluster/Makefile +++ b/multicluster/Makefile @@ -4,7 +4,7 @@ IMG ?= antrea/antrea-mc-controller:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) # For controller-gen, float value is not allowed by default as it is considered dangerous # See https://github.com/kubernetes-sigs/controller-tools/issues/245 -# However the ResourceExport/Import refers to ACNP type definition and the priority field in ACNP spec is of type float64. +# However the ResourceExport/Import refers to ACNP type definition and the priority field in ACNP spec is type float64. # Hence, before any ACNP spec bumps that changes the priorty field to a different type, # the allowDangerousTypes flag is needed for CRD manifests to generate correctly. CRD_OPTIONS ?= "crd:trivialVersions=true,allowDangerousTypes=true,preserveUnknownFields=false" diff --git a/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller.go b/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller.go index d8d09c7693a..661a7f66e37 100644 --- a/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller.go +++ b/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller.go @@ -16,6 +16,7 @@ package commonarea import ( "context" "errors" + "fmt" "math/rand" corev1 "k8s.io/api/core/v1" @@ -35,7 +36,6 @@ import ( const ( nameSuffixLength int = 5 acnpImportStatusPrefix string = "acnp-import-status-" - acnpImportSucceeded string = "ACNPImportSucceeded" acnpImportFailed string = "ACNPImportFailed" ) @@ -64,9 +64,10 @@ func (r *ResourceImportReconciler) handleResImpUpdateForClusterNetworkPolicy(ctx } if !acnpNotFound { if _, ok := acnp.Annotations[common.AntreaMCACNPAnnotation]; !ok { - err := errors.New("unable to import Antrea ClusterNetworkPolicy which conflicts with existing one") + msg := "Unable to import Antrea ClusterNetworkPolicy which conflicts with existing one in cluster " + r.localClusterID + err := errors.New(msg) klog.ErrorS(err, "", "acnp", klog.KObj(acnp)) - return ctrl.Result{}, err + return ctrl.Result{}, r.reportStatusEvent(msg, ctx, resImp) } } acnpObj := getMCAntreaClusterPolicy(resImp) @@ -78,55 +79,30 @@ func (r *ResourceImportReconciler) handleResImpUpdateForClusterNetworkPolicy(ctx // Create or update the ACNP if necessary. if acnpNotFound { if err = r.localClusterClient.Create(ctx, acnpObj, &client.CreateOptions{}); err != nil { - klog.ErrorS(err, "failed to create imported Antrea ClusterNetworkPolicy", "acnp", klog.KObj(acnpObj)) - return ctrl.Result{}, err + msg := "Failed to create imported Antrea ClusterNetworkPolicy in cluster " + r.localClusterID + klog.ErrorS(err, msg, "acnp", klog.KObj(acnpObj)) + return ctrl.Result{}, r.reportStatusEvent(msg, ctx, resImp) } + r.installedResImports.Add(*resImp) } else if !apiequality.Semantic.DeepEqual(acnp.Spec, acnpObj.Spec) { acnp.Spec = acnpObj.Spec if err = r.localClusterClient.Update(ctx, acnp, &client.UpdateOptions{}); err != nil { - klog.ErrorS(err, "failed to update imported Antrea ClusterNetworkPolicy", "acnp", klog.KObj(acnpObj)) - return ctrl.Result{}, err + msg := "Failed to update imported Antrea ClusterNetworkPolicy in cluster " + r.localClusterID + klog.ErrorS(err, msg, "acnp", klog.KObj(acnpObj)) + return ctrl.Result{}, r.reportStatusEvent(msg, ctx, resImp) } } } else if tierNotFound && !acnpNotFound { // The ACNP Tier does not exist, and the policy cannot be realized in this particular importing member cluster. // If there is an ACNP previously created via import (which has a valid Tier by then), it should be cleaned up. if err = r.localClusterClient.Delete(ctx, acnpObj, &client.DeleteOptions{}); err != nil { - klog.ErrorS(err, "failed to delete imported Antrea ClusterNetworkPolicy that no longer has a valid Tier for the current cluster", "acnp", klog.KObj(acnpObj)) - return ctrl.Result{}, err + msg := "Failed to delete imported Antrea ClusterNetworkPolicy that no longer has a valid Tier for cluster " + r.localClusterID + klog.ErrorS(err, msg, "acnp", klog.KObj(acnpObj)) + return ctrl.Result{}, r.reportStatusEvent(msg, ctx, resImp) } - } - - statusEvent := &corev1.Event{ - ObjectMeta: metav1.ObjectMeta{ - Name: randName(acnpImportStatusPrefix + r.localClusterID + "-"), - Namespace: resImp.Namespace, - }, - InvolvedObject: corev1.ObjectReference{ - APIVersion: resourceImportAPIVersion, - Kind: resourceImportKind, - Name: resImp.Name, - Namespace: resImp.Namespace, - UID: resImp.GetUID(), - }, - FirstTimestamp: metav1.Now(), - LastTimestamp: metav1.Now(), - ReportingController: acnpEventReportingController, - ReportingInstance: acnpEventReportingInstance, - Action: "reconciled", - } - if tierNotFound { - statusEvent.Type = corev1.EventTypeWarning - statusEvent.Reason = acnpImportFailed - statusEvent.Message = "ACNP Tier does not exist in the importing cluster " + r.localClusterID - } else { - statusEvent.Type = corev1.EventTypeNormal - statusEvent.Reason = acnpImportSucceeded - statusEvent.Message = "ACNP successfully created in the importing cluster " + r.localClusterID - } - if err = r.remoteCommonArea.Create(ctx, statusEvent, &client.CreateOptions{}); err != nil { - klog.ErrorS(err, "failed to create acnp import event for resourceimport", "resImp", klog.KObj(resImp)) - return ctrl.Result{}, err + } else if tierNotFound { + msg := fmt.Sprintf("ACNP Tier %s does not exist in importing cluster %s", tierName, r.localClusterID) + return ctrl.Result{}, r.reportStatusEvent(msg, ctx, resImp) } return ctrl.Result{}, nil } @@ -170,6 +146,38 @@ func getMCAntreaClusterPolicy(resImp *multiclusterv1alpha1.ResourceImport) *v1al } } +func (r *ResourceImportReconciler) reportStatusEvent(errMsg string, ctx context.Context, resImp *multiclusterv1alpha1.ResourceImport) error { + if errMsg == "" { + return nil + } + statusEvent := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + Name: randName(acnpImportStatusPrefix + r.localClusterID + "-"), + Namespace: resImp.Namespace, + }, + Type: corev1.EventTypeWarning, + Reason: acnpImportFailed, + Message: errMsg, + InvolvedObject: corev1.ObjectReference{ + APIVersion: resourceImportAPIVersion, + Kind: resourceImportKind, + Name: resImp.Name, + Namespace: resImp.Namespace, + UID: resImp.GetUID(), + }, + FirstTimestamp: metav1.Now(), + LastTimestamp: metav1.Now(), + ReportingController: acnpEventReportingController, + ReportingInstance: acnpEventReportingInstance, + Action: "synced", + } + if err := r.remoteCommonArea.Create(ctx, statusEvent, &client.CreateOptions{}); err != nil { + klog.ErrorS(err, "Failed to create ACNP import event for ResourceImport", "resImp", klog.KObj(resImp)) + return err + } + return nil +} + func randSeq(n int) string { b := make([]rune, n) for i := range b { diff --git a/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller_test.go b/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller_test.go index 9304c2cfa95..e9b420b4937 100644 --- a/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller_test.go +++ b/multicluster/controllers/multicluster/commonarea/acnp_resourceimport_controller_test.go @@ -21,10 +21,12 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" mcsv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" @@ -147,7 +149,15 @@ func TestResourceImportReconciler_handleCopySpanACNPCreateEvent(t *testing.T) { } else if !tt.expectedSuccess && (err == nil || !apierrors.IsNotFound(err)) { t.Errorf("ResourceImport Reconciler should not import an ACNP whose Tier does not exist in current cluster. Expected NotFound error. Actual err = %v", err) } - //TODO(yang): add Event creation tests + if !tt.expectedSuccess { + errorList := &corev1.EventList{} + if err := fakeRemoteClient.List(ctx, errorList, &client.ListOptions{}); err != nil { + t.Errorf("Failed to list Events in remote Common Area") + } + if len(errorList.Items) == 0 { + t.Errorf("An event should be created for failed ACNP imports") + } + } } }) } diff --git a/multicluster/controllers/multicluster/resourceexport_controller.go b/multicluster/controllers/multicluster/resourceexport_controller.go index 73bbcd438b4..79e5a170ab1 100644 --- a/multicluster/controllers/multicluster/resourceexport_controller.go +++ b/multicluster/controllers/multicluster/resourceexport_controller.go @@ -372,7 +372,7 @@ func (r *ResourceExportReconciler) refreshEndpointsResourceImport( var newSubsets []corev1.EndpointSubset undeleteItems, err := r.getNotDeletedResourceExports(resExport) if err != nil { - klog.ErrorS(err, "failed to list ResourceExports, retry later") + klog.ErrorS(err, "Failed to list ResourceExports, retry later") return newResImport, false, err } for _, re := range undeleteItems { @@ -400,7 +400,7 @@ func (r *ResourceExportReconciler) refreshACNPResourceImport( if !apiequality.Semantic.DeepEqual(resExport.Spec.ClusterNetworkPolicy, resImport.Spec.ClusterNetworkPolicy) { undeletedItems, err := r.getNotDeletedResourceExports(resExport) if err != nil { - klog.ErrorS(err, "failed to list ResourceExports for ACNP, retry later") + klog.ErrorS(err, "Failed to list ResourceExports for ACNP, retry later") return newResImport, false, err } if len(undeletedItems) == 1 && undeletedItems[0].Name == resExport.Name && undeletedItems[0].Namespace == resExport.Namespace { diff --git a/multicluster/controllers/multicluster/stale_controller.go b/multicluster/controllers/multicluster/stale_controller.go index 705593b4cf5..fa03147339c 100644 --- a/multicluster/controllers/multicluster/stale_controller.go +++ b/multicluster/controllers/multicluster/stale_controller.go @@ -33,6 +33,7 @@ import ( mcsv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" "antrea.io/antrea/multicluster/controllers/multicluster/common" "antrea.io/antrea/multicluster/controllers/multicluster/commonarea" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" ) // StaleController will clean up ServiceImport and MC Service if no corresponding ResourceImport @@ -68,17 +69,29 @@ func (c *StaleController) cleanup() error { if *c.remoteCommonAreaManager == nil { return errors.New("ClusterSet has not been initialized properly, no available remote common area") } - remoteCluster, err := getRemoteCommonArea(c.remoteCommonAreaManager) if err != nil { return err } - localClusterID := string((*c.remoteCommonAreaManager).GetLocalClusterID()) if len(localClusterID) == 0 { return errors.New("localClusterID is not initialized, retry later") } + resImpList := &mcsv1alpha1.ResourceImportList{} + if err := remoteCluster.List(ctx, resImpList, &client.ListOptions{Namespace: remoteCluster.GetNamespace()}); err != nil { + return err + } + if err := c.cleanupStaleServiceResources(remoteCluster, localClusterID, resImpList); err != nil { + return err + } + if err := c.cleanupACNPResources(resImpList); err != nil { + return err + } + return nil +} +func (c *StaleController) cleanupStaleServiceResources(remoteCluster commonarea.RemoteCommonArea, + localClusterID string, resImpList *mcsv1alpha1.ResourceImportList) error { svcImpList := &k8smcsv1alpha1.ServiceImportList{} if err := c.List(ctx, svcImpList, &client.ListOptions{}); err != nil { return err @@ -89,11 +102,6 @@ func (c *StaleController) cleanup() error { return err } - resImpList := &mcsv1alpha1.ResourceImportList{} - if err := remoteCluster.List(ctx, resImpList, &client.ListOptions{Namespace: remoteCluster.GetNamespace()}); err != nil { - return err - } - svcImpItems := svcImpList.Items var mcsSvcItems []corev1.Service for _, svc := range svcList.Items { @@ -104,12 +112,11 @@ func (c *StaleController) cleanup() error { for _, resImp := range resImpList.Items { for k, svc := range mcsSvcItems { - if svc.Name == common.AntreaMCSPrefix+resImp.Spec.Name && svc.Namespace == resImp.Spec.Namespace { + if resImp.Spec.Kind == common.ServiceKind && svc.Name == common.AntreaMCSPrefix+resImp.Spec.Name && svc.Namespace == resImp.Spec.Namespace { // Set the valid Service item as empty Service, then all left non-empty items should be removed. mcsSvcItems[k] = corev1.Service{} } } - for n, svcImp := range svcImpItems { if svcImp.Name == resImp.Spec.Name && svcImp.Namespace == resImp.Spec.Namespace { svcImpItems[n] = k8smcsv1alpha1.ServiceImport{} @@ -120,18 +127,17 @@ func (c *StaleController) cleanup() error { for _, svc := range mcsSvcItems { s := svc if s.Name != "" { - klog.InfoS("clean up Service", "service", klog.KObj(&s)) + klog.InfoS("Cleaning up stale Service", "service", klog.KObj(&s)) if err := c.Client.Delete(ctx, &s, &client.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { return err } } } - for _, svcImp := range svcImpItems { si := svcImp if si.Name != "" { - klog.InfoS("clean up ServiceImport", "serviceimport", klog.KObj(&si)) - if err = c.Client.Delete(ctx, &si, &client.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + klog.InfoS("Cleaning up stale ServiceImport", "serviceimport", klog.KObj(&si)) + if err := c.Client.Delete(ctx, &si, &client.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { return err } } @@ -167,7 +173,7 @@ func (c *StaleController) cleanup() error { for _, r := range resExpItems { re := r if re.Name != "" { - klog.InfoS("clean up ResourceExport", "ResourceExport", klog.KObj(&re)) + klog.InfoS("Cleaning up ResourceExport", "ResourceExport", klog.KObj(&re)) if err := remoteCluster.Delete(ctx, &re, &client.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { return err } @@ -176,6 +182,35 @@ func (c *StaleController) cleanup() error { return nil } +func (c *StaleController) cleanupACNPResources(resImpList *mcsv1alpha1.ResourceImportList) error { + acnpList := &crdv1alpha1.ClusterNetworkPolicyList{} + if err := c.List(ctx, acnpList, &client.ListOptions{}); err != nil { + return err + } + staleMCACNPItems := map[string]crdv1alpha1.ClusterNetworkPolicy{} + for _, acnp := range acnpList.Items { + if _, ok := acnp.Annotations[common.AntreaMCACNPAnnotation]; ok { + staleMCACNPItems[acnp.Name] = acnp + } + } + for _, resImp := range resImpList.Items { + if resImp.Spec.Kind == common.AntreaClusterNetworkPolicyKind { + acnpNameFromResImp := common.AntreaMCSPrefix + resImp.Spec.Name + if _, ok := staleMCACNPItems[acnpNameFromResImp]; ok { + delete(staleMCACNPItems, acnpNameFromResImp) + } + } + } + for _, stalePolicy := range staleMCACNPItems { + acnp := stalePolicy + klog.InfoS("Cleaning up stale ACNP", "acnp", klog.KObj(&acnp)) + if err := c.Client.Delete(ctx, &acnp, &client.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + // Enqueue will be called after StaleController is initialized. func (c *StaleController) Enqueue() { // The key can be anything as we only have single item. diff --git a/multicluster/controllers/multicluster/stale_controller_test.go b/multicluster/controllers/multicluster/stale_controller_test.go index eacc0dd66f9..6d401863ce0 100644 --- a/multicluster/controllers/multicluster/stale_controller_test.go +++ b/multicluster/controllers/multicluster/stale_controller_test.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -30,9 +31,11 @@ import ( mcsv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" "antrea.io/antrea/multicluster/controllers/multicluster/common" "antrea.io/antrea/multicluster/controllers/multicluster/commonarea" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" ) func TestStaleController_CleanupService(t *testing.T) { + localClusterID = "cluster-a" remoteMgr := commonarea.NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID)) go remoteMgr.Start() @@ -62,6 +65,7 @@ func TestStaleController_CleanupService(t *testing.T) { Spec: mcsv1alpha1.ResourceImportSpec{ Name: "non-nginx", Namespace: "default", + Kind: common.ServiceKind, }, } tests := []struct { @@ -123,6 +127,94 @@ func TestStaleController_CleanupService(t *testing.T) { } } +func TestStaleController_CleanupACNP(t *testing.T) { + localClusterID = "cluster-a" + remoteMgr := commonarea.NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID)) + go remoteMgr.Start() + + acnpImportName := "acnp-for-isolation" + acnpResImportName := leaderNamespace + "-" + acnpImportName + acnpResImport := mcsv1alpha1.ResourceImport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: acnpResImportName, + }, + Spec: mcsv1alpha1.ResourceImportSpec{ + Name: acnpImportName, + Kind: common.AntreaClusterNetworkPolicyKind, + ClusterNetworkPolicy: &v1alpha1.ClusterNetworkPolicySpec{ + Tier: "securityops", + Priority: 1.0, + AppliedTo: []v1alpha1.NetworkPolicyPeer{ + {NamespaceSelector: &metav1.LabelSelector{}}, + }, + }, + }, + } + acnp1 := v1alpha1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.AntreaMCSPrefix + acnpImportName, + Annotations: map[string]string{common.AntreaMCACNPAnnotation: "true"}, + }, + } + acnp2 := v1alpha1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.AntreaMCSPrefix + "some-deleted-resimp", + Annotations: map[string]string{common.AntreaMCACNPAnnotation: "true"}, + }, + } + acnp3 := v1alpha1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "non-mcs-acnp", + }, + } + tests := []struct { + name string + existingACNPList *v1alpha1.ClusterNetworkPolicyList + existResImpList *mcsv1alpha1.ResourceImportList + expectedACNPRemaining sets.String + }{ + { + name: "cleanup stale ACNP", + existingACNPList: &v1alpha1.ClusterNetworkPolicyList{ + Items: []v1alpha1.ClusterNetworkPolicy{ + acnp1, acnp2, acnp3, + }, + }, + existResImpList: &mcsv1alpha1.ResourceImportList{ + Items: []mcsv1alpha1.ResourceImport{ + acnpResImport, + }, + }, + expectedACNPRemaining: sets.NewString(common.AntreaMCSPrefix+acnpImportName, "non-mcs-acnp"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithLists(tt.existingACNPList).Build() + fakeRemoteClient := fake.NewClientBuilder().WithScheme(scheme).WithLists(tt.existResImpList).Build() + _ = commonarea.NewFakeRemoteCommonArea(scheme, &remoteMgr, fakeRemoteClient, "leader-cluster", "default") + + c := NewStaleController(fakeClient, scheme, &remoteMgr) + if err := c.cleanup(); err != nil { + t.Errorf("StaleController.cleanup() should clean up all stale ACNPs but got err = %v", err) + } + ctx := context.TODO() + acnpList := &v1alpha1.ClusterNetworkPolicyList{} + if err := fakeClient.List(ctx, acnpList, &client.ListOptions{}); err != nil { + t.Errorf("Error when listing the ACNPs after cleanup") + } + acnpRemaining := sets.NewString() + for _, acnp := range acnpList.Items { + acnpRemaining.Insert(acnp.Name) + } + if !acnpRemaining.Equal(tt.expectedACNPRemaining) { + t.Errorf("Unexpected stale ACNP cleanup result. Expected: %v, Actual: %v", tt.expectedACNPRemaining, acnpRemaining) + } + }) + } +} + func TestStaleController_CleanupResourceExport(t *testing.T) { localClusterID = "cluster-a" remoteMgr := commonarea.NewRemoteCommonAreaManager("test-clusterset", common.ClusterID(localClusterID)) diff --git a/multicluster/controllers/multicluster/test_data.go b/multicluster/controllers/multicluster/test_data.go index e8683da85cf..160db53e4c8 100644 --- a/multicluster/controllers/multicluster/test_data.go +++ b/multicluster/controllers/multicluster/test_data.go @@ -29,6 +29,7 @@ import ( k8smcsapi "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" mcsv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" ) var ( @@ -115,4 +116,5 @@ func init() { utilruntime.Must(mcsv1alpha1.AddToScheme(scheme)) utilruntime.Must(k8smcsapi.AddToScheme(scheme)) utilruntime.Must(k8sscheme.AddToScheme(scheme)) + utilruntime.Must(crdv1alpha1.AddToScheme(scheme)) }