diff --git a/apis/cluster/v1beta1/zz_generated.deepcopy.go b/apis/cluster/v1beta1/zz_generated.deepcopy.go index 6d06cb15b..6c25f0189 100644 --- a/apis/cluster/v1beta1/zz_generated.deepcopy.go +++ b/apis/cluster/v1beta1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ Licensed under the MIT license. package v1beta1 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/apis/placement/v1alpha1/zz_generated.deepcopy.go b/apis/placement/v1alpha1/zz_generated.deepcopy.go index d26fe49a5..1a3f72658 100644 --- a/apis/placement/v1alpha1/zz_generated.deepcopy.go +++ b/apis/placement/v1alpha1/zz_generated.deepcopy.go @@ -10,10 +10,11 @@ Licensed under the MIT license. package v1alpha1 import ( - "go.goms.io/fleet/apis/placement/v1beta1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + + "go.goms.io/fleet/apis/placement/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/apis/placement/v1beta1/binding_types.go b/apis/placement/v1beta1/binding_types.go index 2887283f6..2095f4cb9 100644 --- a/apis/placement/v1beta1/binding_types.go +++ b/apis/placement/v1beta1/binding_types.go @@ -10,6 +10,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // SchedulerCRBCleanupFinalizer is a finalizer added to ClusterResourceBindings to ensure we can look up the + // corresponding CRP name for deleting ClusterResourceBindings to trigger a new scheduling cycle. + SchedulerCRBCleanupFinalizer = fleetPrefix + "scheduler-crb-cleanup" +) + // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,categories={fleet,fleet-placement},shortName=rb // +kubebuilder:subresource:status diff --git a/apis/placement/v1beta1/commons.go b/apis/placement/v1beta1/commons.go index dce5bb8e8..e805051a0 100644 --- a/apis/placement/v1beta1/commons.go +++ b/apis/placement/v1beta1/commons.go @@ -50,6 +50,15 @@ const ( // The format is {workPrefix}-configMap-uuid WorkNameWithConfigEnvelopeFmt = "%s-configmap-%s" + // ParentClusterResourceOverrideSnapshotHashAnnotation is the annotation to work that contains the hash of the parent cluster resource override snapshot list. + ParentClusterResourceOverrideSnapshotHashAnnotation = fleetPrefix + "parent-cluster-resource-override-snapshot-hash" + + // ParentResourceOverrideSnapshotHashAnnotation is the annotation to work that contains the hash of the parent resource override snapshot list. + ParentResourceOverrideSnapshotHashAnnotation = fleetPrefix + "parent-resource-override-snapshot-hash" + + // ParentResourceSnapshotNameAnnotation is the annotation applied to work that contains the name of the master resource snapshot that generates the work. + ParentResourceSnapshotNameAnnotation = fleetPrefix + "parent-resource-snapshot-name" + // ParentResourceSnapshotIndexLabel is the label applied to work that contains the index of the resource snapshot that generates the work. ParentResourceSnapshotIndexLabel = fleetPrefix + "parent-resource-snapshot-index" diff --git a/apis/placement/v1beta1/zz_generated.deepcopy.go b/apis/placement/v1beta1/zz_generated.deepcopy.go index f20cf8be5..754effe01 100644 --- a/apis/placement/v1beta1/zz_generated.deepcopy.go +++ b/apis/placement/v1beta1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ Licensed under the MIT license. package v1beta1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 0d4061551..ac4844274 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -11,7 +11,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/cmd/hubagent/workload/setup.go b/cmd/hubagent/workload/setup.go index 358ceabee..908d6ef4c 100644 --- a/cmd/hubagent/workload/setup.go +++ b/cmd/hubagent/workload/setup.go @@ -39,6 +39,7 @@ import ( "go.goms.io/fleet/pkg/scheduler/framework" "go.goms.io/fleet/pkg/scheduler/profile" "go.goms.io/fleet/pkg/scheduler/queue" + schedulercrbwatcher "go.goms.io/fleet/pkg/scheduler/watchers/clusterresourcebinding" schedulercrpwatcher "go.goms.io/fleet/pkg/scheduler/watchers/clusterresourceplacement" schedulercspswatcher "go.goms.io/fleet/pkg/scheduler/watchers/clusterschedulingpolicysnapshot" "go.goms.io/fleet/pkg/scheduler/watchers/membercluster" @@ -278,6 +279,15 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, return err } + klog.Info("Setting up the clusterResourceBinding watcher for scheduler") + if err := (&schedulercrbwatcher.Reconciler{ + Client: mgr.GetClient(), + SchedulerWorkQueue: defaultSchedulingQueue, + }).SetupWithManager(mgr); err != nil { + klog.ErrorS(err, "Unable to set up clusterResourceBinding watcher for scheduler") + return err + } + klog.Info("Setting up the memberCluster watcher for scheduler") if err := (&membercluster.Reconciler{ Client: mgr.GetClient(), diff --git a/go.mod b/go.mod index 918077e2b..719ad99ff 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module go.goms.io/fleet -go 1.22.2 +go 1.22.7 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 github.com/Azure/karpenter v0.2.0 github.com/crossplane/crossplane-runtime v1.17.0 github.com/evanphx/json-patch/v5 v5.9.0 github.com/go-logr/logr v1.4.2 github.com/google/go-cmp v0.6.0 - github.com/onsi/ginkgo/v2 v2.19.1 - github.com/onsi/gomega v1.34.0 + github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/gomega v1.35.1 github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.6.1 github.com/spf13/cobra v1.8.0 @@ -21,8 +21,8 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 - golang.org/x/sync v0.7.0 - golang.org/x/time v0.5.0 + golang.org/x/sync v0.8.0 + golang.org/x/time v0.7.0 k8s.io/api v0.30.2 k8s.io/apiextensions-apiserver v0.30.2 k8s.io/apimachinery v0.30.2 @@ -31,13 +31,35 @@ require ( k8s.io/klog/v2 v2.120.1 k8s.io/metrics v0.25.2 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 + sigs.k8s.io/cloud-provider-azure v1.28.2 + sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.50 sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/work-api v0.0.0-20220407021756-586d707fdb2c ) require ( dario.cat/mergo v1.0.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/aws/karpenter-core v0.32.2-0.20231109191441-e32aafc81fb5 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -53,12 +75,13 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/uuid v1.6.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -78,17 +101,18 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/samber/lo v1.38.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.22.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/tools v0.26.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b828380d0..ea4960345 100644 --- a/go.sum +++ b/go.sum @@ -6,30 +6,60 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go-extensions v0.1.4 h1:XNT7IWmj4u3AfSag3t2mFupHT59J58pknX+daqprjm8= github.com/Azure/azure-sdk-for-go-extensions v0.1.4/go.mod h1:dJfn8QUzuvyO4hGZ8pkROwd7/VQzDG8ER2SRk+V0afY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 h1:/Di3vB4sNeQ+7A8efjUVENvyB945Wruvstucqp7ZArg= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0/go.mod h1:gM3K25LQlsET3QR+4V74zxCsFAy0r6xMNN9n80SZn+4= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 h1:Sg/D8VuUQ+bw+FOYJF+xRKcwizCOP13HL0Se8pWNBzE= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0/go.mod h1:Kyqzdqq0XDoCm+o9aZ25wZBmBUBzPBzPAj1R5rYsT6I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0 h1:0nGmzwBv5ougvzfGPCO2ljFRHvun57KpNrVCMrlk0ns= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0/go.mod h1:gYq8wyDgv6JLhGbAU6gg8amCPgQWRE+aCvrV2gyzdfs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0 h1:HlZMUZW8S4P9oob1nCHxCCKrytxyLc+24nUJGssoEto= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 h1:yzrctSl9GMIQ5lHu7jc8olOsGjWDCsBpJhWqfGa/YIM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0/go.mod h1:GE4m0rnnfwLGX0Y9A9A25Zx5N/90jneT5ABevqzhuFQ= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1 h1:nGiU2ovpbtkcC3x+g/wNHV4S9TOIYe2/yOVAj3wiGHI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1/go.mod h1:T3ZgvD1aRKu12mEA0fU3PPvI7V0Nh0wzIdK0QMBhf0Y= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.2.0 h1:TkNl6WlpHdZSMt0Zngw8y0c9ZMi3GwmYl0kKNbW9PvU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.2.0/go.mod h1:ukmL56lWl275SgNFijuwx0Wv6n6HmzzpPWW4kMoy/wY= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.0 h1:eXnN9kaS8TiDwXjoie3hMRLuwdUBUMW9KRgOqB3mCaw= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.0/go.mod h1:XIpam8wumeZ5rVMuhdDQLMfIPDf1WO3IzrCRO3e3e3o= github.com/Azure/go-armbalancer v0.0.2 h1:NVnxsTWHI5/fEzL6k6TjxPUfcB/3Si3+HFOZXOu0QtA= github.com/Azure/go-armbalancer v0.0.2/go.mod h1:yTg7MA/8YnfKQc9o97tzAJ7fbdVkod1xGsIvKmhYPRE= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= @@ -46,6 +76,8 @@ github.com/Azure/karpenter v0.2.0 h1:PK44Fw1wO5JohCTPm3Lmu+s58PsBAWdyiaiU8eE3M8U github.com/Azure/karpenter v0.2.0/go.mod h1:tnn5M5lA7nKdOslV37R76jae3gtdIbPrKWQ6Orn0cQg= github.com/Azure/skewer v0.0.19 h1:+qA1z8isKmlNkhAwZErNS2wD2jaemSk9NszYKr8dddU= github.com/Azure/skewer v0.0.19/go.mod h1:LVH7jmduRKmPj8YcIz7V4f53xJEntjweL4aoLyChkwk= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= @@ -66,6 +98,8 @@ github.com/crossplane/crossplane-runtime v1.17.0/go.mod h1:vtglCrnnbq2HurAk9yLHa github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= @@ -98,8 +132,10 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -116,8 +152,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -128,6 +164,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -149,10 +187,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= -github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= -github.com/onsi/gomega v1.34.0 h1:eSSPsPNp6ZpsG8X1OVmOTxig+CblTc4AxpPBykhe2Os= -github.com/onsi/gomega v1.34.0/go.mod h1:MIKI8c+f+QLWk+hxbePD4i0LMJSExPaZOVfkoex4cAo= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -170,6 +208,8 @@ github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= +github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -182,26 +222,34 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.goms.io/fleet-networking v0.2.7 h1:lVs2/GiCjo18BRgACib+VPnENUMh+2YbYXoeNtcAvw0= go.goms.io/fleet-networking v0.2.7/go.mod h1:JoWG82La5nV29mooOnPpIhy6/Pi4oGXQk21CPF1UStg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -211,63 +259,88 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= +gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= @@ -298,6 +371,8 @@ knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIE knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE= sigs.k8s.io/cloud-provider-azure v1.28.2 h1:KKrWdC1+p2xXdT1VRmSkT57MhKNzPXk3yPcrwUDIr5I= sigs.k8s.io/cloud-provider-azure v1.28.2/go.mod h1:vDsaFOrvDDEUg0mLF2eoUeneCK+ROlRf4zACA91iwHs= +sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.50 h1:l9igMANNptVwYmZrqGS51oW0zvfSxBGmlOaDPe407FI= +sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.50/go.mod h1:1M90A+akyTabHVnveSKlvIO/Kk9kEr1LjRx+08twKVU= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/controllers/rollout/controller.go b/pkg/controllers/rollout/controller.go index e94e67189..d2b1674d0 100644 --- a/pkg/controllers/rollout/controller.go +++ b/pkg/controllers/rollout/controller.go @@ -281,6 +281,7 @@ func createUpdateInfo(binding *fleetv1beta1.ClusterResourceBinding, crp *fleetv1 desiredBinding.Spec.ResourceSnapshotName = latestResourceSnapshot.Name // update the resource apply strategy when controller rolls out the new changes desiredBinding.Spec.ApplyStrategy = crp.Spec.Strategy.ApplyStrategy + // TODO: check the size of the cro and ro to not exceed the limit desiredBinding.Spec.ClusterResourceOverrideSnapshots = cro desiredBinding.Spec.ResourceOverrideSnapshots = ro return toBeUpdatedBinding{ @@ -520,6 +521,7 @@ func calculateMaxToAdd(crp *fleetv1beta1.ClusterResourcePlacement, targetNumber upperBoundReadyNumber, "maxNumberOfBindingsToAdd", maxNumberToAdd) return maxNumberToAdd } + func (r *Reconciler) calculateRealTarget(crp *fleetv1beta1.ClusterResourcePlacement, schedulerTargetedBinds []*fleetv1beta1.ClusterResourceBinding) int { crpKObj := klog.KObj(crp) // calculate the target number of bindings diff --git a/pkg/controllers/workgenerator/controller.go b/pkg/controllers/workgenerator/controller.go index 97bb4720f..2c9d367c1 100644 --- a/pkg/controllers/workgenerator/controller.go +++ b/pkg/controllers/workgenerator/controller.go @@ -49,13 +49,14 @@ import ( "go.goms.io/fleet/pkg/utils/controller" "go.goms.io/fleet/pkg/utils/informer" "go.goms.io/fleet/pkg/utils/labels" + "go.goms.io/fleet/pkg/utils/resource" ) var ( // maxFailedResourcePlacementLimit indicates the max number of failed resource placements to include in the status. maxFailedResourcePlacementLimit = 100 - errResourceSnapshotNotFound = errors.New("the master resource snapshot is not found") + errResourceSnapshotNotFound = fmt.Errorf("the master resource snapshot is not found") ) // Reconciler watches binding objects and generate work objects in the designated cluster namespace @@ -135,18 +136,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req controllerruntime.Reques workUpdated := false overrideSucceeded := false - // Reset the conditions and failed placements. - for i := condition.OverriddenCondition; i < condition.TotalCondition; i++ { - resourceBinding.RemoveCondition(string(i.ResourceBindingConditionType())) - } - resourceBinding.Status.FailedPlacements = nil // list all the corresponding works works, syncErr := r.listAllWorksAssociated(ctx, &resourceBinding) if syncErr == nil { // generate and apply the workUpdated works if we have all the works overrideSucceeded, workUpdated, syncErr = r.syncAllWork(ctx, &resourceBinding, works, cluster) } - + // Reset the conditions and failed placements. + for i := condition.OverriddenCondition; i < condition.TotalCondition; i++ { + resourceBinding.RemoveCondition(string(i.ResourceBindingConditionType())) + } + resourceBinding.Status.FailedPlacements = nil if overrideSucceeded { overrideReason := condition.OverriddenSucceededReason overrideMessage := "Successfully applied the override rules on the resources" @@ -375,10 +375,28 @@ func (r *Reconciler) listAllWorksAssociated(ctx context.Context, resourceBinding func (r *Reconciler) syncAllWork(ctx context.Context, resourceBinding *fleetv1beta1.ClusterResourceBinding, existingWorks map[string]*fleetv1beta1.Work, cluster clusterv1beta1.MemberCluster) (bool, bool, error) { updateAny := atomic.NewBool(false) resourceBindingRef := klog.KObj(resourceBinding) + // the hash256 function can can handle empty list https://go.dev/play/p/_4HW17fooXM + resourceOverrideSnapshotHash, err := resource.HashOf(resourceBinding.Spec.ResourceOverrideSnapshots) + if err != nil { + return false, false, controller.NewUnexpectedBehaviorError(err) + } + clusterResourceOverrideSnapshotHash, err := resource.HashOf(resourceBinding.Spec.ClusterResourceOverrideSnapshots) + if err != nil { + return false, false, controller.NewUnexpectedBehaviorError(err) + } + // TODO: check all work synced first before fetching the snapshots after we put ParentResourceOverrideSnapshotHashAnnotation and ParentClusterResourceOverrideSnapshotHashAnnotation in all the work objects // Gather all the resource resourceSnapshots resourceSnapshots, err := r.fetchAllResourceSnapshots(ctx, resourceBinding) if err != nil { + if errors.Is(err, errResourceSnapshotNotFound) { + // the resourceIndex is deleted but the works might still be up to date with the binding. + if areAllWorkSynced(existingWorks, resourceBinding, resourceOverrideSnapshotHash, clusterResourceOverrideSnapshotHash) { + klog.V(2).InfoS("All the works are synced with the resourceBinding even if the resource snapshot index is removed", "resourceBinding", resourceBindingRef) + return true, false, nil + } + return false, false, controller.NewUserError(err) + } // TODO(RZ): handle errResourceNotFullyCreated error so we don't need to wait for all the snapshots to be created return false, false, err } @@ -422,7 +440,7 @@ func (r *Reconciler) syncAllWork(ctx context.Context, resourceBinding *fleetv1be if uResource.GetObjectKind().GroupVersionKind() == utils.ConfigMapGVK && len(uResource.GetAnnotations()[fleetv1beta1.EnvelopeConfigMapAnnotation]) != 0 { // get a work object for the enveloped configMap - work, err := r.getConfigMapEnvelopWorkObj(ctx, workNamePrefix, resourceBinding, snapshot, &uResource) + work, err := r.getConfigMapEnvelopWorkObj(ctx, workNamePrefix, resourceBinding, snapshot, &uResource, resourceOverrideSnapshotHash, clusterResourceOverrideSnapshotHash) if err != nil { return true, false, err } @@ -438,7 +456,7 @@ func (r *Reconciler) syncAllWork(ctx context.Context, resourceBinding *fleetv1be // generate a work object for the manifests even if there is nothing to place // to allow CRP to collect the status of the placement // TODO (RZ): revisit to see if we need this hack - work := generateSnapshotWorkObj(workNamePrefix, resourceBinding, snapshot, simpleManifests) + work := generateSnapshotWorkObj(workNamePrefix, resourceBinding, snapshot, simpleManifests, resourceOverrideSnapshotHash, clusterResourceOverrideSnapshotHash) activeWork[work.Name] = work newWork = append(newWork, work) @@ -485,6 +503,32 @@ func (r *Reconciler) syncAllWork(ctx context.Context, resourceBinding *fleetv1be return true, updateAny.Load(), nil } +// areAllWorkSynced checks if all the works are synced with the resource binding. +func areAllWorkSynced(existingWorks map[string]*fleetv1beta1.Work, resourceBinding *fleetv1beta1.ClusterResourceBinding, _, _ string) bool { + syncedCondition := resourceBinding.GetCondition(string(fleetv1beta1.ResourceBindingWorkSynchronized)) + if !condition.IsConditionStatusTrue(syncedCondition, resourceBinding.Generation) { + // The binding has to be synced first before we can check the works + return false + } + // TODO: check resourceOverrideSnapshotHash and clusterResourceOverrideSnapshotHash after all the work has the ParentResourceOverrideSnapshotHashAnnotation and ParentClusterResourceOverrideSnapshotHashAnnotation + resourceSnapshotName := resourceBinding.Spec.ResourceSnapshotName + for _, work := range existingWorks { + recordedName, exist := work.Annotations[fleetv1beta1.ParentResourceSnapshotNameAnnotation] + if !exist { + // TODO: remove this block after all the work has the ParentResourceSnapshotNameAnnotation + // the parent resource snapshot name is not recorded in the work, we need to construct it from the labels + crpName := resourceBinding.Labels[fleetv1beta1.CRPTrackingLabel] + index, _ := labels.ExtractResourceSnapshotIndexFromWork(work) + recordedName = fmt.Sprintf(fleetv1beta1.ResourceSnapshotNameFmt, crpName, index) + } + if recordedName != resourceSnapshotName { + klog.V(2).InfoS("The work is not synced with the resourceBinding", "work", klog.KObj(work), "resourceBinding", klog.KObj(resourceBinding), "annotationExist", exist, "recordedName", recordedName, "resourceSnapshotName", resourceSnapshotName) + return false + } + } + return true +} + // fetchAllResourceSnapshots gathers all the resource snapshots for the resource binding. func (r *Reconciler) fetchAllResourceSnapshots(ctx context.Context, resourceBinding *fleetv1beta1.ClusterResourceBinding) (map[string]*fleetv1beta1.ClusterResourceSnapshot, error) { // fetch the master snapshot first @@ -504,7 +548,7 @@ func (r *Reconciler) fetchAllResourceSnapshots(ctx context.Context, resourceBind // getConfigMapEnvelopWorkObj first try to locate a work object for the corresponding envelopObj of type configMap. // we create a new one if the work object doesn't exist. We do this to avoid repeatedly delete and create the same work object. func (r *Reconciler) getConfigMapEnvelopWorkObj(ctx context.Context, workNamePrefix string, resourceBinding *fleetv1beta1.ClusterResourceBinding, - resourceSnapshot *fleetv1beta1.ClusterResourceSnapshot, envelopeObj *unstructured.Unstructured) (*fleetv1beta1.Work, error) { + resourceSnapshot *fleetv1beta1.ClusterResourceSnapshot, envelopeObj *unstructured.Unstructured, resourceOverrideSnapshotHash, clusterResourceOverrideSnapshotHash string) (*fleetv1beta1.Work, error) { // we group all the resources in one configMap to one work manifest, err := extractResFromConfigMap(envelopeObj) if err != nil { @@ -514,6 +558,7 @@ func (r *Reconciler) getConfigMapEnvelopWorkObj(ctx context.Context, workNamePre } klog.V(2).InfoS("Successfully extract the enveloped resources from the configMap", "numOfResources", len(manifest), "snapshot", klog.KObj(resourceSnapshot), "resourceBinding", klog.KObj(resourceBinding), "configMapWrapper", klog.KObj(envelopeObj)) + // Try to see if we already have a work represent the same enveloped object for this CRP in the same cluster // The ParentResourceSnapshotIndexLabel can change between snapshots so we have to exclude that label in the match envelopWorkLabelMatcher := client.MatchingLabels{ @@ -544,6 +589,11 @@ func (r *Reconciler) getConfigMapEnvelopWorkObj(ctx context.Context, workNamePre fleetv1beta1.EnvelopeNameLabel: envelopeObj.GetName(), fleetv1beta1.EnvelopeNamespaceLabel: envelopeObj.GetNamespace(), }, + Annotations: map[string]string{ + fleetv1beta1.ParentResourceSnapshotNameAnnotation: resourceBinding.Spec.ResourceSnapshotName, + fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation: resourceOverrideSnapshotHash, + fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: clusterResourceOverrideSnapshotHash, + }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: fleetv1beta1.GroupVersion.String(), @@ -567,16 +617,19 @@ func (r *Reconciler) getConfigMapEnvelopWorkObj(ctx context.Context, workNamePre klog.ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("find %d work representing configMap", len(workList.Items))), "snapshot", klog.KObj(resourceSnapshot), "resourceBinding", klog.KObj(resourceBinding), "configMapWrapper", klog.KObj(envelopeObj)) } - // we just pick the first one if there are more than one. work := workList.Items[0] work.Labels[fleetv1beta1.ParentResourceSnapshotIndexLabel] = resourceSnapshot.Labels[fleetv1beta1.ResourceIndexLabel] + work.Annotations[fleetv1beta1.ParentResourceSnapshotNameAnnotation] = resourceBinding.Spec.ResourceSnapshotName + work.Annotations[fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation] = resourceOverrideSnapshotHash + work.Annotations[fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation] = clusterResourceOverrideSnapshotHash work.Spec.Workload.Manifests = manifest work.Spec.ApplyStrategy = resourceBinding.Spec.ApplyStrategy return &work, nil } // generateSnapshotWorkObj generates the work object for the corresponding snapshot -func generateSnapshotWorkObj(workName string, resourceBinding *fleetv1beta1.ClusterResourceBinding, resourceSnapshot *fleetv1beta1.ClusterResourceSnapshot, manifest []fleetv1beta1.Manifest) *fleetv1beta1.Work { +func generateSnapshotWorkObj(workName string, resourceBinding *fleetv1beta1.ClusterResourceBinding, resourceSnapshot *fleetv1beta1.ClusterResourceSnapshot, + manifest []fleetv1beta1.Manifest, resourceOverrideSnapshotHash, clusterResourceOverrideSnapshotHash string) *fleetv1beta1.Work { return &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, @@ -586,6 +639,11 @@ func generateSnapshotWorkObj(workName string, resourceBinding *fleetv1beta1.Clus fleetv1beta1.CRPTrackingLabel: resourceBinding.Labels[fleetv1beta1.CRPTrackingLabel], fleetv1beta1.ParentResourceSnapshotIndexLabel: resourceSnapshot.Labels[fleetv1beta1.ResourceIndexLabel], }, + Annotations: map[string]string{ + fleetv1beta1.ParentResourceSnapshotNameAnnotation: resourceBinding.Spec.ResourceSnapshotName, + fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation: resourceOverrideSnapshotHash, + fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: clusterResourceOverrideSnapshotHash, + }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: fleetv1beta1.GroupVersion.String(), @@ -619,6 +677,7 @@ func (r *Reconciler) upsertWork(ctx context.Context, newWork, existingWork *flee "resourceSnapshot", resourceSnapshotObj, "work", workObj) return true, nil } + // TODO: remove the compare after we did the check on all work in the sync all // check if we need to update the existing work object workResourceIndex, err := labels.ExtractResourceSnapshotIndexFromWork(existingWork) if err != nil { @@ -628,12 +687,20 @@ func (r *Reconciler) upsertWork(ctx context.Context, newWork, existingWork *flee // we already checked the label in fetchAllResourceSnapShots function so no need to check again resourceIndex, _ := labels.ExtractResourceIndexFromClusterResourceSnapshot(resourceSnapshot) if workResourceIndex == resourceIndex { - // no need to do anything if the work is generated from the same resource snapshot group since the resource snapshot is immutable. - klog.V(2).InfoS("Work is already associated with the desired resourceSnapshot", "resourceIndex", resourceIndex, "work", workObj, "resourceSnapshot", resourceSnapshotObj) - return false, nil + // no need to do anything if the work is generated from the same snapshots + if existingWork.Annotations[fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation] == newWork.Annotations[fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation] && + existingWork.Annotations[fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation] == newWork.Annotations[fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation] { + klog.V(2).InfoS("Work is not associated with the desired override snapshots", "existingROHash", existingWork.Annotations[fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation], + "existingCROHash", existingWork.Annotations[fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation], "work", workObj) + return false, nil + } + klog.V(2).InfoS("Work is already associated with the desired resourceSnapshot but still not having the right override snapshots", "resourceIndex", resourceIndex, "work", workObj, "resourceSnapshot", resourceSnapshotObj) } - // need to update the existing work, only two possible changes: - existingWork.Labels[fleetv1beta1.ParentResourceSnapshotIndexLabel] = resourceSnapshot.Labels[fleetv1beta1.ResourceIndexLabel] + // need to copy the new work to the existing work, only 5 possible changes: + existingWork.Labels[fleetv1beta1.ParentResourceSnapshotIndexLabel] = newWork.Labels[fleetv1beta1.ParentResourceSnapshotIndexLabel] + existingWork.Annotations[fleetv1beta1.ParentResourceSnapshotNameAnnotation] = newWork.Annotations[fleetv1beta1.ParentResourceSnapshotNameAnnotation] + existingWork.Annotations[fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation] = newWork.Annotations[fleetv1beta1.ParentResourceOverrideSnapshotHashAnnotation] + existingWork.Annotations[fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation] = newWork.Annotations[fleetv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation] existingWork.Spec.Workload.Manifests = newWork.Spec.Workload.Manifests if err := r.Client.Update(ctx, existingWork); err != nil { klog.ErrorS(err, "Failed to update the work associated with the resourceSnapshot", "resourceSnapshot", resourceSnapshotObj, "work", workObj) diff --git a/pkg/controllers/workgenerator/controller_integration_test.go b/pkg/controllers/workgenerator/controller_integration_test.go index cf2e39925..40e60c726 100644 --- a/pkg/controllers/workgenerator/controller_integration_test.go +++ b/pkg/controllers/workgenerator/controller_integration_test.go @@ -6,6 +6,8 @@ Licensed under the MIT license. package workgenerator import ( + "crypto/sha256" + "encoding/json" "fmt" "strconv" "time" @@ -64,13 +66,14 @@ const ( ) var _ = Describe("Test Work Generator Controller", func() { - Context("Test Bound ClusterResourceBinding", func() { var binding *placementv1beta1.ClusterResourceBinding ignoreTypeMeta := cmpopts.IgnoreFields(metav1.TypeMeta{}, "Kind", "APIVersion") ignoreWorkOption := cmpopts.IgnoreFields(metav1.ObjectMeta{}, "UID", "ResourceVersion", "ManagedFields", "CreationTimestamp", "Generation") - + var emptyArray []string + jsonBytes, _ := json.Marshal(emptyArray) + emptyHash := fmt.Sprintf("%x", sha256.Sum256(jsonBytes)) BeforeEach(func() { memberClusterName = "cluster-" + utils.RandStr() testCRPName = "crp" + utils.RandStr() @@ -211,7 +214,7 @@ var _ = Describe("Test Work Generator Controller", func() { By(fmt.Sprintf("work %s is created in %s", work.Name, work.Namespace)) }) - It("Should handle the case that the snapshot is deleted", func() { + It("Should handle the case that the binding is deleted", func() { // generate master resource snapshot masterSnapshot := generateResourceSnapshot(1, 1, 0, [][]byte{ testResourceCRD, testNameSpace, testResource, @@ -306,6 +309,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentBindingLabel: binding.Name, placementv1beta1.ParentResourceSnapshotIndexLabel: "1", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -359,7 +367,7 @@ var _ = Describe("Test Work Generator Controller", func() { verifyBindingStatusSyncedNotApplied(binding, false, false) }) - Context("Test Bound ClusterResourceBinding with failed to apply manifests", func() { + Context("Test Bound ClusterResourceBinding with manifests go from not applied to available", func() { work := placementv1beta1.Work{} BeforeEach(func() { // check the binding status till the bound condition is true @@ -395,6 +403,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentBindingLabel: binding.Name, placementv1beta1.ParentResourceSnapshotIndexLabel: "1", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -441,6 +454,59 @@ var _ = Describe("Test Work Generator Controller", func() { // check the binding status that it should be marked as available true eventually verifyBindStatusAvail(binding, false) }) + + It("Should continue to update the binding status even if the master resource snapshot is deleted after the work is synced", func() { + // delete the snapshot after the work is synced with binding + Expect(k8sClient.Delete(ctx, masterSnapshot)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + // mark the work applied which should trigger a reconcile loop and copy the status from the work to the binding + markWorkApplied(&work) + // check the binding status that it should be marked as applied true eventually + verifyBindStatusAppliedNotAvailable(binding, false) + // delete the ParentResourceSnapshotNameAnnotation after the work is synced with binding + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf(placementv1beta1.FirstWorkNameFmt, testCRPName), Namespace: memberClusterNamespaceName}, &work)).Should(Succeed()) + delete(work.Annotations, placementv1beta1.ParentResourceSnapshotNameAnnotation) + Expect(k8sClient.Update(ctx, &work)).Should(Succeed()) + // mark the work available which should trigger a reconcile loop and copy the status from the work to the binding even if the work has no annotation + markWorkAvailable(&work) + // check the binding status that it should be marked as available true eventually + verifyBindStatusAvail(binding, false) + }) + + It("Should mark the binding as failed to sync if the master resource snapshot does not exist and the work do not sync ", func() { + // delete the snapshot after the work is synced with binding + Expect(k8sClient.Delete(ctx, masterSnapshot)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + // mark the work applied which should trigger a reconcile loop and copy the status from the work to the binding + markWorkApplied(&work) + // check the binding status that it should be marked as applied true eventually + verifyBindStatusAppliedNotAvailable(binding, false) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding)).Should(Succeed()) + // update the resource snapshot to the next version that doesn't exist + binding.Spec.ResourceSnapshotName = "next" + Expect(k8sClient.Update(ctx, binding)).Should(Succeed()) + updateRolloutStartedGeneration(&binding) + // check the binding status that it should be marked as override succeed but not synced + Eventually(func() string { + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding)).Should(Succeed()) + wantStatus := placementv1beta1.ResourceBindingStatus{ + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ResourceBindingRolloutStarted), + Status: metav1.ConditionTrue, + Reason: condition.RolloutStartedReason, + ObservedGeneration: binding.GetGeneration(), + }, + { + Type: string(placementv1beta1.ResourceBindingOverridden), + Status: metav1.ConditionFalse, + Reason: condition.OverriddenFailedReason, + ObservedGeneration: binding.GetGeneration(), + }, + }, + FailedPlacements: nil, + } + return cmp.Diff(wantStatus, binding.Status, cmpConditionOption) + }, timeout, interval).Should(BeEmpty(), fmt.Sprintf("binding(%s) mismatch (-want +got)", binding.Name)) + }) }) }) @@ -492,6 +558,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentBindingLabel: binding.Name, placementv1beta1.ParentResourceSnapshotIndexLabel: "1", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -531,6 +602,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.EnvelopeNameLabel: "envelop-configmap", placementv1beta1.EnvelopeNamespaceLabel: "app", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -609,6 +685,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentBindingLabel: binding.Name, placementv1beta1.ParentResourceSnapshotIndexLabel: "2", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -647,6 +728,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.EnvelopeNameLabel: "envelop-configmap", placementv1beta1.EnvelopeNamespaceLabel: "app", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -787,6 +873,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentResourceSnapshotIndexLabel: "2", placementv1beta1.ParentBindingLabel: binding.Name, }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -857,6 +948,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentResourceSnapshotIndexLabel: "2", placementv1beta1.ParentBindingLabel: binding.Name, }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -1082,6 +1178,7 @@ var _ = Describe("Test Work Generator Controller", func() { Context("Test Bound ClusterResourceBinding with a single resource snapshot and valid overrides", func() { var masterSnapshot *placementv1beta1.ClusterResourceSnapshot + var roHash, croHash string BeforeEach(func() { masterSnapshot = generateResourceSnapshot(1, 1, 0, [][]byte{ @@ -1089,21 +1186,25 @@ var _ = Describe("Test Work Generator Controller", func() { }) Expect(k8sClient.Create(ctx, masterSnapshot)).Should(Succeed()) By(fmt.Sprintf("master resource snapshot %s created", masterSnapshot.Name)) - spec := placementv1beta1.ResourceBindingSpec{ - State: placementv1beta1.BindingStateBound, - ResourceSnapshotName: masterSnapshot.Name, - TargetCluster: memberClusterName, - ClusterResourceOverrideSnapshots: []string{ - validClusterResourceOverrideSnapshotName, - }, - ResourceOverrideSnapshots: []placementv1beta1.NamespacedName{ - { - Name: validResourceOverrideSnapshotName, - Namespace: appNamespaceName, - }, + crolist := []string{validClusterResourceOverrideSnapshotName} + roList := []placementv1beta1.NamespacedName{ + { + Name: validResourceOverrideSnapshotName, + Namespace: appNamespaceName, }, } + spec := placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + ResourceSnapshotName: masterSnapshot.Name, + TargetCluster: memberClusterName, + ClusterResourceOverrideSnapshots: crolist, + ResourceOverrideSnapshots: roList, + } createClusterResourceBinding(&binding, spec) + jsonBytes, _ := json.Marshal(roList) + roHash = fmt.Sprintf("%x", sha256.Sum256(jsonBytes)) + jsonBytes, _ = json.Marshal(crolist) + croHash = fmt.Sprintf("%x", sha256.Sum256(jsonBytes)) }) AfterEach(func() { @@ -1146,6 +1247,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentBindingLabel: binding.Name, placementv1beta1.ParentResourceSnapshotIndexLabel: "1", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: croHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: roHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ @@ -1293,7 +1399,6 @@ var _ = Describe("Test Work Generator Controller", func() { // binding should have a finalizer Expect(k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding)).Should(Succeed()) Expect(len(binding.Finalizers)).Should(Equal(1)) - Eventually(func() string { Expect(k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding)).Should(Succeed()) wantStatus := placementv1beta1.ResourceBindingStatus{ @@ -1374,6 +1479,11 @@ var _ = Describe("Test Work Generator Controller", func() { placementv1beta1.ParentBindingLabel: binding.Name, placementv1beta1.ParentResourceSnapshotIndexLabel: "1", }, + Annotations: map[string]string{ + placementv1beta1.ParentResourceSnapshotNameAnnotation: binding.Spec.ResourceSnapshotName, + placementv1beta1.ParentClusterResourceOverrideSnapshotHashAnnotation: emptyHash, + placementv1beta1.ParentResourceOverrideSnapshotHashAnnotation: emptyHash, + }, }, Spec: placementv1beta1.WorkSpec{ Workload: placementv1beta1.WorkloadTemplate{ diff --git a/pkg/scheduler/framework/framework.go b/pkg/scheduler/framework/framework.go index 1d2ca26ca..e2349bcb6 100644 --- a/pkg/scheduler/framework/framework.go +++ b/pkg/scheduler/framework/framework.go @@ -24,6 +24,7 @@ import ( "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" @@ -287,15 +288,21 @@ func (f *framework) RunSchedulingCycleFor(ctx context.Context, crpName string, p // * dangling bindings, i.e., bindings that are associated with a cluster that is no longer // in a normally operating state (the cluster has left the fleet, or is in the state of leaving), // yet has not been marked as unscheduled by the scheduler; and - // + // * deleting bindings, i.e., bindings that have a deletionTimeStamp on them. // Any deleted binding is also ignored. // Note that bindings marked as unscheduled are ignored by the scheduler, as they // are irrelevant to the scheduling cycle. However, we will reconcile them with the latest scheduling // result so that we won't have a ever increasing chain of flip flop bindings. - bound, scheduled, obsolete, unscheduled, dangling := classifyBindings(policy, bindings, clusters) + bound, scheduled, obsolete, unscheduled, dangling, deleting := classifyBindings(policy, bindings, clusters) + + // Remove scheduler CRB cleanup finalizer on all deleting bindings. + if err := f.updateBindings(ctx, deleting, removeFinalizerAndUpdate); err != nil { + klog.ErrorS(err, "Failed to remove finalizers from deleting bindings", "clusterSchedulingPolicySnapshot", policyRef) + return ctrl.Result{}, err + } // Mark all dangling bindings as unscheduled. - if err := f.markAsUnscheduledFor(ctx, dangling); err != nil { + if err := f.updateBindings(ctx, dangling, markUnscheduledForAndUpdate); err != nil { klog.ErrorS(err, "Failed to mark dangling bindings as unscheduled", "clusterSchedulingPolicySnapshot", policyRef) return ctrl.Result{}, err } @@ -349,31 +356,51 @@ func (f *framework) collectBindings(ctx context.Context, crpName string) ([]plac return bindingList.Items, nil } -// markAsUnscheduledFor marks a list of bindings as unscheduled. -func (f *framework) markAsUnscheduledFor(ctx context.Context, bindings []*placementv1beta1.ClusterResourceBinding) error { +// markAsUnscheduledForAndUpdate marks a binding as unscheduled and updates it. +var markUnscheduledForAndUpdate = func(ctx context.Context, hubClient client.Client, binding *placementv1beta1.ClusterResourceBinding) error { + // Remember the previous unscheduledBinding state so that we might be able to revert this change if this + // cluster is being selected again before the resources are removed from it. Need to do a get and set if + // we add more annotations to the binding. + binding.SetAnnotations(map[string]string{placementv1beta1.PreviousBindingStateAnnotation: string(binding.Spec.State)}) + // Mark the unscheduledBinding as unscheduled which can conflict with the rollout controller which also changes the state of a + // unscheduledBinding from "scheduled" to "bound". + binding.Spec.State = placementv1beta1.BindingStateUnscheduled + err := hubClient.Update(ctx, binding, &client.UpdateOptions{}) + if err == nil { + klog.V(2).InfoS("Marked binding as unscheduled", "clusterResourceBinding", klog.KObj(binding)) + } + return err +} + +// removeFinalizerAndUpdate removes scheduler CRB cleanup finalizer from ClusterResourceBinding and updates it. +var removeFinalizerAndUpdate = func(ctx context.Context, hubClient client.Client, binding *placementv1beta1.ClusterResourceBinding) error { + controllerutil.RemoveFinalizer(binding, placementv1beta1.SchedulerCRBCleanupFinalizer) + err := hubClient.Update(ctx, binding, &client.UpdateOptions{}) + if err == nil { + klog.V(2).InfoS("Removed scheduler CRB cleanup finalizer", "clusterResourceBinding", klog.KObj(binding)) + } + return err +} + +// updateBindings iterates over bindings and updates them using the update function provided. +func (f *framework) updateBindings(ctx context.Context, bindings []*placementv1beta1.ClusterResourceBinding, updateFn func(ctx context.Context, client client.Client, binding *placementv1beta1.ClusterResourceBinding) error) error { // issue all the update requests in parallel errs, cctx := errgroup.WithContext(ctx) for _, binding := range bindings { - unscheduledBinding := binding + updateBinding := binding errs.Go(func() error { return retry.OnError(retry.DefaultBackoff, func(err error) bool { return apierrors.IsServiceUnavailable(err) || apierrors.IsServerTimeout(err) || apierrors.IsConflict(err) }, func() error { - // Remember the previous unscheduledBinding state so that we might be able to revert this change if this - // cluster is being selected again before the resources are removed from it. Need to do a get and set if - // we add more annotations to the binding. - unscheduledBinding.SetAnnotations(map[string]string{placementv1beta1.PreviousBindingStateAnnotation: string(unscheduledBinding.Spec.State)}) - // Mark the unscheduledBinding as unscheduled which can conflict with the rollout controller which also changes the state of a - // unscheduledBinding from "scheduled" to "bound". - unscheduledBinding.Spec.State = placementv1beta1.BindingStateUnscheduled - err := f.client.Update(cctx, unscheduledBinding, &client.UpdateOptions{}) - klog.V(2).InfoS("Marking binding as unscheduled", "clusterResourceBinding", klog.KObj(unscheduledBinding), "error", err) - // We will just retry for conflict errors since the scheduler holds the truth here. + err := updateFn(cctx, f.client, updateBinding) + // We will retry on conflicts. if apierrors.IsConflict(err) { // get the binding again to make sure we have the latest version to update again. - return f.client.Get(cctx, client.ObjectKeyFromObject(unscheduledBinding), unscheduledBinding) + if getErr := f.client.Get(cctx, client.ObjectKeyFromObject(updateBinding), updateBinding); getErr != nil { + return getErr + } } return err }) @@ -656,7 +683,7 @@ func (f *framework) manipulateBindings( // // This is set to happen after new bindings are created and old bindings are updated, to // avoid interruptions (deselected then reselected) in a best effort manner. - if err := f.markAsUnscheduledFor(ctx, toDelete); err != nil { + if err := f.updateBindings(ctx, toDelete, markUnscheduledForAndUpdate); err != nil { klog.ErrorS(err, "Failed to mark bindings as unschedulable", "clusterSchedulingPolicySnapshot", policyRef) return err } @@ -809,7 +836,7 @@ func (f *framework) runSchedulingCycleForPickNPlacementType( klog.V(2).InfoS("Downscaling is needed", "clusterSchedulingPolicySnapshot", policyRef, "downscaleCount", downscaleCount) // Mark all obsolete bindings as unscheduled first. - if err := f.markAsUnscheduledFor(ctx, obsolete); err != nil { + if err := f.updateBindings(ctx, obsolete, markUnscheduledForAndUpdate); err != nil { klog.ErrorS(err, "Failed to mark obsolete bindings as unscheduled", "clusterSchedulingPolicySnapshot", policyRef) return ctrl.Result{}, err } @@ -986,10 +1013,10 @@ func (f *framework) downscale(ctx context.Context, scheduled, bound []*placement bindingsToDelete = append(bindingsToDelete, sortedScheduled[i]) } - return sortedScheduled[count:], bound, f.markAsUnscheduledFor(ctx, bindingsToDelete) + return sortedScheduled[count:], bound, f.updateBindings(ctx, bindingsToDelete, markUnscheduledForAndUpdate) case count == len(scheduled): // Trim all scheduled bindings. - return nil, bound, f.markAsUnscheduledFor(ctx, scheduled) + return nil, bound, f.updateBindings(ctx, scheduled, markUnscheduledForAndUpdate) case count < len(scheduled)+len(bound): // Trim all scheduled bindings and part of bound bindings. bindingsToDelete := make([]*placementv1beta1.ClusterResourceBinding, 0, count) @@ -1010,13 +1037,13 @@ func (f *framework) downscale(ctx context.Context, scheduled, bound []*placement bindingsToDelete = append(bindingsToDelete, sortedBound[i]) } - return nil, sortedBound[left:], f.markAsUnscheduledFor(ctx, bindingsToDelete) + return nil, sortedBound[left:], f.updateBindings(ctx, bindingsToDelete, markUnscheduledForAndUpdate) case count == len(scheduled)+len(bound): // Trim all scheduled and bound bindings. bindingsToDelete := make([]*placementv1beta1.ClusterResourceBinding, 0, count) bindingsToDelete = append(bindingsToDelete, scheduled...) bindingsToDelete = append(bindingsToDelete, bound...) - return nil, nil, f.markAsUnscheduledFor(ctx, bindingsToDelete) + return nil, nil, f.updateBindings(ctx, bindingsToDelete, markUnscheduledForAndUpdate) default: // Normally this branch will never run, as an earlier check has guaranteed that // count <= len(scheduled) + len(bound). diff --git a/pkg/scheduler/framework/framework_test.go b/pkg/scheduler/framework/framework_test.go index aeed4174e..07ba7b1ff 100644 --- a/pkg/scheduler/framework/framework_test.go +++ b/pkg/scheduler/framework/framework_test.go @@ -7,6 +7,7 @@ package framework import ( "context" + "errors" "fmt" "log" "os" @@ -16,8 +17,10 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" @@ -65,6 +68,9 @@ var ( lessFuncFilteredCluster = func(filtered1, filtered2 *filteredClusterWithStatus) bool { return filtered1.cluster.Name < filtered2.cluster.Name } + lessFuncBinding = func(binding1, binding2 placementv1beta1.ClusterResourceBinding) bool { + return binding1.Name < binding2.Name + } ) // A few utilities for generating a large number of objects. @@ -352,8 +358,9 @@ func TestClassifyBindings(t *testing.T) { wantObsolete := []*placementv1beta1.ClusterResourceBinding{&obsoleteBinding} wantUnscheduled := []*placementv1beta1.ClusterResourceBinding{&unscheduledBinding} wantDangling := []*placementv1beta1.ClusterResourceBinding{&associatedWithLeavingClusterBinding, &assocaitedWithDisappearedClusterBinding} + wantDeleting := []*placementv1beta1.ClusterResourceBinding{&deletingBinding} - bound, scheduled, obsolete, unscheduled, dangling := classifyBindings(policy, bindings, clusters) + bound, scheduled, obsolete, unscheduled, dangling, deleting := classifyBindings(policy, bindings, clusters) if diff := cmp.Diff(bound, wantBound); diff != "" { t.Errorf("classifyBindings() bound diff (-got, +want): %s", diff) } @@ -373,10 +380,130 @@ func TestClassifyBindings(t *testing.T) { if diff := cmp.Diff(dangling, wantDangling); diff != "" { t.Errorf("classifyBindings() dangling diff (-got, +want) = %s", diff) } + + if diff := cmp.Diff(deleting, wantDeleting); diff != "" { + t.Errorf("classifyBIndings() deleting diff (-got, +want) = %s", diff) + } } -// TestMarkAsUnscheduledFor tests the markAsUnscheduledFor method. -func TestMarkAsUnscheduledFor(t *testing.T) { +func TestUpdateBindingsWithErrors(t *testing.T) { + binding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: bindingName, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme.Scheme). + Build() + + var genericUpdateFn = func(ctx context.Context, hubClient client.Client, binding *placementv1beta1.ClusterResourceBinding) error { + binding.SetLabels(map[string]string{"test-key": "test-value"}) + return hubClient.Update(ctx, binding, &client.UpdateOptions{}) + } + + testCases := []struct { + name string + bindings []*placementv1beta1.ClusterResourceBinding + customClient client.Client + wantErr error + }{ + { + name: "service unavailable error on update, successful get & retry update keeps returning service unavailable error", + bindings: []*placementv1beta1.ClusterResourceBinding{&binding}, + customClient: &errorClient{ + Client: fakeClient, + // set large error retry count to keep returning of update error. + errorForRetryCount: 1000000, + returnUpdateErr: "ServiceUnavailable", + }, + wantErr: k8serrors.NewServiceUnavailable("service is unavailable"), + }, + { + name: "service unavailable error on update, successful get & retry update returns nil", + bindings: []*placementv1beta1.ClusterResourceBinding{&binding}, + customClient: &errorClient{ + Client: fakeClient, + errorForRetryCount: 1, + returnUpdateErr: "ServiceUnavailable", + }, + wantErr: nil, + }, + { + name: "server timeout error on update, successful get & retry update returns nil", + bindings: []*placementv1beta1.ClusterResourceBinding{&binding}, + customClient: &errorClient{ + Client: fakeClient, + errorForRetryCount: 1, + returnUpdateErr: "ServerTimeout", + }, + wantErr: nil, + }, + { + name: "conflict error on update, get failed, retry update returns get error", + bindings: []*placementv1beta1.ClusterResourceBinding{&binding}, + customClient: &errorClient{ + Client: fakeClient, + errorForRetryCount: 1, + returnUpdateErr: "Conflict", + returnGetErr: "GetError", + }, + wantErr: errors.New("get error"), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Construct framework manually instead of using NewFramework() to avoid mocking the controller manager. + f := &framework{ + client: tc.customClient, + } + ctx := context.Background() + gotErr := f.updateBindings(ctx, tc.bindings, genericUpdateFn) + got, want := gotErr != nil, tc.wantErr != nil + if got != want { + t.Fatalf("updateBindings() = %v, want %v", gotErr, tc.wantErr) + } + if got && want && !strings.Contains(gotErr.Error(), tc.wantErr.Error()) { + t.Errorf("updateBindings() = %v, want %v", gotErr, tc.wantErr) + } + }) + } +} + +type errorClient struct { + client.Client + errorForRetryCount int + returnGetErr string + returnUpdateErr string +} + +func (e *errorClient) Get(_ context.Context, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + if e.returnGetErr == "GetError" { + return errors.New("get error") + } + return nil +} + +func (e *errorClient) Update(_ context.Context, _ client.Object, _ ...client.UpdateOption) error { + if e.returnUpdateErr == "ServiceUnavailable" && e.errorForRetryCount > 0 { + e.errorForRetryCount-- + return k8serrors.NewServiceUnavailable("service is unavailable") + } + if e.returnUpdateErr == "ServerTimeout" && e.errorForRetryCount > 0 { + e.errorForRetryCount-- + return k8serrors.NewServerTimeout(schema.GroupResource{Group: placementv1beta1.GroupVersion.Group, Resource: "clusterresourcebinding"}, "UPDATE", 0) + } + if e.returnUpdateErr == "Conflict" && e.errorForRetryCount > 0 { + e.errorForRetryCount-- + return k8serrors.NewConflict(schema.GroupResource{Group: placementv1beta1.GroupVersion.Group, Resource: "clusterresourcebinding"}, "UPDATE", errors.New("conflict")) + } + return nil +} + +// TestUpdateBindingsMarkAsUnscheduledForAndUpdate tests the updateBinding method by passing markUnscheduledForAndUpdate update function. +func TestUpdateBindingsMarkAsUnscheduledForAndUpdate(t *testing.T) { boundBinding := placementv1beta1.ClusterResourceBinding{ ObjectMeta: metav1.ObjectMeta{ Name: bindingName, @@ -403,10 +530,10 @@ func TestMarkAsUnscheduledFor(t *testing.T) { f := &framework{ client: fakeClient, } - // call markAsUnscheduledFor + // call markAsUnscheduledForAndUpdate ctx := context.Background() - if err := f.markAsUnscheduledFor(ctx, []*placementv1beta1.ClusterResourceBinding{&boundBinding, &scheduledBinding}); err != nil { - t.Fatalf("markAsUnscheduledFor() = %v, want no error", err) + if err := f.updateBindings(ctx, []*placementv1beta1.ClusterResourceBinding{&boundBinding, &scheduledBinding}, markUnscheduledForAndUpdate); err != nil { + t.Fatalf("updateBindings() = %v, want no error", err) } // check if the boundBinding has been updated if err := fakeClient.Get(ctx, types.NamespacedName{Name: bindingName}, &boundBinding); err != nil { @@ -446,6 +573,87 @@ func TestMarkAsUnscheduledFor(t *testing.T) { } } +func TestUpdateBindingRemoveFinalizerAndUpdate(t *testing.T) { + boundBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: bindingName, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + } + scheduledBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: altBindingName, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateScheduled, + }, + } + unScheduledBinding := placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: anotherBindingName, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + }, + } + + // setup fake client with bindings + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(&boundBinding, &scheduledBinding, &unScheduledBinding). + Build() + // Construct framework manually instead of using NewFramework() to avoid mocking the controller manager. + f := &framework{ + client: fakeClient, + } + // call markAsUnscheduledForAndUpdate + ctx := context.Background() + if err := f.updateBindings(ctx, []*placementv1beta1.ClusterResourceBinding{&boundBinding, &scheduledBinding, &unScheduledBinding}, removeFinalizerAndUpdate); err != nil { + t.Fatalf("updateBindings() = %v, want no error", err) + } + + var clusterResourceBindingList placementv1beta1.ClusterResourceBindingList + if err := f.client.List(ctx, &clusterResourceBindingList); err != nil { + t.Fatalf("List cluster resource boundBindings returned %v, want no error", err) + } + + want := []placementv1beta1.ClusterResourceBinding{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: bindingName, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateBound, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: altBindingName, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateScheduled, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: anotherBindingName, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + State: placementv1beta1.BindingStateUnscheduled, + }, + }, + } + + if diff := cmp.Diff(clusterResourceBindingList.Items, want, ignoreTypeMetaAPIVersionKindFields, ignoreObjectMetaResourceVersionField, cmpopts.SortSlices(lessFuncBinding)); diff != "" { + t.Errorf("diff (-got, +want): %s", diff) + } +} + // TestRunPreFilterPlugins tests the runPreFilterPlugins method. func TestRunPreFilterPlugins(t *testing.T) { dummyPreFilterPluginNameA := fmt.Sprintf(dummyAllPurposePluginNameFormat, 0) @@ -1274,6 +1482,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1296,6 +1505,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1318,6 +1528,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1510,6 +1721,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1619,6 +1831,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1641,6 +1854,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1663,6 +1877,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1706,6 +1921,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1728,6 +1944,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -1813,6 +2030,7 @@ func TestCrossReferencePickedClustersAndDeDupBindings(t *testing.T) { Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, diff --git a/pkg/scheduler/framework/frameworkutils.go b/pkg/scheduler/framework/frameworkutils.go index ac8f26cd9..525031747 100644 --- a/pkg/scheduler/framework/frameworkutils.go +++ b/pkg/scheduler/framework/frameworkutils.go @@ -29,16 +29,18 @@ import ( // - dangling bindings, i.e., bindings that are associated with a cluster that is no longer in // a normally operating state (the cluster has left the fleet, or is in the state of leaving), // yet has not been marked as unscheduled by the scheduler; and -// - unscheduled bindings, i.e., bindings that are marked to be removed by the scheduler. +// - unscheduled bindings, i.e., bindings that are marked to be removed by the scheduler; and // - obsolete bindings, i.e., bindings that are no longer associated with the latest scheduling -// policy. -func classifyBindings(policy *placementv1beta1.ClusterSchedulingPolicySnapshot, bindings []placementv1beta1.ClusterResourceBinding, clusters []clusterv1beta1.MemberCluster) (bound, scheduled, obsolete, unscheduled, dangling []*placementv1beta1.ClusterResourceBinding) { +// policy; and +// - deleting bindings, i.e., bindings that have a deletionTimeStamp on them. +func classifyBindings(policy *placementv1beta1.ClusterSchedulingPolicySnapshot, bindings []placementv1beta1.ClusterResourceBinding, clusters []clusterv1beta1.MemberCluster) (bound, scheduled, obsolete, unscheduled, dangling, deleting []*placementv1beta1.ClusterResourceBinding) { // Pre-allocate arrays. bound = make([]*placementv1beta1.ClusterResourceBinding, 0, len(bindings)) scheduled = make([]*placementv1beta1.ClusterResourceBinding, 0, len(bindings)) obsolete = make([]*placementv1beta1.ClusterResourceBinding, 0, len(bindings)) unscheduled = make([]*placementv1beta1.ClusterResourceBinding, 0, len(bindings)) dangling = make([]*placementv1beta1.ClusterResourceBinding, 0, len(bindings)) + deleting = make([]*placementv1beta1.ClusterResourceBinding, 0, len(bindings)) // Build a map for clusters for quick lookup. clusterMap := make(map[string]clusterv1beta1.MemberCluster) @@ -52,11 +54,8 @@ func classifyBindings(policy *placementv1beta1.ClusterSchedulingPolicySnapshot, switch { case !binding.DeletionTimestamp.IsZero(): - // Ignore any binding that has been deleted. - // - // Note that the scheduler will not add any cleanup scheduler to a binding, as - // in normal operations bound and scheduled bindings will not be deleted, and - // unscheduled bindings are disregarded by the scheduler. + // we need remove scheduler CRB cleanup finalizer from deleting ClusterResourceBindings. + deleting = append(deleting, &binding) case binding.Spec.State == placementv1beta1.BindingStateUnscheduled: // we need to remember those bindings so that we will not create another one. unscheduled = append(unscheduled, &binding) @@ -83,7 +82,7 @@ func classifyBindings(policy *placementv1beta1.ClusterSchedulingPolicySnapshot, } } - return bound, scheduled, obsolete, unscheduled, dangling + return bound, scheduled, obsolete, unscheduled, dangling, deleting } // bindingWithPatch is a helper struct that includes a binding that needs to be patched and the @@ -186,6 +185,7 @@ func crossReferencePickedClustersAndDeDupBindings( Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -684,6 +684,7 @@ func crossReferenceValidTargetsWithBindings( Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 60ee67f3c..a1734e02b 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -282,7 +282,7 @@ func (s *Scheduler) cleanUpAllBindingsFor(ctx context.Context, crp *fleetv1beta1 return controller.NewAPIServerError(false, err) } - // Remove the scheduler cleanup finalizer from all the bindings, and delete them. + // Remove scheduler CRB cleanup finalizer from deleting bindings. // // Note that once a CRP has been marked for deletion, it will no longer enter the scheduling cycle, // so any cleanup finalizer has to be removed here. @@ -291,16 +291,19 @@ func (s *Scheduler) cleanUpAllBindingsFor(ctx context.Context, crp *fleetv1beta1 // the scheduler no longer marks them as deleting and waits for another controller to actually // run the deletion. for idx := range bindingList.Items { - binding := bindingList.Items[idx] + binding := &bindingList.Items[idx] + controllerutil.RemoveFinalizer(binding, fleetv1beta1.SchedulerCRBCleanupFinalizer) + if err := s.client.Update(ctx, binding); err != nil { + klog.ErrorS(err, "Failed to remove scheduler reconcile finalizer from cluster resource binding", "clusterResourceBinding", klog.KObj(binding)) + return controller.NewUpdateIgnoreConflictError(err) + } // Delete the binding if it has not been marked for deletion yet. if binding.DeletionTimestamp == nil { - if err := s.client.Delete(ctx, &binding); err != nil && !errors.IsNotFound(err) { - klog.ErrorS(err, "Failed to delete binding", "clusterResourceBinding", klog.KObj(&binding)) + if err := s.client.Delete(ctx, binding); err != nil && !errors.IsNotFound(err) { + klog.ErrorS(err, "Failed to delete binding", "clusterResourceBinding", klog.KObj(binding)) return controller.NewAPIServerError(false, err) } } - - // Note that the scheduler will not add any cleanup finalizer to a binding. } // All bindings have been deleted; remove the scheduler cleanup finalizer from the CRP. diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 7373b3d25..bdf0109dd 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -70,6 +70,7 @@ func TestCleanUpAllBindingsFor(t *testing.T) { Labels: map[string]string{ fleetv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{fleetv1beta1.SchedulerCRBCleanupFinalizer}, }, }, { @@ -78,6 +79,7 @@ func TestCleanUpAllBindingsFor(t *testing.T) { Labels: map[string]string{ fleetv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{fleetv1beta1.SchedulerCRBCleanupFinalizer}, }, }, } diff --git a/pkg/scheduler/watchers/clusterresourcebinding/controller_integration_test.go b/pkg/scheduler/watchers/clusterresourcebinding/controller_integration_test.go new file mode 100644 index 000000000..bc2951185 --- /dev/null +++ b/pkg/scheduler/watchers/clusterresourcebinding/controller_integration_test.go @@ -0,0 +1,122 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourcebinding + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +const ( + eventuallyDuration = time.Second * 5 + eventuallyInterval = time.Millisecond * 250 + consistentlyDuration = time.Second + consistentlyInterval = time.Millisecond * 200 + + crbName = "test-crb" + crpName = "test-crp" + clusterName = "test-cluster" +) + +var ( + noKeyEnqueuedActual = func() error { + if queueLen := keyCollector.Len(); queueLen != 0 { + return fmt.Errorf("work queue is not empty: current len %d, want 0", queueLen) + } + return nil + } + + expectedKeySetEnqueuedActual = func() error { + if isAllPresent, absentKeys := keyCollector.IsPresent(crpName); !isAllPresent { + return fmt.Errorf("expected key(s) %v is not found", absentKeys) + } + + if queueLen := keyCollector.Len(); queueLen != 1 { + return fmt.Errorf("more than one key is enqueued: current len %d, want 1", queueLen) + } + + return nil + } +) + +// This container cannot be run in parallel since we are trying to access a common shared queue. +var _ = Describe("scheduler - cluster resource binding watcher", Ordered, func() { + BeforeAll(func() { + Eventually(noKeyEnqueuedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Workqueue is not empty") + Consistently(noKeyEnqueuedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Workqueue is not empty") + }) + + Context("create, update & delete cluster resource binding", func() { + BeforeAll(func() { + affinityScore := int32(1) + topologyScore := int32(1) + crb := fleetv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbName, + Labels: map[string]string{ + fleetv1beta1.CRPTrackingLabel: crpName, + }, + Finalizers: []string{fleetv1beta1.SchedulerCRBCleanupFinalizer}, + }, + Spec: fleetv1beta1.ResourceBindingSpec{ + State: fleetv1beta1.BindingStateScheduled, + // Leave the associated resource snapshot name empty; it is up to another controller + // to fulfill this field. + SchedulingPolicySnapshotName: "test-policy", + TargetCluster: clusterName, + ClusterDecision: fleetv1beta1.ClusterDecision{ + ClusterName: clusterName, + Selected: true, + ClusterScore: &fleetv1beta1.ClusterScore{ + AffinityScore: &affinityScore, + TopologySpreadScore: &topologyScore, + }, + Reason: "test-reason", + }, + }, + } + // Create cluster resource binding. + Expect(hubClient.Create(ctx, &crb)).Should(Succeed(), "Failed to create cluster resource binding") + }) + + It("should not enqueue the CRP name when CRB is created", func() { + Consistently(noKeyEnqueuedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Workqueue is not empty") + }) + + It("update cluster resource binding", func() { + var crb fleetv1beta1.ClusterResourceBinding + Expect(hubClient.Get(ctx, client.ObjectKey{Name: crbName}, &crb)).Should(Succeed()) + crb.Spec.State = fleetv1beta1.BindingStateBound + Expect(hubClient.Update(ctx, &crb)).Should(Succeed()) + }) + + It("should not enqueue the CRP name when it CRB is updated", func() { + Consistently(noKeyEnqueuedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Workqueue is not empty") + }) + + It("delete cluster resource binding", func() { + var crb fleetv1beta1.ClusterResourceBinding + Expect(hubClient.Get(ctx, client.ObjectKey{Name: crbName}, &crb)).Should(Succeed()) + Expect(hubClient.Delete(ctx, &crb)).Should(Succeed()) + }) + + It("should enqueue CRP name when CRB is deleted", func() { + Eventually(expectedKeySetEnqueuedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Workqueue is either empty or it contains more than one element") + Consistently(expectedKeySetEnqueuedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Workqueue is either empty or it contains more than one element") + }) + + AfterAll(func() { + keyCollector.Reset() + }) + }) +}) diff --git a/pkg/scheduler/watchers/clusterresourcebinding/suite_test.go b/pkg/scheduler/watchers/clusterresourcebinding/suite_test.go new file mode 100644 index 000000000..a0c4a5f4c --- /dev/null +++ b/pkg/scheduler/watchers/clusterresourcebinding/suite_test.go @@ -0,0 +1,105 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourcebinding + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/scheduler/queue" + "go.goms.io/fleet/test/utils/keycollector" +) + +var ( + hubTestEnv *envtest.Environment + hubClient client.Client + ctx context.Context + cancel context.CancelFunc + keyCollector *keycollector.SchedulerWorkqueueKeyCollector +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Scheduler Source Cluster Resource Binding Controller Suite") +} + +var _ = BeforeSuite(func() { + klog.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrap the test environment") + + // Start the hub cluster. + hubTestEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + hubCfg, err := hubTestEnv.Start() + Expect(err).ToNot(HaveOccurred(), "Failed to start test environment") + Expect(hubCfg).ToNot(BeNil(), "Hub cluster configuration is nil") + + // Add custom APIs to the runtime scheme. + Expect(fleetv1beta1.AddToScheme(scheme.Scheme)).Should(Succeed()) + + // Set up a client for the hub cluster. + hubClient, err = client.New(hubCfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred(), "Failed to create hub cluster client") + Expect(hubClient).ToNot(BeNil(), "Hub cluster client is nil") + + // Set up a controller manager and let it manage the hub cluster controller. + ctrlMgr, err := ctrl.NewManager(hubCfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + }) + Expect(err).NotTo(HaveOccurred(), "Failed to create controller manager") + + schedulerWorkQueue := queue.NewSimpleClusterResourcePlacementSchedulingQueue() + + // Create ClusterResourceBinding watcher reconciler. + reconciler := &Reconciler{ + Client: hubClient, + SchedulerWorkQueue: schedulerWorkQueue, + } + err = reconciler.SetupWithManager(ctrlMgr) + Expect(err).ToNot(HaveOccurred(), "Failed to set up controller with controller manager") + + // Start the key collector. + keyCollector = keycollector.NewSchedulerWorkqueueKeyCollector(schedulerWorkQueue) + go func() { + keyCollector.Run(ctx) + }() + + // Start the controller manager. + go func() { + defer GinkgoRecover() + err := ctrlMgr.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "Failed to start controller manager") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + cancel() + + By("tearing down the test environment") + Expect(hubTestEnv.Stop()).Should(Succeed(), "Failed to stop test environment") +}) diff --git a/pkg/scheduler/watchers/clusterresourcebinding/watcher.go b/pkg/scheduler/watchers/clusterresourcebinding/watcher.go new file mode 100644 index 000000000..8e2f6b900 --- /dev/null +++ b/pkg/scheduler/watchers/clusterresourcebinding/watcher.go @@ -0,0 +1,103 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package clusterresourcebinding + +import ( + "context" + "fmt" + "time" + + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/scheduler/queue" + "go.goms.io/fleet/pkg/utils/controller" +) + +// Reconciler reconciles the deletion of a ClusterResourceBinding. +type Reconciler struct { + // Client is the client the controller uses to access the hub cluster. + client.Client + // SchedulerWorkQueue is the workqueue in use by the scheduler. + SchedulerWorkQueue queue.ClusterResourcePlacementSchedulingQueueWriter +} + +// Reconcile reconciles the CRB. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + crbRef := klog.KRef("", req.Name) + startTime := time.Now() + klog.V(2).InfoS("Scheduler source reconciliation starts", "clusterResourceBinding", crbRef) + defer func() { + latency := time.Since(startTime).Milliseconds() + klog.V(2).InfoS("Scheduler source reconciliation ends", "clusterResourceBinding", crbRef, "latency", latency) + }() + + // Retrieve the CRB. + crb := &fleetv1beta1.ClusterResourceBinding{} + if err := r.Client.Get(ctx, req.NamespacedName, crb); err != nil { + klog.ErrorS(err, "Failed to get cluster resource binding", "clusterResourceBinding", crbRef) + return ctrl.Result{}, controller.NewAPIServerError(true, client.IgnoreNotFound(err)) + } + + // Check if the CRB has been deleted and has the scheduler CRB cleanup finalizer. + if crb.DeletionTimestamp != nil && controllerutil.ContainsFinalizer(crb, fleetv1beta1.SchedulerCRBCleanupFinalizer) { + // The CRB has been deleted and still has the scheduler CRB cleanup finalizer; enqueue it's corresponding CRP + // for the scheduler to process. + crpName, exist := crb.GetLabels()[fleetv1beta1.CRPTrackingLabel] + if !exist { + err := controller.NewUnexpectedBehaviorError(fmt.Errorf("clusterResourceBinding %s doesn't have CRP tracking label", crb.Name)) + klog.ErrorS(err, "Failed to enqueue CRP name for CRB") + // error cannot be retried. + return ctrl.Result{}, nil + } + r.SchedulerWorkQueue.AddRateLimited(queue.ClusterResourcePlacementKey(crpName)) + } + + // No action is needed for the scheduler to take in other cases. + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the manager. +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + customPredicate := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + // Ignore creation events. + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // Ignore deletion events (events emitted when the object is actually removed + // from storage). + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + // Check if the update event is valid. + if e.ObjectOld == nil || e.ObjectNew == nil { + err := controller.NewUnexpectedBehaviorError(fmt.Errorf("update event is invalid")) + klog.ErrorS(err, "Failed to process update event") + return false + } + + // Check if the deletion timestamp has been set. + oldDeletionTimestamp := e.ObjectOld.GetDeletionTimestamp() + newDeletionTimestamp := e.ObjectNew.GetDeletionTimestamp() + if oldDeletionTimestamp == nil && newDeletionTimestamp != nil { + return true + } + + return false + }, + } + + return ctrl.NewControllerManagedBy(mgr). + For(&fleetv1beta1.ClusterResourceBinding{}). + WithEventFilter(customPredicate). + Complete(r) +} diff --git a/pkg/utils/cloudconfig/azure/config.go b/pkg/utils/cloudconfig/azure/config.go new file mode 100644 index 000000000..d37f5bffe --- /dev/null +++ b/pkg/utils/cloudconfig/azure/config.go @@ -0,0 +1,145 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +// Package azure provides utilities to load, parse, and validate Azure cloud configuration. +package azure + +import ( + "fmt" + "io" + "os" + "strings" + + "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/policy/ratelimit" + "sigs.k8s.io/cloud-provider-azure/pkg/consts" +) + +// CloudConfig holds the configuration parsed from the --cloud-config flag. +type CloudConfig struct { + azclient.ARMClientConfig `json:",inline" mapstructure:",squash"` + azclient.AzureAuthConfig `json:",inline" mapstructure:",squash"` + *ratelimit.Config `json:",inline" mapstructure:",squash"` // if nil, the rate limiting will be enabled by default + + // azure resource location + Location string `json:"location,omitempty" mapstructure:"location,omitempty"` + // subscription ID + SubscriptionID string `json:"subscriptionID,omitempty" mapstructure:"subscriptionID,omitempty"` + // default resource group where the azure resources are deployed + ResourceGroup string `json:"resourceGroup,omitempty" mapstructure:"resourceGroup,omitempty"` + // name of the virtual network of cluster + VnetName string `json:"vnetName,omitempty" mapstructure:"vnetName,omitempty"` + // name of the resource group where the virtual network is deployed + VnetResourceGroup string `json:"vnetResourceGroup,omitempty" mapstructure:"vnetResourceGroup,omitempty"` +} + +// NewCloudConfigFromFile loads cloud config from a file given the file path. +func NewCloudConfigFromFile(filePath string) (*CloudConfig, error) { + if filePath == "" { + return nil, fmt.Errorf("failed to load cloud config: file path is empty") + } + + var config CloudConfig + configReader, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open cloud config file: %w, file path: %s", err, filePath) + } + defer configReader.Close() + + contents, err := io.ReadAll(configReader) + if err != nil { + return nil, fmt.Errorf("failed to read cloud config file: %w, file path: %s", err, filePath) + } + + if err := yaml.Unmarshal(contents, &config); err != nil { + return nil, fmt.Errorf("failed to unmarshal cloud config: %w, file path: %s", err, filePath) + } + + config.trimSpace() + if err := config.validate(); err != nil { + return nil, fmt.Errorf("failed to validate cloud config: %w, file contents: `%s`", err, string(contents)) + } + + return &config, nil +} + +// SetUserAgent sets the user agent string to access Azure resources. +func (cfg *CloudConfig) SetUserAgent(userAgent string) { + cfg.UserAgent = userAgent +} + +func (cfg *CloudConfig) validate() error { + if cfg.Cloud == "" { + return fmt.Errorf("cloud is empty") + } + + if cfg.Location == "" { + return fmt.Errorf("location is empty") + } + + if cfg.SubscriptionID == "" { + return fmt.Errorf("subscription ID is empty") + } + + if cfg.ResourceGroup == "" { + return fmt.Errorf("resource group is empty") + } + + if cfg.VnetName == "" { + return fmt.Errorf("virtual network name is empty") + } + + if cfg.VnetResourceGroup == "" { + cfg.VnetResourceGroup = cfg.ResourceGroup + } + + if !cfg.UseManagedIdentityExtension { + if cfg.UserAssignedIdentityID != "" { + return fmt.Errorf("useManagedIdentityExtension needs to be true when userAssignedIdentityID is provided") + } + if cfg.AADClientID == "" || cfg.AADClientSecret == "" { + return fmt.Errorf("AAD client ID or AAD client secret is empty") + } + } + + // if not specified, apply default rate limit config + if cfg.Config == nil { + cfg.Config = &ratelimit.Config{CloudProviderRateLimit: true} + } + + if cfg.CloudProviderRateLimit { + // Assign read rate limit defaults if no configuration was passed in. + if cfg.CloudProviderRateLimitQPS == 0 { + cfg.CloudProviderRateLimitQPS = consts.RateLimitQPSDefault + } + if cfg.CloudProviderRateLimitBucket == 0 { + cfg.CloudProviderRateLimitBucket = consts.RateLimitBucketDefault + } + // Assign write rate limit defaults if no configuration was passed in. + if cfg.CloudProviderRateLimitQPSWrite == 0 { + cfg.CloudProviderRateLimitQPSWrite = cfg.CloudProviderRateLimitQPS + } + if cfg.CloudProviderRateLimitBucketWrite == 0 { + cfg.CloudProviderRateLimitBucketWrite = cfg.CloudProviderRateLimitBucket + } + } + + return nil +} + +func (cfg *CloudConfig) trimSpace() { + cfg.Cloud = strings.TrimSpace(cfg.Cloud) + cfg.TenantID = strings.TrimSpace(cfg.TenantID) + cfg.UserAgent = strings.TrimSpace(cfg.UserAgent) + cfg.SubscriptionID = strings.TrimSpace(cfg.SubscriptionID) + cfg.Location = strings.TrimSpace(cfg.Location) + cfg.ResourceGroup = strings.TrimSpace(cfg.ResourceGroup) + cfg.UserAssignedIdentityID = strings.TrimSpace(cfg.UserAssignedIdentityID) + cfg.AADClientID = strings.TrimSpace(cfg.AADClientID) + cfg.AADClientSecret = strings.TrimSpace(cfg.AADClientSecret) + cfg.VnetName = strings.TrimSpace(cfg.VnetName) + cfg.VnetResourceGroup = strings.TrimSpace(cfg.VnetResourceGroup) +} diff --git a/pkg/utils/cloudconfig/azure/config_test.go b/pkg/utils/cloudconfig/azure/config_test.go new file mode 100644 index 000000000..8c094a128 --- /dev/null +++ b/pkg/utils/cloudconfig/azure/config_test.go @@ -0,0 +1,598 @@ +package azure + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/policy/ratelimit" + "sigs.k8s.io/cloud-provider-azure/pkg/consts" +) + +func TestTrimSpace(t *testing.T) { + t.Run("test spaces are trimmed", func(t *testing.T) { + config := CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: " test \n", + UserAgent: " test \n", + TenantID: " test \t \n", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: " test \n", + AADClientID: "\n test \n", + AADClientSecret: " test \n", + }, + Location: " test \n", + SubscriptionID: " test \n", + ResourceGroup: "\r\n test \n", + VnetName: " test ", + VnetResourceGroup: " \t test ", + } + + want := CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "test", + TenantID: "test", + UserAgent: "test", + }, + Location: "test", + SubscriptionID: "test", + ResourceGroup: "test", + VnetName: "test", + VnetResourceGroup: "test", + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "test", + AADClientID: "test", + AADClientSecret: "test", + }, + } + config.trimSpace() + if diff := cmp.Diff(config, want); diff != "" { + t.Errorf("trimSpace() mismatch (-got +want):\n%s", diff) + } + }) +} + +func TestSetUserAgent(t *testing.T) { + config := &CloudConfig{} + config.SetUserAgent("test") + if config.UserAgent != "test" { + t.Errorf("SetUserAgent(test) = %s, want test", config.UserAgent) + } +} + +func TestValidate(t *testing.T) { + tests := map[string]struct { + config *CloudConfig + wantConfig *CloudConfig + wantPass bool + }{ + "Cloud empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + }, + wantPass: false, + }, + "Location empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + }, + wantPass: false, + }, + "SubscriptionID empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "", + ResourceGroup: "v", + VnetName: "vn", + }, + wantPass: false, + }, + "ResourceGroup empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "", + VnetName: "vn", + }, + wantPass: false, + }, + "VnetName empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "", + }, + wantPass: false, + }, + "VnetResourceGroup empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 1.0, + CloudProviderRateLimitBucket: 5, + CloudProviderRateLimitQPSWrite: 1.0, + CloudProviderRateLimitBucketWrite: 5, + }, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantPass: true, + }, + "ratelimit.Config nil": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: nil, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "a", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantPass: true, + }, + "UserAssignedIdentityID not empty when UseManagedIdentityExtension is false": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "aaaa", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + }, + wantPass: false, + }, + "AADClientID empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + }, + wantPass: false, + }, + "AADClientSecret empty": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + }, + wantPass: false, + }, + "ratelimit.Config values are zero": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 0, + CloudProviderRateLimitBucket: 0, + CloudProviderRateLimitQPSWrite: 0, + CloudProviderRateLimitBucketWrite: 0, + }, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantPass: true, + }, + "ratelimit.Config with non-zero values": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 2, + CloudProviderRateLimitBucket: 4, + CloudProviderRateLimitQPSWrite: 2, + CloudProviderRateLimitBucketWrite: 4, + }, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: 2, + CloudProviderRateLimitBucket: 4, + CloudProviderRateLimitQPSWrite: 2, + CloudProviderRateLimitBucketWrite: 4, + }, + }, + wantPass: true, + }, + "CloudProviderRateLimit is false": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + Config: &ratelimit.Config{ + CloudProviderRateLimit: false, + }, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: false, + }, + }, + wantPass: true, + }, + "has all required properties with secret and default values": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: false, + UserAssignedIdentityID: "", + AADClientID: "1", + AADClientSecret: "2", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantPass: true, + }, + "has all required properties with msi and specified values": { + config: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "u", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "c", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "u", + }, + Location: "l", + SubscriptionID: "s", + ResourceGroup: "v", + VnetName: "vn", + VnetResourceGroup: "v", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantPass: true, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + err := test.config.validate() + if got := err == nil; got != test.wantPass { + t.Fatalf("validate() = got %v, want %v", got, test.wantPass) + } + + if err == nil { + if diff := cmp.Diff(test.config, test.wantConfig); diff != "" { + t.Errorf("validate() mismatch (-got +want):\n%s", diff) + } + } + }) + } +} + +func TestNewCloudConfigFromFile(t *testing.T) { + tests := map[string]struct { + filePath string + wantErr bool + wantConfig *CloudConfig + }{ + "file path is empty": { + filePath: "", + wantErr: true, + }, + "failed to open file": { + filePath: "./test/not_exist.json", + wantErr: true, + }, + "failed to unmarshal file": { + filePath: "./test/azure_config_nojson.txt", + wantErr: true, + }, + "failed to validate config": { + filePath: "./test/azure_invalid_config.json", + wantErr: true, + }, + "succeeded to load config": { + filePath: "./test/azure_valid_config.json", + wantConfig: &CloudConfig{ + ARMClientConfig: azclient.ARMClientConfig{ + Cloud: "AzurePublicCloud", + TenantID: "00000000-0000-0000-0000-000000000000", + }, + AzureAuthConfig: azclient.AzureAuthConfig{ + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "11111111-1111-1111-1111-111111111111", + AADClientID: "", + AADClientSecret: "", + }, + Location: "eastus", + SubscriptionID: "00000000-0000-0000-0000-000000000000", + ResourceGroup: "test-rg", + VnetName: "test-vnet", + VnetResourceGroup: "test-rg", + Config: &ratelimit.Config{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPS: consts.RateLimitQPSDefault, + CloudProviderRateLimitBucket: consts.RateLimitBucketDefault, + CloudProviderRateLimitBucketWrite: consts.RateLimitBucketDefault, + CloudProviderRateLimitQPSWrite: consts.RateLimitQPSDefault, + }, + }, + wantErr: false, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + config, err := NewCloudConfigFromFile(test.filePath) + if got := err != nil; got != test.wantErr { + t.Fatalf("Failed to run NewCloudConfigFromFile(%s): got %v, want %v", test.filePath, got, test.wantErr) + } + if diff := cmp.Diff(config, test.wantConfig); diff != "" { + t.Errorf("NewCloudConfigFromFile(%s) mismatch (-got +want):\n%s", test.filePath, diff) + } + }) + } +} diff --git a/pkg/utils/cloudconfig/azure/test/azure_config_nojson.txt b/pkg/utils/cloudconfig/azure/test/azure_config_nojson.txt new file mode 100644 index 000000000..241a08026 --- /dev/null +++ b/pkg/utils/cloudconfig/azure/test/azure_config_nojson.txt @@ -0,0 +1 @@ +This is an invalid json file for testing purposes. \n \ No newline at end of file diff --git a/pkg/utils/cloudconfig/azure/test/azure_invalid_config.json b/pkg/utils/cloudconfig/azure/test/azure_invalid_config.json new file mode 100644 index 000000000..0fd9aa60d --- /dev/null +++ b/pkg/utils/cloudconfig/azure/test/azure_invalid_config.json @@ -0,0 +1,10 @@ +{ + "cloud": "AzurePublicCloud", + "tenantId": "00000000-0000-0000-0000-000000000000", + "subscriptionId": "00000000-0000-0000-0000-000000000000", + "useManagedIdentityExtension": false, + "aadClientId": "00000000-0000-0000-0000-000000000000", + "resourceGroup": " test-rg ", + "location": " eastus ", + "vnetName": "test -vnet" +} diff --git a/pkg/utils/cloudconfig/azure/test/azure_valid_config.json b/pkg/utils/cloudconfig/azure/test/azure_valid_config.json new file mode 100644 index 000000000..4c938e78c --- /dev/null +++ b/pkg/utils/cloudconfig/azure/test/azure_valid_config.json @@ -0,0 +1,11 @@ +{ + "cloud": "AzurePublicCloud", + "tenantId": "00000000-0000-0000-0000-000000000000", + "subscriptionId": "00000000-0000-0000-0000-000000000000", + "useManagedIdentityExtension": true, + "userAssignedIdentityID": "11111111-1111-1111-1111-111111111111", + "resourceGroup": "test-rg", + "location": "eastus", + "vnetName": "test-vnet", + "vnetResourceGroup": "test-rg" +} diff --git a/test/apis/v1alpha1/zz_generated.deepcopy.go b/test/apis/v1alpha1/zz_generated.deepcopy.go index 0b5d2e30b..ef7e4433a 100644 --- a/test/apis/v1alpha1/zz_generated.deepcopy.go +++ b/test/apis/v1alpha1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ Licensed under the MIT license. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index a9698c4e0..dddf87dd4 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -420,7 +420,7 @@ func resourcePlacementRolloutCompletedConditions(generation int64, resourceIsTra } } -func resourcePlacementRolloutFailedConditions(generation int64) []metav1.Condition { +func resourcePlacementScheduleFailedConditions(generation int64) []metav1.Condition { return []metav1.Condition{ { Type: string(placementv1beta1.ResourceScheduledConditionType), @@ -689,7 +689,7 @@ func customizedCRPStatusUpdatedActual(crpName string, } for i := 0; i < len(wantUnselectedClusters); i++ { wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.ResourcePlacementStatus{ - Conditions: resourcePlacementRolloutFailedConditions(crp.Generation), + Conditions: resourcePlacementScheduleFailedConditions(crp.Generation), }) } diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index 1db441dfa..7537c9dd9 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -17,6 +17,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -472,6 +473,162 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { }) }) + Context("Test a CRP place workload successful and update it to be failed and then delete the resource snapshot,"+ + "rollout should eventually be successful after we correct the image", Ordered, func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + workNamespace := appNamespace() + var wantSelectedResources []placementv1beta1.ResourceIdentifier + var testDeployment appv1.Deployment + + BeforeAll(func() { + // Create the test resources. + readDeploymentTestManifest(&testDeployment) + wantSelectedResources = []placementv1beta1.ResourceIdentifier{ + { + Kind: utils.NamespaceKind, + Name: workNamespace.Name, + Version: corev1.SchemeGroupVersion.Version, + }, + { + Group: appv1.SchemeGroupVersion.Group, + Version: appv1.SchemeGroupVersion.Version, + Kind: utils.DeploymentKind, + Name: testDeployment.Name, + Namespace: workNamespace.Name, + }, + } + }) + + It("create the deployment resource in the namespace", func() { + Expect(hubClient.Create(ctx, &workNamespace)).To(Succeed(), "Failed to create namespace %s", workNamespace.Name) + testDeployment.Namespace = workNamespace.Name + Expect(hubClient.Create(ctx, &testDeployment)).To(Succeed(), "Failed to create test deployment %s", testDeployment.Name) + }) + + It("create the CRP that select the namespace", func() { + crp := buildCRPForSafeRollout() + crp.Spec.RevisionHistoryLimit = ptr.To(int32(1)) + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + }) + + It("should update CRP status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(wantSelectedResources, allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") + }) + + It("should place the resources on all member clusters", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + workResourcesPlacedActual := waitForDeploymentPlacementToReady(memberCluster, &testDeployment) + Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName) + } + }) + + It("change the image name in deployment, to make it unavailable", func() { + Eventually(func() error { + var dep appv1.Deployment + err := hubClient.Get(ctx, types.NamespacedName{Name: testDeployment.Name, Namespace: testDeployment.Namespace}, &dep) + if err != nil { + return err + } + dep.Spec.Template.Spec.Containers[0].Image = randomImageName + return hubClient.Update(ctx, &dep) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to change the image name in deployment") + }) + + It("should update CRP status on deployment failed as expected", func() { + failedDeploymentResourceIdentifier := placementv1beta1.ResourceIdentifier{ + Group: appv1.SchemeGroupVersion.Group, + Version: appv1.SchemeGroupVersion.Version, + Kind: utils.DeploymentKind, + Name: testDeployment.Name, + Namespace: testDeployment.Namespace, + } + crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1", 2) + Eventually(crpStatusActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") + }) + + It("update work to trigger a work generator reconcile", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx].ClusterName + namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster) + workName := fmt.Sprintf(placementv1beta1.FirstWorkNameFmt, crpName) + work := placementv1beta1.Work{} + Expect(hubClient.Get(ctx, types.NamespacedName{Name: workName, Namespace: namespaceName}, &work)).Should(Succeed(), "Failed to get the work") + if work.Status.ManifestConditions != nil { + work.Status.ManifestConditions = nil + } else { + meta.SetStatusCondition(&work.Status.Conditions, metav1.Condition{ + Type: placementv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: "WorkNotAvailable", + }) + } + Expect(hubClient.Status().Update(ctx, &work)).Should(Succeed(), "Failed to update the work") + } + }) + + It("change the image name in deployment, to roll over the resourcesnapshot", func() { + crsList := &placementv1beta1.ClusterResourceSnapshotList{} + Expect(hubClient.List(ctx, crsList, client.MatchingLabels{placementv1beta1.CRPTrackingLabel: crpName})).Should(Succeed(), "Failed to list the resourcesnapshot") + Expect(len(crsList.Items) == 1).Should(BeTrue()) + oldCRS := crsList.Items[0].Name + Expect(hubClient.Get(ctx, types.NamespacedName{Name: testDeployment.Name, Namespace: testDeployment.Namespace}, &testDeployment)).Should(Succeed(), "Failed to get deployment") + testDeployment.Spec.Template.Spec.Containers[0].Image = "extra-snapshot" + Expect(hubClient.Update(ctx, &testDeployment)).Should(Succeed(), "Failed to change the image name in deployment") + // wait for the new resourcesnapshot to be created + Eventually(func() bool { + Expect(hubClient.List(ctx, crsList, client.MatchingLabels{placementv1beta1.CRPTrackingLabel: crpName})).Should(Succeed(), "Failed to list the resourcesnapshot") + Expect(len(crsList.Items) == 1).Should(BeTrue()) + return crsList.Items[0].Name != oldCRS + }, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to remove the old resourcensnapshot") + }) + + It("update work to trigger a work generator reconcile", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx].ClusterName + namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster) + workName := fmt.Sprintf(placementv1beta1.FirstWorkNameFmt, crpName) + work := placementv1beta1.Work{} + Expect(hubClient.Get(ctx, types.NamespacedName{Name: workName, Namespace: namespaceName}, &work)).Should(Succeed(), "Failed to get the work") + if work.Status.ManifestConditions != nil { + work.Status.ManifestConditions = nil + } else { + meta.SetStatusCondition(&work.Status.Conditions, metav1.Condition{ + Type: placementv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: "WorkNotAvailable", + }) + } + Expect(hubClient.Status().Update(ctx, &work)).Should(Succeed(), "Failed to update the work") + } + }) + + It("change the image name in deployment, to make it available again", func() { + Eventually(func() error { + err := hubClient.Get(ctx, types.NamespacedName{Name: testDeployment.Name, Namespace: testDeployment.Namespace}, &testDeployment) + if err != nil { + return err + } + testDeployment.Spec.Template.Spec.Containers[0].Image = "1.26.2" + return hubClient.Update(ctx, &testDeployment) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to change the image name in deployment") + }) + + It("should place the resources on all member clusters", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + workResourcesPlacedActual := waitForDeploymentPlacementToReady(memberCluster, &testDeployment) + Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters) + }) + }) + Context("Test a CRP place workload objects successfully, don't block rollout based on job availability", Ordered, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) workNamespace := appNamespace() @@ -821,9 +978,12 @@ func waitForDeploymentPlacementToReady(memberCluster *framework.Cluster, testDep } } if placedDeployment.Status.ObservedGeneration == placedDeployment.Generation && depCond != nil && depCond.Status == corev1.ConditionTrue { + if placedDeployment.Spec.Template.Spec.Containers[0].Image != testDeployment.Spec.Template.Spec.Containers[0].Image { + return fmt.Errorf("deployment spec`%s` is not updated, placedDeployment = %+v, testDeployment = %+v", testDeployment.Name, placedDeployment.Spec, testDeployment.Spec) + } return nil } - return nil + return fmt.Errorf("deployment `%s` is not updated", testDeployment.Name) } } @@ -841,6 +1001,9 @@ func waitForDaemonSetPlacementToReady(memberCluster *framework.Cluster, testDaem if placedDaemonSet.Status.ObservedGeneration == placedDaemonSet.Generation && placedDaemonSet.Status.NumberAvailable == placedDaemonSet.Status.DesiredNumberScheduled && placedDaemonSet.Status.CurrentNumberScheduled == placedDaemonSet.Status.UpdatedNumberScheduled { + if placedDaemonSet.Spec.Template.Spec.Containers[0].Image != testDaemonSet.Spec.Template.Spec.Containers[0].Image { + return fmt.Errorf("daemonSet spec`%s` is not updated", testDaemonSet.Name) + } return nil } return errors.New("daemonset is not ready") @@ -861,6 +1024,9 @@ func waitForStatefulSetPlacementToReady(memberCluster *framework.Cluster, testSt if placedStatefulSet.Status.ObservedGeneration == placedStatefulSet.Generation && placedStatefulSet.Status.CurrentReplicas == *placedStatefulSet.Spec.Replicas && placedStatefulSet.Status.CurrentReplicas == placedStatefulSet.Status.UpdatedReplicas { + if placedStatefulSet.Spec.Template.Spec.Containers[0].Image != testStatefulSet.Spec.Template.Spec.Containers[0].Image { + return fmt.Errorf("statefulSet spec`%s` is not updated", placedStatefulSet.Name) + } return nil } return errors.New("statefulset is not ready") diff --git a/test/scheduler/actuals_test.go b/test/scheduler/actuals_test.go index d9831f994..de7e1a6f4 100644 --- a/test/scheduler/actuals_test.go +++ b/test/scheduler/actuals_test.go @@ -109,6 +109,7 @@ func scheduledBindingsCreatedOrUpdatedForClustersActual(clusters []string, score Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateScheduled, @@ -169,6 +170,7 @@ func boundBindingsCreatedOrUpdatedForClustersActual(clusters []string, scoreByCl Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateBound, @@ -229,6 +231,7 @@ func unscheduledBindingsCreatedOrUpdatedForClustersActual(clusters []string, sco Labels: map[string]string{ placementv1beta1.CRPTrackingLabel: crpName, }, + Finalizers: []string{placementv1beta1.SchedulerCRBCleanupFinalizer}, }, Spec: placementv1beta1.ResourceBindingSpec{ State: placementv1beta1.BindingStateUnscheduled, diff --git a/test/scheduler/suite_test.go b/test/scheduler/suite_test.go index d1168e11c..1b2b20102 100644 --- a/test/scheduler/suite_test.go +++ b/test/scheduler/suite_test.go @@ -38,6 +38,7 @@ import ( "go.goms.io/fleet/pkg/scheduler" "go.goms.io/fleet/pkg/scheduler/clustereligibilitychecker" "go.goms.io/fleet/pkg/scheduler/queue" + "go.goms.io/fleet/pkg/scheduler/watchers/clusterresourcebinding" "go.goms.io/fleet/pkg/scheduler/watchers/clusterresourceplacement" "go.goms.io/fleet/pkg/scheduler/watchers/clusterschedulingpolicysnapshot" "go.goms.io/fleet/pkg/scheduler/watchers/membercluster" @@ -584,6 +585,13 @@ func beforeSuiteForProcess1() []byte { err = memberClusterWatcher.SetupWithManager(ctrlMgr) Expect(err).NotTo(HaveOccurred(), "Failed to set up member cluster watcher with controller manager") + clusterResourceBindingWatcher := clusterresourcebinding.Reconciler{ + Client: hubClient, + SchedulerWorkQueue: schedulerWorkQueue, + } + err = clusterResourceBindingWatcher.SetupWithManager(ctrlMgr) + Expect(err).NotTo(HaveOccurred(), "Failed to set up cluster resource binding watcher with controller manager") + // Set up the scheduler. fw := buildSchedulerFramework(ctrlMgr, clusterEligibilityChecker) sched := scheduler.NewScheduler(defaultSchedulerName, fw, schedulerWorkQueue, ctrlMgr, 3)