diff --git a/apis/kueue/v1alpha1/multikueueconfig_types.go b/apis/kueue/v1alpha1/multikueueconfig_types.go index db43b5390a..4d554c4f7a 100644 --- a/apis/kueue/v1alpha1/multikueueconfig_types.go +++ b/apis/kueue/v1alpha1/multikueueconfig_types.go @@ -52,10 +52,15 @@ type KubeConfig struct { type MultiKueueClusterSpec struct { // Information how to connect to the cluster. - KubeConfig KubeConfig `json:"kubeconfigRef"` + KubeConfig KubeConfig `json:"kubeConfig"` } type MultiKueueClusterStatus struct { + // +optional + // +listType=map + // +listMapKey=type + // +patchStrategy=merge + // +patchMergeKey=type Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml index 696bdd72b3..1d0ae133c5 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml @@ -46,7 +46,7 @@ spec: type: object spec: properties: - kubeconfigRef: + kubeConfig: description: Information how to connect to the cluster. properties: location: @@ -67,7 +67,7 @@ spec: - name type: object required: - - kubeconfigRef + - kubeConfig type: object status: properties: @@ -139,6 +139,9 @@ spec: - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map type: object type: object served: true diff --git a/client-go/applyconfiguration/kueue/v1alpha1/multikueueclusterspec.go b/client-go/applyconfiguration/kueue/v1alpha1/multikueueclusterspec.go index af40891fc2..e7e304f7ea 100644 --- a/client-go/applyconfiguration/kueue/v1alpha1/multikueueclusterspec.go +++ b/client-go/applyconfiguration/kueue/v1alpha1/multikueueclusterspec.go @@ -20,7 +20,7 @@ package v1alpha1 // MultiKueueClusterSpecApplyConfiguration represents an declarative configuration of the MultiKueueClusterSpec type for use // with apply. type MultiKueueClusterSpecApplyConfiguration struct { - KubeConfig *KubeConfigApplyConfiguration `json:"kubeconfigRef,omitempty"` + KubeConfig *KubeConfigApplyConfiguration `json:"kubeConfig,omitempty"` } // MultiKueueClusterSpecApplyConfiguration constructs an declarative configuration of the MultiKueueClusterSpec type for use with diff --git a/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml b/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml index d934088e28..bf911a066c 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml @@ -33,7 +33,7 @@ spec: type: object spec: properties: - kubeconfigRef: + kubeConfig: description: Information how to connect to the cluster. properties: location: @@ -54,7 +54,7 @@ spec: - name type: object required: - - kubeconfigRef + - kubeConfig type: object status: properties: @@ -126,6 +126,9 @@ spec: - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map type: object type: object served: true diff --git a/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go b/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go index fe33b5b65e..564ceb3d0d 100644 --- a/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go +++ b/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go @@ -129,6 +129,35 @@ func TestReconcile(t *testing.T) { Obj(), }, }, + "missing and inactive cluster": { + reconcileFor: "ac1", + checks: []kueue.AdmissionCheck{ + *utiltesting.MakeAdmissionCheck("ac1"). + ControllerName(ControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "config1"). + Obj(), + }, + configs: []kueuealpha.MultiKueueConfig{ + *utiltesting.MakeMultiKueueConfig("config1").Clusters("worker1", "worker2", "worker3").Obj(), + }, + + clusters: []kueuealpha.MultiKueueCluster{ + *utiltesting.MakeMultiKueueCluster("worker1").Active(metav1.ConditionFalse, "ByTest", "by test").Obj(), + *utiltesting.MakeMultiKueueCluster("worker2").Active(metav1.ConditionTrue, "ByTest", "by test").Obj(), + }, + wantChecks: []kueue.AdmissionCheck{ + *utiltesting.MakeAdmissionCheck("ac1"). + ControllerName(ControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "config1"). + Condition(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionFalse, + Reason: "Inactive", + Message: "Missing clusters: [worker3], Inactive clusters: [worker1]", + }). + Obj(), + }, + }, "active": { reconcileFor: "ac1", checks: []kueue.AdmissionCheck{ diff --git a/pkg/controller/admissionchecks/multikueue/indexer_test.go b/pkg/controller/admissionchecks/multikueue/indexer_test.go index bc9bd8c52a..14f556b9e1 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer_test.go +++ b/pkg/controller/admissionchecks/multikueue/indexer_test.go @@ -103,7 +103,7 @@ func TestListMultikueClustersUsingKubeconfig(t *testing.T) { k8sclient := builder.Build() for _, req := range tc.clusters { if err := k8sclient.Create(ctx, req); err != nil { - t.Errorf("Unable to create %s request: %v", client.ObjectKeyFromObject(req), err) + t.Fatalf("Unable to create %s cluster: %v", client.ObjectKeyFromObject(req), err) } } @@ -160,7 +160,7 @@ func TestListMultikueConfigsUsingMultikueueClusters(t *testing.T) { k8sclient := builder.Build() for _, config := range tc.configs { if err := k8sclient.Create(ctx, config); err != nil { - t.Errorf("Unable to create %s config: %v", client.ObjectKeyFromObject(config), err) + t.Fatalf("Unable to create %s config: %v", client.ObjectKeyFromObject(config), err) } } diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go index 23e25d630c..29e74154bb 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go @@ -213,7 +213,7 @@ func (c *clustersReconciler) Reconcile(ctx context.Context, req reconcile.Reques // get the kubeconfig kubeConfig, retry, err := c.getKubeConfig(ctx, &cluster.Spec.KubeConfig) if retry { - return reconcile.Result{}, nil + return reconcile.Result{}, err } if err != nil { log.Error(err, "reading kubeconfig") diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go index 71adfe8bc3..f64716b317 100644 --- a/test/e2e/multikueue/e2e_test.go +++ b/test/e2e/multikueue/e2e_test.go @@ -49,13 +49,13 @@ var _ = ginkgo.Describe("MultiKueue", func() { worker1Ns *corev1.Namespace worker2Ns *corev1.Namespace - multiKueueCluster1 *kueuealpha.MultiKueueCluster - multiKueueCluster2 *kueuealpha.MultiKueueCluster - multiKueueConfig *kueuealpha.MultiKueueConfig - multiKueueAc *kueue.AdmissionCheck - managerFlavor *kueue.ResourceFlavor - managerCq *kueue.ClusterQueue - managerLq *kueue.LocalQueue + workerCluster1 *kueuealpha.MultiKueueCluster + workerCluster2 *kueuealpha.MultiKueueCluster + multiKueueConfig *kueuealpha.MultiKueueConfig + multiKueueAc *kueue.AdmissionCheck + managerFlavor *kueue.ResourceFlavor + managerCq *kueue.ClusterQueue + managerLq *kueue.LocalQueue worker1Flavor *kueue.ResourceFlavor worker1Cq *kueue.ClusterQueue @@ -88,11 +88,11 @@ var _ = ginkgo.Describe("MultiKueue", func() { } gomega.Expect(k8sWorker2Client.Create(ctx, worker2Ns)).To(gomega.Succeed()) - multiKueueCluster1 = utiltesting.MakeMultiKueueCluster("worker1").Secret("worker1", "multikueue1").Obj() - gomega.Expect(k8sManagerClient.Create(ctx, multiKueueCluster1)).To(gomega.Succeed()) + workerCluster1 = utiltesting.MakeMultiKueueCluster("worker1").Secret("worker1", "multikueue1").Obj() + gomega.Expect(k8sManagerClient.Create(ctx, workerCluster1)).To(gomega.Succeed()) - multiKueueCluster2 = utiltesting.MakeMultiKueueCluster("worker2").Secret("worker2", "multikueue2").Obj() - gomega.Expect(k8sManagerClient.Create(ctx, multiKueueCluster2)).To(gomega.Succeed()) + workerCluster2 = utiltesting.MakeMultiKueueCluster("worker2").Secret("worker2", "multikueue2").Obj() + gomega.Expect(k8sManagerClient.Create(ctx, workerCluster2)).To(gomega.Succeed()) multiKueueConfig = utiltesting.MakeMultiKueueConfig("multikueueconfig").Clusters("worker1", "worker2").Obj() gomega.Expect(k8sManagerClient.Create(ctx, multiKueueConfig)).Should(gomega.Succeed()) @@ -177,8 +177,8 @@ var _ = ginkgo.Describe("MultiKueue", func() { util.ExpectResourceFlavorToBeDeleted(ctx, k8sManagerClient, managerFlavor, true) util.ExpectAdmissionCheckToBeDeleted(ctx, k8sManagerClient, multiKueueAc, true) gomega.Expect(k8sManagerClient.Delete(ctx, multiKueueConfig)).To(gomega.Succeed()) - gomega.Expect(k8sManagerClient.Delete(ctx, multiKueueCluster1)).To(gomega.Succeed()) - gomega.Expect(k8sManagerClient.Delete(ctx, multiKueueCluster2)).To(gomega.Succeed()) + gomega.Expect(k8sManagerClient.Delete(ctx, workerCluster1)).To(gomega.Succeed()) + gomega.Expect(k8sManagerClient.Delete(ctx, workerCluster2)).To(gomega.Succeed()) }) ginkgo.When("Creating a multikueue admission check", func() { diff --git a/test/integration/multikueue/multikueue_test.go b/test/integration/multikueue/multikueue_test.go index 38e5e61404..e4d537d1bf 100644 --- a/test/integration/multikueue/multikueue_test.go +++ b/test/integration/multikueue/multikueue_test.go @@ -48,8 +48,8 @@ var _ = ginkgo.Describe("Multikueue", func() { managerMultikueueSecret1 *corev1.Secret managerMultikueueSecret2 *corev1.Secret - managerClusterWorker1 *kueuealpha.MultiKueueCluster - managerClusterWorker2 *kueuealpha.MultiKueueCluster + workerCluster1 *kueuealpha.MultiKueueCluster + workerCluster2 *kueuealpha.MultiKueueCluster managerMultiKueueConfig *kueuealpha.MultiKueueConfig multikueueAC *kueue.AdmissionCheck managerCq *kueue.ClusterQueue @@ -67,26 +67,26 @@ var _ = ginkgo.Describe("Multikueue", func() { GenerateName: "multikueue-", }, } - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerNs)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerNs)).To(gomega.Succeed()) worker1Ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: managerNs.Name, }, } - gomega.Expect(worker1Cluster.client.Create(worker1Cluster.ctx, worker1Ns)).To(gomega.Succeed()) + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Ns)).To(gomega.Succeed()) worker2Ns = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: managerNs.Name, }, } - gomega.Expect(worker2Cluster.client.Create(worker2Cluster.ctx, worker2Ns)).To(gomega.Succeed()) + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Ns)).To(gomega.Succeed()) - w1Kubeconfig, err := worker1Cluster.kubeConfigBytes() + w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - w2Kubeconfig, err := worker2Cluster.kubeConfigBytes() + w2Kubeconfig, err := worker2TestCluster.kubeConfigBytes() gomega.Expect(err).NotTo(gomega.HaveOccurred()) managerMultikueueSecret1 = &corev1.Secret{ @@ -98,7 +98,7 @@ var _ = ginkgo.Describe("Multikueue", func() { kueuealpha.MultiKueueConfigSecretKey: w1Kubeconfig, }, } - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerMultikueueSecret1)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultikueueSecret1)).To(gomega.Succeed()) managerMultikueueSecret2 = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -109,28 +109,28 @@ var _ = ginkgo.Describe("Multikueue", func() { kueuealpha.MultiKueueConfigSecretKey: w2Kubeconfig, }, } - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerMultikueueSecret2)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultikueueSecret2)).To(gomega.Succeed()) - managerClusterWorker1 = utiltesting.MakeMultiKueueCluster("worker1").Secret("worker1", managerMultikueueSecret1.Name).Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerClusterWorker1)).To(gomega.Succeed()) + workerCluster1 = utiltesting.MakeMultiKueueCluster("worker1").Secret("worker1", managerMultikueueSecret1.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster1)).To(gomega.Succeed()) - managerClusterWorker2 = utiltesting.MakeMultiKueueCluster("worker2").Secret("worker2", managerMultikueueSecret2.Name).Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerClusterWorker2)).To(gomega.Succeed()) + workerCluster2 = utiltesting.MakeMultiKueueCluster("worker2").Secret("worker2", managerMultikueueSecret2.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster2)).To(gomega.Succeed()) - managerMultiKueueConfig = utiltesting.MakeMultiKueueConfig("multikueueconfig").Clusters(managerClusterWorker1.Name, managerClusterWorker2.Name).Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerMultiKueueConfig)).Should(gomega.Succeed()) + managerMultiKueueConfig = utiltesting.MakeMultiKueueConfig("multikueueconfig").Clusters(workerCluster1.Name, workerCluster2.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueConfig)).Should(gomega.Succeed()) multikueueAC = utiltesting.MakeAdmissionCheck("ac1"). ControllerName(multikueue.ControllerName). Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", managerMultiKueueConfig.Name). Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, multikueueAC)).Should(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multikueueAC)).Should(gomega.Succeed()) ginkgo.By("wait for check active", func() { updatetedAc := kueue.AdmissionCheck{} acKey := client.ObjectKeyFromObject(multikueueAC) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, acKey, &updatetedAc)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatetedAc)).To(gomega.Succeed()) cond := apimeta.FindStatusCondition(updatetedAc.Status.Conditions, kueue.AdmissionCheckActive) g.Expect(cond).NotTo(gomega.BeNil()) g.Expect(cond.Status).To(gomega.Equal(metav1.ConditionTrue), "Reason: %s, Message: %q", cond.Reason, cond.Status) @@ -140,42 +140,42 @@ var _ = ginkgo.Describe("Multikueue", func() { managerCq = utiltesting.MakeClusterQueue("q1"). AdmissionChecks(multikueueAC.Name). Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerCq)).Should(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerCq)).Should(gomega.Succeed()) managerLq = utiltesting.MakeLocalQueue(managerCq.Name, managerNs.Name).ClusterQueue(managerCq.Name).Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, managerLq)).Should(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerLq)).Should(gomega.Succeed()) worker1Cq = utiltesting.MakeClusterQueue("q1").Obj() - gomega.Expect(worker1Cluster.client.Create(worker1Cluster.ctx, worker1Cq)).Should(gomega.Succeed()) + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Cq)).Should(gomega.Succeed()) worker1Lq = utiltesting.MakeLocalQueue(worker1Cq.Name, worker1Ns.Name).ClusterQueue(worker1Cq.Name).Obj() - gomega.Expect(worker1Cluster.client.Create(worker1Cluster.ctx, worker1Lq)).Should(gomega.Succeed()) + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Lq)).Should(gomega.Succeed()) worker2Cq = utiltesting.MakeClusterQueue("q1").Obj() - gomega.Expect(worker2Cluster.client.Create(worker2Cluster.ctx, worker2Cq)).Should(gomega.Succeed()) + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Cq)).Should(gomega.Succeed()) worker2Lq = utiltesting.MakeLocalQueue(worker2Cq.Name, worker2Ns.Name).ClusterQueue(worker2Cq.Name).Obj() - gomega.Expect(worker2Cluster.client.Create(worker2Cluster.ctx, worker2Lq)).Should(gomega.Succeed()) + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Lq)).Should(gomega.Succeed()) }) ginkgo.AfterEach(func() { - gomega.Expect(util.DeleteNamespace(managerCluster.ctx, managerCluster.client, managerNs)).To(gomega.Succeed()) - gomega.Expect(util.DeleteNamespace(worker1Cluster.ctx, worker1Cluster.client, worker1Ns)).To(gomega.Succeed()) - gomega.Expect(util.DeleteNamespace(worker2Cluster.ctx, worker2Cluster.client, worker2Ns)).To(gomega.Succeed()) - util.ExpectClusterQueueToBeDeleted(managerCluster.ctx, managerCluster.client, managerCq, true) - util.ExpectClusterQueueToBeDeleted(worker1Cluster.ctx, worker1Cluster.client, worker1Cq, true) - util.ExpectClusterQueueToBeDeleted(worker2Cluster.ctx, worker2Cluster.client, worker2Cq, true) - util.ExpectAdmissionCheckToBeDeleted(managerCluster.ctx, managerCluster.client, multikueueAC, true) - gomega.Expect(managerCluster.client.Delete(managerCluster.ctx, managerMultiKueueConfig)).To(gomega.Succeed()) - gomega.Expect(managerCluster.client.Delete(managerCluster.ctx, managerClusterWorker1)).To(gomega.Succeed()) - gomega.Expect(managerCluster.client.Delete(managerCluster.ctx, managerClusterWorker2)).To(gomega.Succeed()) - gomega.Expect(managerCluster.client.Delete(managerCluster.ctx, managerMultikueueSecret1)).To(gomega.Succeed()) - gomega.Expect(managerCluster.client.Delete(managerCluster.ctx, managerMultikueueSecret2)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(managerTestCluster.ctx, managerTestCluster.client, managerNs)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ns)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker2TestCluster.ctx, worker2TestCluster.client, worker2Ns)).To(gomega.Succeed()) + util.ExpectClusterQueueToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerCq, true) + util.ExpectClusterQueueToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq, true) + util.ExpectClusterQueueToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Cq, true) + util.ExpectAdmissionCheckToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, multikueueAC, true) + gomega.Expect(managerTestCluster.client.Delete(managerTestCluster.ctx, managerMultiKueueConfig)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Delete(managerTestCluster.ctx, workerCluster1)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Delete(managerTestCluster.ctx, workerCluster2)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Delete(managerTestCluster.ctx, managerMultikueueSecret1)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Delete(managerTestCluster.ctx, managerMultikueueSecret2)).To(gomega.Succeed()) }) ginkgo.It("Should run a job on worker if admitted", func() { job := testingjob.MakeJob("job", managerNs.Name). Queue(managerLq.Name). Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, job)).Should(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, job)).Should(gomega.Succeed()) createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name), Namespace: managerNs.Name} @@ -183,18 +183,18 @@ var _ = ginkgo.Describe("Multikueue", func() { ginkgo.By("setting workload reservation in the management cluster", func() { admission := utiltesting.MakeAdmission(managerCq.Name).Obj() gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(util.SetQuotaReservation(managerCluster.ctx, managerCluster.client, createdWorkload, admission)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(util.SetQuotaReservation(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, admission)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("checking the workload creation in the worker clusters", func() { managerWl := &kueue.Workload{} - gomega.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(worker1Cluster.client.Get(worker1Cluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.Spec).To(gomega.BeComparableTo(managerWl.Spec)) - g.Expect(worker2Cluster.client.Get(worker2Cluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.Spec).To(gomega.BeComparableTo(managerWl.Spec)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -203,12 +203,12 @@ var _ = ginkgo.Describe("Multikueue", func() { admission := utiltesting.MakeAdmission(managerCq.Name).Obj() gomega.Eventually(func(g gomega.Gomega) { - g.Expect(worker1Cluster.client.Get(worker1Cluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(util.SetQuotaReservation(worker1Cluster.ctx, worker1Cluster.client, createdWorkload, admission)).To(gomega.Succeed()) + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(util.SetQuotaReservation(worker1TestCluster.ctx, worker1TestCluster.client, createdWorkload, admission)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) acs := workload.FindAdmissionCheck(createdWorkload.Status.AdmissionChecks, multikueueAC.Name) g.Expect(acs).NotTo(gomega.BeNil()) g.Expect(acs.State).To(gomega.Equal(kueue.CheckStatePending)) @@ -216,25 +216,25 @@ var _ = ginkgo.Describe("Multikueue", func() { }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(worker2Cluster.client.Get(worker2Cluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("finishing the worker job", func() { gomega.Eventually(func(g gomega.Gomega) { createdJob := batchv1.Job{} - g.Expect(worker1Cluster.client.Get(worker1Cluster.ctx, client.ObjectKeyFromObject(job), &createdJob)).To(gomega.Succeed()) + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, client.ObjectKeyFromObject(job), &createdJob)).To(gomega.Succeed()) createdJob.Status.Conditions = append(createdJob.Status.Conditions, batchv1.JobCondition{ Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastProbeTime: metav1.Now(), LastTransitionTime: metav1.Now(), }) - g.Expect(worker1Cluster.client.Status().Update(worker1Cluster.ctx, &createdJob)).To(gomega.Succeed()) + g.Expect(worker1TestCluster.client.Status().Update(worker1TestCluster.ctx, &createdJob)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(apimeta.FindStatusCondition(createdWorkload.Status.Conditions, kueue.WorkloadFinished)).To(gomega.BeComparableTo(&metav1.Condition{ Type: kueue.WorkloadFinished, @@ -246,7 +246,7 @@ var _ = ginkgo.Describe("Multikueue", func() { gomega.Eventually(func(g gomega.Gomega) { createdWorkload := &kueue.Workload{} - g.Expect(worker1Cluster.client.Get(worker1Cluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -269,7 +269,7 @@ var _ = ginkgo.Describe("Multikueue", func() { }, ). Obj() - gomega.Expect(managerCluster.client.Create(managerCluster.ctx, jobSet)).Should(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, jobSet)).Should(gomega.Succeed()) createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadjobset.GetWorkloadNameForJobSet(jobSet.Name), Namespace: managerNs.Name} @@ -284,30 +284,30 @@ var _ = ginkgo.Describe("Multikueue", func() { ginkgo.By("setting workload reservation in the management cluster", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(util.SetQuotaReservation(managerCluster.ctx, managerCluster.client, createdWorkload, admission)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(util.SetQuotaReservation(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, admission)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("checking the workload creation in the worker clusters", func() { managerWl := &kueue.Workload{} - gomega.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + gomega.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(worker2Cluster.client.Get(worker2Cluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.Spec).To(gomega.BeComparableTo(managerWl.Spec)) - g.Expect(worker1Cluster.client.Get(worker1Cluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.Spec).To(gomega.BeComparableTo(managerWl.Spec)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("setting workload reservation in worker2, the workload is admitted in manager amd worker1 wl is removed", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(worker2Cluster.client.Get(worker2Cluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(util.SetQuotaReservation(worker2Cluster.ctx, worker2Cluster.client, createdWorkload, admission)).To(gomega.Succeed()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(util.SetQuotaReservation(worker2TestCluster.ctx, worker2TestCluster.client, createdWorkload, admission)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) acs := workload.FindAdmissionCheck(createdWorkload.Status.AdmissionChecks, multikueueAC.Name) g.Expect(acs).NotTo(gomega.BeNil()) g.Expect(acs.State).To(gomega.Equal(kueue.CheckStateReady)) @@ -323,25 +323,25 @@ var _ = ginkgo.Describe("Multikueue", func() { }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(worker1Cluster.client.Get(worker1Cluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("finishing the worker jobSet, the manager's wl is marked as finished and the worker2 wl removed", func() { gomega.Eventually(func(g gomega.Gomega) { createdJobSet := jobset.JobSet{} - g.Expect(worker2Cluster.client.Get(worker2Cluster.ctx, client.ObjectKeyFromObject(jobSet), &createdJobSet)).To(gomega.Succeed()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, client.ObjectKeyFromObject(jobSet), &createdJobSet)).To(gomega.Succeed()) apimeta.SetStatusCondition(&createdJobSet.Status.Conditions, metav1.Condition{ Type: string(jobset.JobSetCompleted), Status: metav1.ConditionTrue, Reason: "ByTest", Message: "by test", }) - g.Expect(worker2Cluster.client.Status().Update(worker2Cluster.ctx, &createdJobSet)).To(gomega.Succeed()) + g.Expect(worker2TestCluster.client.Status().Update(worker2TestCluster.ctx, &createdJobSet)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerCluster.client.Get(managerCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(apimeta.FindStatusCondition(createdWorkload.Status.Conditions, kueue.WorkloadFinished)).To(gomega.BeComparableTo(&metav1.Condition{ Type: kueue.WorkloadFinished, @@ -353,7 +353,7 @@ var _ = ginkgo.Describe("Multikueue", func() { gomega.Eventually(func(g gomega.Gomega) { createdWorkload := &kueue.Workload{} - g.Expect(worker2Cluster.client.Get(worker2Cluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/multikueue/suite_test.go b/test/integration/multikueue/suite_test.go index c362339950..a614e3dc0a 100644 --- a/test/integration/multikueue/suite_test.go +++ b/test/integration/multikueue/suite_test.go @@ -80,9 +80,9 @@ func (c *cluster) kubeConfigBytes() ([]byte, error) { } var ( - managerCluster cluster - worker1Cluster cluster - worker2Cluster cluster + managerTestCluster cluster + worker1TestCluster cluster + worker2TestCluster cluster managersConfigNamespace *corev1.Namespace ) @@ -107,15 +107,15 @@ func createCluster(setupFnc framework.ManagerSetup) cluster { } var _ = ginkgo.BeforeSuite(func() { - managerCluster = createCluster(managerAndMultiKueueSetup) - worker1Cluster = createCluster(managerSetup) - worker2Cluster = createCluster(managerSetup) + managerTestCluster = createCluster(managerAndMultiKueueSetup) + worker1TestCluster = createCluster(managerSetup) + worker2TestCluster = createCluster(managerSetup) }) var _ = ginkgo.AfterSuite(func() { - managerCluster.fwk.Teardown() - worker1Cluster.fwk.Teardown() - worker2Cluster.fwk.Teardown() + managerTestCluster.fwk.Teardown() + worker1TestCluster.fwk.Teardown() + worker2TestCluster.fwk.Teardown() }) func managerSetup(mgr manager.Manager, ctx context.Context) {