Merge pull request kubernetes-sigs#10442 from cahillsf/replace-exec

🌱  Replace exec kubectl with client CreateOrUpdate

scripts/ci-e2e.sh

@@ -26,8 +26,6 @@ source "${REPO_ROOT}/scripts/ci-e2e-lib.sh"
 
 # shellcheck source=./hack/ensure-go.sh
 source "${REPO_ROOT}/hack/ensure-go.sh"
-# shellcheck source=./hack/ensure-kubectl.sh
-source "${REPO_ROOT}/hack/ensure-kubectl.sh"
 # shellcheck source=./hack/ensure-kind.sh
 source "${REPO_ROOT}/hack/ensure-kind.sh"
 

test/e2e/clusterclass_rollout.go

@@ -418,7 +418,7 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust
 				clusterv1.TemplateClonedFromNameAnnotation:      clusterClass.Spec.ControlPlane.MachineInfrastructure.Ref.Name,
 			},
 			clusterClassObjects.ControlPlaneInfrastructureMachineTemplate.GetAnnotations(),
-		).without(g, corev1.LastAppliedConfigAnnotation),
+		),
 	))
 
 	// ControlPlane InfrastructureMachineTemplate.spec.template.metadata
@@ -590,7 +590,7 @@ func assertMachineDeployments(g Gomega, clusterClassObjects clusterClassObjects,
 					clusterv1.TemplateClonedFromNameAnnotation:      mdClass.Template.Infrastructure.Ref.Name,
 				},
 				ccInfrastructureMachineTemplate.GetAnnotations(),
-			).without(g, corev1.LastAppliedConfigAnnotation),
+			),
 		))
 		// MachineDeployment InfrastructureMachineTemplate.spec.template.metadata
 		g.Expect(infrastructureMachineTemplateTemplateMetadata.Labels).To(BeEquivalentTo(
@@ -622,7 +622,7 @@ func assertMachineDeployments(g Gomega, clusterClassObjects clusterClassObjects,
 					clusterv1.TemplateClonedFromNameAnnotation:      mdClass.Template.Bootstrap.Ref.Name,
 				},
 				ccBootstrapConfigTemplate.GetAnnotations(),
-			).without(g, corev1.LastAppliedConfigAnnotation),
+			),
 		))
 		// MachineDeployment BootstrapConfigTemplate.spec.template.metadata
 		g.Expect(bootstrapConfigTemplateTemplateMetadata.Labels).To(BeEquivalentTo(

test/e2e/clusterctl_upgrade.go

@@ -448,7 +448,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 		Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template")
 
 		log.Logf("Applying the cluster template yaml to the cluster")
-		Expect(managementClusterProxy.Apply(ctx, workloadClusterTemplate)).To(Succeed())
+		Expect(managementClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate)).To(Succeed())
 
 		if input.PreWaitForCluster != nil {
 			By("Running PreWaitForCluster steps against the management cluster")

test/e2e/kcp_adoption.go

@@ -117,7 +117,12 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu
 		if input.InfrastructureProvider != nil {
 			infrastructureProvider = *input.InfrastructureProvider
 		}
-
+		must := func(r *labels.Requirement, err error) labels.Requirement {
+			if err != nil {
+				panic(err)
+			}
+			return *r
+		}
 		workloadClusterTemplate := clusterctl.ConfigCluster(ctx, clusterctl.ConfigClusterInput{
 			// pass reference to the management cluster hosting this test
 			KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
@@ -138,7 +143,8 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu
 		Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template")
 
 		By("Applying the cluster template yaml to the cluster with the 'initial' selector")
-		Expect(input.BootstrapClusterProxy.Apply(ctx, workloadClusterTemplate, "--selector", "kcp-adoption.step1")).ShouldNot(HaveOccurred())
+		selector := labels.NewSelector().Add(must(labels.NewRequirement("kcp-adoption.step1", selection.Exists, nil)))
+		Expect(input.BootstrapClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate, framework.WithLabelSelector(selector))).ShouldNot(HaveOccurred())
 
 		cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{
 			Getter:    client,
@@ -159,7 +165,8 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu
 		}, WaitForControlPlaneIntervals...)
 
 		By("Applying the cluster template yaml to the cluster with the 'kcp' selector")
-		Expect(input.BootstrapClusterProxy.Apply(ctx, workloadClusterTemplate, "--selector", "kcp-adoption.step2")).ShouldNot(HaveOccurred())
+		selector = labels.NewSelector().Add(must(labels.NewRequirement("kcp-adoption.step2", selection.Exists, nil)))
+		Expect(input.BootstrapClusterProxy.CreateOrUpdate(ctx, workloadClusterTemplate, framework.WithLabelSelector(selector))).ShouldNot(HaveOccurred())
 
 		var controlPlane *controlplanev1.KubeadmControlPlane
 		Eventually(func() *controlplanev1.KubeadmControlPlane {
@@ -177,12 +184,6 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu
 		})
 
 		By("Taking stable ownership of the Machines")
-		must := func(r *labels.Requirement, err error) labels.Requirement {
-			if err != nil {
-				panic(err)
-			}
-			return *r
-		}
 		machines := clusterv1.MachineList{}
 		Expect(client.List(ctx, &machines,
 			ctrlclient.InNamespace(namespace.Name),

test/e2e/kcp_remediations.go

@@ -489,7 +489,7 @@ func createWorkloadClusterAndWait(ctx context.Context, input createWorkloadClust
 	Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template")
 
 	Eventually(func() error {
-		return input.Proxy.Apply(ctx, workloadClusterTemplate)
+		return input.Proxy.CreateOrUpdate(ctx, workloadClusterTemplate)
 	}, 10*time.Second).Should(Succeed(), "Failed to apply the cluster template")
 
 	log.Logf("Waiting for the cluster infrastructure to be provisioned")

test/e2e/scale.go

@@ -290,7 +290,7 @@ func scaleSpec(ctx context.Context, inputGetter func() scaleSpecInput) {
 				clusterClassYAML := bytes.Replace(baseClusterClassYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespace.Name), -1)
 				log.Logf("Apply ClusterClass")
 				Eventually(func() error {
-					return input.BootstrapClusterProxy.Apply(ctx, clusterClassYAML)
+					return input.BootstrapClusterProxy.CreateOrUpdate(ctx, clusterClassYAML)
 				}, 1*time.Minute).Should(Succeed())
 			} else {
 				log.Logf("ClusterClass already exists. Skipping creation.")
@@ -551,7 +551,7 @@ func getClusterCreateAndWaitFn(input clusterctl.ApplyCustomClusterTemplateAndWai
 			WaitForClusterIntervals:      input.WaitForClusterIntervals,
 			WaitForControlPlaneIntervals: input.WaitForControlPlaneIntervals,
 			WaitForMachineDeployments:    input.WaitForMachineDeployments,
-			Args:                         input.Args,
+			CreateOrUpdateOpts:           input.CreateOrUpdateOpts,
 			PreWaitForCluster:            input.PreWaitForCluster,
 			PostMachinesProvisioned:      input.PostMachinesProvisioned,
 			ControlPlaneWaiters:          input.ControlPlaneWaiters,
@@ -563,7 +563,7 @@ func getClusterCreateFn(clusterProxy framework.ClusterProxy) clusterCreator {
 	return func(ctx context.Context, namespace, clusterName string, clusterTemplateYAML []byte) {
 		log.Logf("Applying the cluster template yaml of cluster %s", klog.KRef(namespace, clusterName))
 		Eventually(func() error {
-			return clusterProxy.Apply(ctx, clusterTemplateYAML)
+			return clusterProxy.CreateOrUpdate(ctx, clusterTemplateYAML)
 		}, 1*time.Minute).Should(Succeed(), "Failed to apply the cluster template of cluster %s", klog.KRef(namespace, clusterName))
 	}
 }
@@ -616,7 +616,7 @@ func createClusterWorker(ctx context.Context, clusterProxy framework.ClusterProx
 					log.Logf("Apply ClusterClass in namespace %", namespaceName)
 					clusterClassYAML := bytes.Replace(baseClusterClassYAML, []byte(scaleClusterNamespacePlaceholder), []byte(namespaceName), -1)
 					Eventually(func() error {
-						return clusterProxy.Apply(ctx, clusterClassYAML)
+						return clusterProxy.CreateOrUpdate(ctx, clusterClassYAML)
 					}, 1*time.Minute).Should(Succeed())
 				}
 

test/framework/autoscaler_helpers.go

@@ -107,7 +107,7 @@ func ApplyAutoscalerToWorkloadCluster(ctx context.Context, input ApplyAutoscaler
 		},
 	})
 	Expect(err).ToNot(HaveOccurred(), "failed to parse %s", workloadYamlTemplate)
-	Expect(input.WorkloadClusterProxy.Apply(ctx, workloadYaml)).To(Succeed(), "failed to apply %s", workloadYamlTemplate)
+	Expect(input.WorkloadClusterProxy.CreateOrUpdate(ctx, workloadYaml)).To(Succeed(), "failed to apply %s", workloadYamlTemplate)
 
 	By("Wait for the autoscaler deployment and collect logs")
 	deployment := &appsv1.Deployment{

test/framework/cluster_proxy.go

@@ -22,17 +22,20 @@ import (
 	"fmt"
 	"net/url"
 	"os"
-	"os/exec"
 	"path"
 	goruntime "runtime"
 	"sync"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	pkgerrors "github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -44,9 +47,9 @@ import (
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
-	testexec "sigs.k8s.io/cluster-api/test/framework/exec"
 	"sigs.k8s.io/cluster-api/test/framework/internal/log"
 	"sigs.k8s.io/cluster-api/test/infrastructure/container"
+	"sigs.k8s.io/cluster-api/util/yaml"
 )
 
 const (
@@ -89,8 +92,8 @@ type ClusterProxy interface {
 	// GetLogCollector returns the machine log collector for the Kubernetes cluster.
 	GetLogCollector() ClusterLogCollector
 
-	// Apply to apply YAML to the Kubernetes cluster, `kubectl apply`.
-	Apply(ctx context.Context, resources []byte, args ...string) error
+	// CreateOrUpdate creates or updates objects using the clusterProxy client
+	CreateOrUpdate(ctx context.Context, resources []byte, options ...CreateOrUpdateOption) error
 
 	// GetWorkloadCluster returns a proxy to a workload cluster defined in the Kubernetes cluster.
 	GetWorkloadCluster(ctx context.Context, namespace, name string, options ...Option) ClusterProxy
@@ -103,6 +106,21 @@ type ClusterProxy interface {
 	Dispose(context.Context)
 }
 
+// createOrUpdateConfig contains options for use with CreateOrUpdate.
+type createOrUpdateConfig struct {
+	labelSelector labels.Selector
+}
+
+// CreateOrUpdateOption is a configuration option supplied to CreateOrUpdate.
+type CreateOrUpdateOption func(*createOrUpdateConfig)
+
+// WithLabelSelector allows definition of the LabelSelector to be used in CreateOrUpdate.
+func WithLabelSelector(labelSelector labels.Selector) CreateOrUpdateOption {
+	return func(c *createOrUpdateConfig) {
+		c.labelSelector = labelSelector
+	}
+}
+
 // ClusterLogCollector defines an object that can collect logs from a machine.
 type ClusterLogCollector interface {
 	// CollectMachineLog collects log from a machine.
@@ -247,20 +265,52 @@ func (p *clusterProxy) GetCache(ctx context.Context) cache.Cache {
 	return p.cache
 }
 
-// Apply wraps `kubectl apply ...` and prints the output so we can see what gets applied to the cluster.
-func (p *clusterProxy) Apply(ctx context.Context, resources []byte, args ...string) error {
-	Expect(ctx).NotTo(BeNil(), "ctx is required for Apply")
-	Expect(resources).NotTo(BeNil(), "resources is required for Apply")
-
-	if err := testexec.KubectlApply(ctx, p.kubeconfigPath, resources, args...); err != nil {
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
-			return pkgerrors.New(fmt.Sprintf("%s: stderr: %s", err.Error(), exitErr.Stderr))
-		}
+// CreateOrUpdate creates or updates objects using the clusterProxy client.
+func (p *clusterProxy) CreateOrUpdate(ctx context.Context, resources []byte, opts ...CreateOrUpdateOption) error {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for CreateOrUpdate")
+	Expect(resources).NotTo(BeNil(), "resources is required for CreateOrUpdate")
+	labelSelector := labels.Everything()
+	config := &createOrUpdateConfig{}
+	for _, opt := range opts {
+		opt(config)
+	}
+	if config.labelSelector != nil {
+		labelSelector = config.labelSelector
+	}
+	objs, err := yaml.ToUnstructured(resources)
+	if err != nil {
 		return err
 	}
 
-	return nil
+	existingObject := &unstructured.Unstructured{}
+	var retErrs []error
+	for _, o := range objs {
+		objectKey := types.NamespacedName{
+			Name:      o.GetName(),
+			Namespace: o.GetNamespace(),
+		}
+		existingObject.SetAPIVersion(o.GetAPIVersion())
+		existingObject.SetKind(o.GetKind())
+		labels := labels.Set(o.GetLabels())
+		if labelSelector.Matches(labels) {
+			if err := p.GetClient().Get(ctx, objectKey, existingObject); err != nil {
+				// Expected error -- if the object does not exist, create it
+				if apierrors.IsNotFound(err) {
+					if err := p.GetClient().Create(ctx, &o); err != nil {
+						retErrs = append(retErrs, err)
+					}
+				} else {
+					retErrs = append(retErrs, err)
+				}
+			} else {
+				o.SetResourceVersion(existingObject.GetResourceVersion())
+				if err := p.GetClient().Update(ctx, &o); err != nil {
+					retErrs = append(retErrs, err)
+				}
+			}
+		}
+	}
+	return kerrors.NewAggregate(retErrs)
 }
 
 func (p *clusterProxy) GetRESTConfig() *rest.Config {

test/framework/clusterctl/clusterctl_helpers.go

@@ -246,7 +246,7 @@ type ApplyClusterTemplateAndWaitInput struct {
 	WaitForControlPlaneIntervals []interface{}
 	WaitForMachineDeployments    []interface{}
 	WaitForMachinePools          []interface{}
-	Args                         []string // extra args to be used during `kubectl apply`
+	CreateOrUpdateOpts           []framework.CreateOrUpdateOption // options to be passed to CreateOrUpdate function config
 	PreWaitForCluster            func()
 	PostMachinesProvisioned      func()
 	ControlPlaneWaiters
@@ -358,7 +358,7 @@ func ApplyClusterTemplateAndWait(ctx context.Context, input ApplyClusterTemplate
 		WaitForControlPlaneIntervals: input.WaitForControlPlaneIntervals,
 		WaitForMachineDeployments:    input.WaitForMachineDeployments,
 		WaitForMachinePools:          input.WaitForMachinePools,
-		Args:                         input.Args,
+		CreateOrUpdateOpts:           input.CreateOrUpdateOpts,
 		PreWaitForCluster:            input.PreWaitForCluster,
 		PostMachinesProvisioned:      input.PostMachinesProvisioned,
 		ControlPlaneWaiters:          input.ControlPlaneWaiters,
@@ -377,7 +377,7 @@ type ApplyCustomClusterTemplateAndWaitInput struct {
 	WaitForControlPlaneIntervals []interface{}
 	WaitForMachineDeployments    []interface{}
 	WaitForMachinePools          []interface{}
-	Args                         []string // extra args to be used during `kubectl apply`
+	CreateOrUpdateOpts           []framework.CreateOrUpdateOption // options to be passed to CreateOrUpdate function config
 	PreWaitForCluster            func()
 	PostMachinesProvisioned      func()
 	ControlPlaneWaiters
@@ -412,7 +412,7 @@ func ApplyCustomClusterTemplateAndWait(ctx context.Context, input ApplyCustomClu
 
 	log.Logf("Applying the cluster template yaml of cluster %s", klog.KRef(input.Namespace, input.ClusterName))
 	Eventually(func() error {
-		return input.ClusterProxy.Apply(ctx, input.CustomTemplateYAML, input.Args...)
+		return input.ClusterProxy.CreateOrUpdate(ctx, input.CustomTemplateYAML, input.CreateOrUpdateOpts...)
 	}, 1*time.Minute).Should(Succeed(), "Failed to apply the cluster template")
 
 	// Once we applied the cluster template we can run PreWaitForCluster.
@@ -448,7 +448,7 @@ func ApplyCustomClusterTemplateAndWait(ctx context.Context, input ApplyCustomClu
 		cniYaml, err := os.ReadFile(input.CNIManifestPath)
 		Expect(err).ShouldNot(HaveOccurred())
 
-		Expect(workloadCluster.Apply(ctx, cniYaml)).ShouldNot(HaveOccurred())
+		Expect(workloadCluster.CreateOrUpdate(ctx, cniYaml)).ShouldNot(HaveOccurred())
 	}
 
 	log.Logf("Waiting for control plane of cluster %s to be ready", klog.KRef(input.Namespace, input.ClusterName))