Update upgrade-test-harness to work with dedicated CRD manifest

hack/upgrade-test-harness/.gitignore

@@ -1 +1,2 @@
 testdata/upcoming/install.yaml
+testdata/upcoming/crds.yaml

hack/upgrade-test-harness/conf.yaml

@@ -35,3 +35,11 @@ testParams:
     operatorVersion: 1.5.0
     stackVersion: 7.12.0
     crdVersion: v1
+  - name: v160
+    operatorVersion: 1.6.0
+    stackVersion: 7.13.0
+    crdVersion: v1
+  - name: upcoming
+    operatorVersion: 1.7.0
+    stackVersion: 7.13.0
+    crdVersion: v1

hack/upgrade-test-harness/fixture/eck.go

@@ -66,9 +66,11 @@ func (tp TestParam) GVR(kind string) schema.GroupVersionResource {
 
 // TestInstallOperator is the fixture for installing an operator.
 func TestInstallOperator(param TestParam) *Fixture {
+	crdPath := param.Path("crds.yaml")
 	return &Fixture{
 		Name: param.Suffixed("TestInstallOperator"),
 		Steps: []*TestStep{
+			noRetry(param.Suffixed("InstallCRDs"), ifExists(crdPath, replaceManifests(crdPath))),
 			noRetry(param.Suffixed("InstallOperator"), applyManifests(param.Path("install.yaml"))),
 			pause(5 * time.Second),
 			retryRetriable("CheckOperatorIsReady", checkOperatorIsReady),

hack/upgrade-test-harness/fixture/fixture.go

@@ -7,6 +7,7 @@ package fixture
 import (
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/elastic/cloud-on-k8s/hack/upgrade-test-harness/k8s"
@@ -154,6 +155,17 @@ func applyManifests(path string) func(*TestContext) error {
 	}
 }
 
+func replaceManifests(path string) func(ctx *TestContext) error {
+	return func(ctx *TestContext) error {
+		manifests, err := ctx.LoadResources(path)
+		if err != nil {
+			return err
+		}
+		ctx.AddCleanupFunc(deleteResources(manifests))
+		return ctx.Replace(manifests)
+	}
+}
+
 // deleteManifests is a convenience test function that deletes the provided manifests to the cluster.
 func deleteManifests(path string) func(*TestContext) error {
 	return func(ctx *TestContext) error {
@@ -166,6 +178,16 @@ func deleteManifests(path string) func(*TestContext) error {
 	}
 }
 
+// ifExists tests if the given path does exist in the filesystem
+func ifExists(path string, fn func(ctx *TestContext) error) func(ctx *TestContext) error {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return func(_ *TestContext) error {
+			return nil
+		}
+	}
+	return fn
+}
+
 // CleanupFunc is a function for cleaning up resources after the test run.
 type CleanupFunc func(*TestContext) error
 

hack/upgrade-test-harness/k8s/kubectl.go

@@ -198,6 +198,25 @@ func (h *Kubectl) DynamicClient() (dynamic.Interface, error) {
 	return h.factory.DynamicClient()
 }
 
+func (h *Kubectl) Replace(resources resource.Visitor) error {
+	return resources.Visit(func(info *resource.Info, err error) error {
+		if err != nil {
+			return err
+		}
+		if err := util.CreateOrUpdateAnnotation(false, info.Object, scheme.DefaultJSONEncoder()); err != nil {
+			return cmdutil.AddSourceToErr("replacing", info.Source, err)
+		}
+		obj, err := resource.NewHelper(info.Client, info.Mapping).
+			WithFieldManager("kubectl-replace").
+			Replace(info.Namespace, info.Name, true, info.Object)
+		if err != nil {
+			return cmdutil.AddSourceToErr("replacing", info.Source, err)
+		}
+		info.Refresh(obj, true)
+		return nil
+	})
+}
+
 // Delete the given set of resources from the cluster.
 func (h *Kubectl) Delete(resources *resource.Result, timeout time.Duration) error {
 	resources = resources.IgnoreErrors(errors.IsNotFound)

hack/upgrade-test-harness/main.go

@@ -20,15 +20,16 @@ import (
 )
 
 type configOpts struct {
-	confFile            string
-	fromRelease         string
-	logLevel            string
-	retryCount          uint
-	retryDelay          time.Duration
-	retryTimeout        time.Duration
-	skipCleanup         bool
-	toRelease           string
-	upcomingReleaseYAML string
+	confFile                string
+	fromRelease             string
+	logLevel                string
+	retryCount              uint
+	retryDelay              time.Duration
+	retryTimeout            time.Duration
+	skipCleanup             bool
+	toRelease               string
+	upcomingReleaseOperator string
+	upcomingReleaseCRDs     string
 }
 
 var (
@@ -55,7 +56,8 @@ func main() {
 	cmd.Flags().DurationVar(&opts.retryTimeout, "retry-timeout", 300*time.Second, "Time limit for retries")
 	cmd.Flags().BoolVar(&opts.skipCleanup, "skip-cleanup", false, "Skip cleaning up after test run")
 	cmd.Flags().StringVar(&opts.toRelease, "to-release", "upcoming", "Release to finish with (alpha, beta, v101, v112, upcoming)")
-	cmd.Flags().StringVar(&opts.upcomingReleaseYAML, "upcoming-release-yaml", "../../config/all-in-one.yaml", "YAML file for installing the upcoming release")
+	cmd.Flags().StringVar(&opts.upcomingReleaseCRDs, "upcoming-release-crds", "../../config/crds.yaml", "YAML file for installing the CRDs for the upcoming release")
+	cmd.Flags().StringVar(&opts.upcomingReleaseOperator, "upcoming-release-operator", "../../config/operator.yaml", "YAML file for installing the upcoming release")
 
 	kubeConfFlags.AddFlags(cmd.Flags())
 
@@ -84,7 +86,10 @@ func doRun(_ *cobra.Command, _ []string) error {
 
 	// setup upcoming release if necessary
 	if conf.TestParams[to].Name == "upcoming" {
-		if err := setupUpcomingRelease(opts.upcomingReleaseYAML); err != nil {
+		if err := setupUpcomingRelease(opts.upcomingReleaseCRDs, "crds"); err != nil {
+			return fmt.Errorf("failed to setup upcoming release: %w", err)
+		}
+		if err := setupUpcomingRelease(opts.upcomingReleaseOperator, "install"); err != nil {
 			return fmt.Errorf("failed to setup upcoming release: %w", err)
 		}
 	}
@@ -142,15 +147,15 @@ func mustGetReleasePos(conf *config.File, name string) int {
 	return pos
 }
 
-func setupUpcomingRelease(installYAML string) error {
+func setupUpcomingRelease(installYAML, targetYAML string) error {
 	in, err := os.Open(installYAML)
 	if err != nil {
 		return fmt.Errorf("failed to open %s for reading: %w", installYAML, err)
 	}
 
 	defer in.Close()
 
-	outFile := "testdata/upcoming/install.yaml"
+	outFile := fmt.Sprintf("testdata/upcoming/%s.yaml", targetYAML)
 
 	out, err := os.OpenFile(outFile, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0755)
 	if err != nil {

hack/upgrade-test-harness/testdata/upcoming/stack.yaml

@@ -4,7 +4,7 @@ kind: Elasticsearch
 metadata:
   name: es
 spec:
-  version: 7.9.2
+  version: 7.13.0
   nodeSets:
   - name: default
     count: 3
@@ -16,7 +16,7 @@ kind: Kibana
 metadata:
   name: kb
 spec:
-  version: 7.9.2
+  version: 7.13.0
   count: 1
   elasticsearchRef:
     name: es
@@ -26,7 +26,7 @@ kind: ApmServer
 metadata:
   name: apm
 spec:
-  version: 7.9.2
+  version: 7.13.0
   count: 1
   elasticsearchRef:
     name: es