diff --git a/test/e2e/tests/eg_upgrade.go b/test/e2e/tests/eg_upgrade.go index 385952bd2a5..4ca4db15637 100644 --- a/test/e2e/tests/eg_upgrade.go +++ b/test/e2e/tests/eg_upgrade.go @@ -12,14 +12,17 @@ import ( "context" "fmt" "os" + "strings" "testing" "time" + "github.com/stretchr/testify/require" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/kube" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -50,11 +53,11 @@ var EGUpgradeTest = suite.ConformanceTest{ chartPath := "../../../charts/gateway-helm" relName := "eg" depNS := "envoy-gateway-system" - lastVersionTag := os.Getenv("last_version_tag") + lastVersionTag := os.Getenv("LAST_VERSION_TAG") if lastVersionTag == "" { - // Use v1.0.2 instead of v1.1.2 due to https://github.com/envoyproxy/gateway/issues/4336 - lastVersionTag = "v1.0.2" // Default version tag if not specified + lastVersionTag = "v1.1.2" // Default version tag if not specified } + t.Logf("Upgrading from version: %s", lastVersionTag) // Uninstall the current version of EG relNamespace := "envoy-gateway-system" @@ -105,18 +108,25 @@ var EGUpgradeTest = suite.ConformanceTest{ suite.Applier.GatewayClass = suite.GatewayClassName suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.BaseManifests, suite.Cleanup) + // verify latestVersion is working + kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) + + // let's make sure the gateway is up and running + ns := "gateway-upgrade-infra" + gwNN := types.NamespacedName{Name: "ha-gateway", Namespace: ns} + _, err = kubernetes.WaitForGatewayAddress(t, suite.Client, suite.TimeoutConfig, kubernetes.GatewayRef{ + NamespacedName: gwNN, + }) + require.NoErrorf(t, err, "timed out waiting for Gateway address to be assigned") + + // Apply the test manifests for _, manifestLocation := range []string{"testdata/eg-upgrade.yaml"} { tlog.Logf(t, "Applying %s", manifestLocation) suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true) } // wait for everything to startup - kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) - - // verify latestVersion is working - ns := "gateway-upgrade-infra" routeNN := types.NamespacedName{Name: "http-backend-eg-upgrade", Namespace: ns} - gwNN := types.NamespacedName{Name: "ha-gateway", Namespace: ns} gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS}) @@ -266,47 +276,53 @@ func updateChartCRDs(actionConfig *action.Configuration, gatewayChart *chart.Cha return err } -// TODO: proper migration framework required -func migrateChartCRDs(actionConfig *action.Configuration, gatewayChart *chart.Chart, timeout time.Duration) error { +func migrateChartCRDs(actionConfig *action.Configuration, gatewayChart *chart.Chart, _ time.Duration) error { crds, err := extractCRDs(actionConfig, gatewayChart) if err != nil { return err } + // https: //gateway-api.sigs.k8s.io/guides/?h=upgrade#v12-upgrade-notes + storedVersionsMap := map[string]string{ + "referencegrants.gateway.networking.k8s.io": "v1beta1", + "grpcroutes.gateway.networking.k8s.io": "v1", + } + + restCfg, err := actionConfig.RESTClientGetter.ToRESTConfig() + if err != nil { + return err + } + + cli, err := client.New(restCfg, client.Options{}) + if err != nil { + return err + } + for _, crd := range crds { - if crd.Name == "backendtlspolicies.gateway.networking.k8s.io" || - crd.Name == "grpcroutes.gateway.networking.k8s.io" { - newVersion, err := getGWAPIVersion(crd.Object) + storedVersion, ok := storedVersionsMap[crd.Name] + if !ok { + continue + } + + newVersion, err := getGWAPIVersion(crd.Object) + if err != nil { + return err + } + + if strings.HasPrefix(newVersion, "v1.2.0") { + existingCRD := &apiextensionsv1.CustomResourceDefinition{} + err := cli.Get(context.Background(), types.NamespacedName{Name: crd.Name}, existingCRD) + if kerrors.IsNotFound(err) { + continue + } if err != nil { - return err + return fmt.Errorf("failed to get CRD: %s", err.Error()) } - // https://gateway-api.sigs.k8s.io/guides/?h=upgrade#v11-upgrade-notes - if newVersion == "v1.2.0-rc2" { - helper := resource.NewHelper(crd.Client, crd.Mapping) - existingCRD, err := helper.Get(crd.Namespace, crd.Name) - if kerrors.IsNotFound(err) { - continue - } - // previous version exists - existingVersion, err := getGWAPIVersion(existingCRD) - if err != nil { - return err - } + existingCRD.Status.StoredVersions = []string{storedVersion} - if existingVersion == "v1.0.0" { - // Delete the existing instance of the BTLS and GRPCRoute CRDs - _, errs := actionConfig.KubeClient.Delete([]*resource.Info{crd}) - if errs != nil { - return fmt.Errorf("failed to delete backendtlspolicies: %s", util.MultipleErrors("", errs)) - } - - if kubeClient, ok := actionConfig.KubeClient.(kube.InterfaceExt); ok { - if err := kubeClient.WaitForDelete([]*resource.Info{crd}, timeout); err != nil { - return fmt.Errorf("failed to wait for backendtlspolicies deletion: %s", err.Error()) - } - } - } + if err := cli.Status().Patch(context.Background(), existingCRD, client.MergeFrom(existingCRD)); err != nil { + return fmt.Errorf("failed to patch CRD: %s", err.Error()) } } } @@ -354,7 +370,7 @@ func getGWAPIVersion(object runtime.Object) (string, error) { if ok { return newVersion, nil } - return "", fmt.Errorf("failed to determine Gateway API CRD version") + return "", fmt.Errorf("failed to determine Gateway API CRD version: %v", annotations) } // extractCRDs Extract the CRDs part of the chart diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 2344a3b868d..c659e234787 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -158,7 +158,7 @@ ifeq ($(E2E_RUN_TEST),) go test $(E2E_TEST_ARGS) ./test/e2e --gateway-class=envoy-gateway --debug=true --cleanup-base-resources=false go test $(E2E_TEST_ARGS) ./test/e2e/merge_gateways --gateway-class=merge-gateways --debug=true --cleanup-base-resources=false go test $(E2E_TEST_ARGS) ./test/e2e/multiple_gc --debug=true --cleanup-base-resources=true - go test $(E2E_TEST_ARGS) ./test/e2e/upgrade --gateway-class=upgrade --debug=true --cleanup-base-resources=$(E2E_CLEANUP) + LAST_VERSION_TAG=$(shell cat VERSION) go test $(E2E_TEST_ARGS) ./test/e2e/upgrade --gateway-class=upgrade --debug=true --cleanup-base-resources=$(E2E_CLEANUP) else go test $(E2E_TEST_ARGS) ./test/e2e --gateway-class=envoy-gateway --debug=true --cleanup-base-resources=$(E2E_CLEANUP) \ --run-test $(E2E_RUN_TEST)