From 844cca45978c13b6456e526ee10563107aa82d29 Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 24 Oct 2023 19:34:22 +0800 Subject: [PATCH 1/9] validation: init CEL validation Signed-off-by: zirain --- .github/workflows/build_and_test.yaml | 39 ++++---- api/v1alpha1/envoyproxy_types.go | 1 + .../validation/envoyproxy_validate.go | 2 +- .../gateway.envoyproxy.io_envoyproxies.yaml | 3 + test/validation/envoyproxy_test.go | 89 +++++++++++++++++++ test/validation/main_test.go | 43 +++++++++ tools/make/kube.mk | 9 ++ 7 files changed, 163 insertions(+), 23 deletions(-) create mode 100644 test/validation/envoyproxy_test.go create mode 100644 test/validation/main_test.go diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index c7265d52d15..198dbf046bc 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -81,17 +81,6 @@ jobs: - uses: actions/checkout@v4 - uses: ./tools/github-actions/setup-deps - - name: Download EG Binaries - uses: actions/download-artifact@v3 - with: - name: envoy-gateway - path: bin/ - - - name: Give Privileges To EG Binaries - run: | - chmod +x bin/linux/amd64/envoy-gateway - chmod +x bin/linux/arm64/envoy-gateway - # conformance - name: Run Standard Conformance Tests env: @@ -99,6 +88,23 @@ jobs: IMAGE_PULL_POLICY: IfNotPresent run: make conformance + crd-validation: + runs-on: ubuntu-latest + needs: [ build ] + strategy: + matrix: + version: [ v1.26.6, v1.27.3, v1.28.0 ] + steps: + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + # CRD Validation Tests + - name: Run Validation Tests + env: + KIND_NODE_TAG: ${{ matrix.version }} + IMAGE_PULL_POLICY: IfNotPresent + run: make validation + e2e-test: runs-on: ubuntu-latest needs: [build] @@ -109,17 +115,6 @@ jobs: - uses: actions/checkout@v4 - uses: ./tools/github-actions/setup-deps - - name: Download EG Binaries - uses: actions/download-artifact@v3 - with: - name: envoy-gateway - path: bin/ - - - name: Give Privileges To EG Binaries - run: | - chmod +x bin/linux/amd64/envoy-gateway - chmod +x bin/linux/arm64/envoy-gateway - # E2E - name: Run E2E Tests env: diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 76da4babce1..95ffb23a035 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -97,6 +97,7 @@ type EnvoyProxyProvider struct { // optional auxiliary control planes. Supported types are "Kubernetes". // // +unionDiscriminator + // +kubebuilder:validation:XValidation:message="type must be one of ['Kubernetes']",rule="self == 'Kubernetes' ? true : false" Type ProviderType `json:"type"` // Kubernetes defines the desired state of the Kubernetes resource provider. // Kubernetes provides infrastructure resources for running the data plane, diff --git a/api/v1alpha1/validation/envoyproxy_validate.go b/api/v1alpha1/validation/envoyproxy_validate.go index 1ba367ef0b8..1ea980990fc 100644 --- a/api/v1alpha1/validation/envoyproxy_validate.go +++ b/api/v1alpha1/validation/envoyproxy_validate.go @@ -24,7 +24,7 @@ import ( _ "github.com/envoyproxy/gateway/internal/xds/extensions" // register the generated types to support protojson unmarshalling ) -// Validate validates the provided EnvoyProxy. +// ValidateEnvoyProxy validates the provided EnvoyProxy. func ValidateEnvoyProxy(proxy *egv1a1.EnvoyProxy) error { var errs []error if proxy == nil { diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 33aed66b3a6..de4c6d9c6bc 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5291,6 +5291,9 @@ spec: enum: - Kubernetes type: string + x-kubernetes-validations: + - message: type must be one of ['Kubernetes'] + rule: 'self == ''Kubernetes'' ? true : false' required: - type type: object diff --git a/test/validation/envoyproxy_test.go b/test/validation/envoyproxy_test.go new file mode 100644 index 00000000000..4a6a0fbe6c5 --- /dev/null +++ b/test/validation/envoyproxy_test.go @@ -0,0 +1,89 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build validation +// +build validation + +package validation + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +func TestEnvoyProxyProvider(t *testing.T) { + ctx := context.Background() + baseEnvoyProxy := egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proxy", + Namespace: metav1.NamespaceDefault, + }, + Spec: egv1a1.EnvoyProxySpec{}, + } + + cases := []struct { + desc string + mutate func(envoy *egv1a1.EnvoyProxy) + mutateStatus func(envoy *egv1a1.EnvoyProxy) + wantErrors []string + }{ + { + desc: "nil provider", + mutate: func(envoy *egv1a1.EnvoyProxy) { + + }, + wantErrors: []string{}, + }, + { + desc: "unsupported provider", + mutate: func(envoy *egv1a1.EnvoyProxy) { + envoy.Spec = egv1a1.EnvoyProxySpec{ + Provider: &egv1a1.EnvoyProxyProvider{ + Type: "foo", + }, + } + }, + wantErrors: []string{"type must be one of ['Kubernetes']"}, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + proxy := baseEnvoyProxy.DeepCopy() + proxy.Name = fmt.Sprintf("proxy-%v", time.Now().UnixNano()) + + if tc.mutate != nil { + tc.mutate(proxy) + } + err := c.Create(ctx, proxy) + + if tc.mutateStatus != nil { + tc.mutateStatus(proxy) + err = c.Status().Update(ctx, proxy) + } + + if (len(tc.wantErrors) != 0) != (err != nil) { + t.Fatalf("Unexpected response while creating EnvoyProxy; got err=\n%v\n;want error=%v", err, tc.wantErrors != nil) + } + + var missingErrorStrings []string + for _, wantError := range tc.wantErrors { + if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(wantError)) { + missingErrorStrings = append(missingErrorStrings, wantError) + } + } + if len(missingErrorStrings) != 0 { + t.Errorf("Unexpected response while creating EnvoyProxy; got err=\n%v\n;missing strings within error=%q", err, missingErrorStrings) + } + }) + } +} diff --git a/test/validation/main_test.go b/test/validation/main_test.go new file mode 100644 index 00000000000..3206012160a --- /dev/null +++ b/test/validation/main_test.go @@ -0,0 +1,43 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build validation +// +build validation + +package validation + +import ( + "fmt" + "os" + "path" + "testing" + + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +var c client.Client + +func TestMain(m *testing.M) { + kc := os.Getenv("KUBECONFIG") + if kc == "" { + kc = path.Join(os.Getenv("HOME"), ".kube/config") + } + + rest, err := clientcmd.BuildConfigFromFlags("", kc) + if err != nil { + panic(fmt.Sprintf("Failed to build config from BuildConfigFromFlags: %v", err)) + } + + c, err = client.New(rest, client.Options{}) + if err != nil { + panic(fmt.Sprintf("Error initializing client: %v", err)) + } + _ = egv1a1.AddToScheme(c.Scheme()) + + os.Exit(m.Run()) +} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 0489b60bc53..5218443ad31 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -210,3 +210,12 @@ generate-egctl-releases: ## Generate egctl releases curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_darwin_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_darwin_arm64.tar.gz curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_amd64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_amd64.tar.gz curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_arm64.tar.gz + +.PHONY: validation +validation: create-cluster kube-install-image kube-deploy run-validation delete-cluster + +.PHONY: run-validation +run-validation: ## Run validation tests. + @$(LOG_TARGET) + kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + go test -v -tags validation ./test/validation From a0400506fdb02f5f734e3ebb08173d37fd52c358 Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 24 Oct 2023 19:37:17 +0800 Subject: [PATCH 2/9] simplify Signed-off-by: zirain --- api/v1alpha1/envoyproxy_types.go | 2 +- .../crds/generated/gateway.envoyproxy.io_envoyproxies.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 95ffb23a035..f5f09cdf3c9 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -97,7 +97,7 @@ type EnvoyProxyProvider struct { // optional auxiliary control planes. Supported types are "Kubernetes". // // +unionDiscriminator - // +kubebuilder:validation:XValidation:message="type must be one of ['Kubernetes']",rule="self == 'Kubernetes' ? true : false" + // +kubebuilder:validation:XValidation:message="type must be one of ['Kubernetes']",rule="self == 'Kubernetes'" Type ProviderType `json:"type"` // Kubernetes defines the desired state of the Kubernetes resource provider. // Kubernetes provides infrastructure resources for running the data plane, diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index de4c6d9c6bc..6411a3b8887 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5293,7 +5293,7 @@ spec: type: string x-kubernetes-validations: - message: type must be one of ['Kubernetes'] - rule: 'self == ''Kubernetes'' ? true : false' + rule: self == 'Kubernetes' required: - type type: object From dee99d492f8793df5aa17335998d9cb5e8517ca1 Mon Sep 17 00:00:00 2001 From: zirain Date: Tue, 24 Oct 2023 19:48:33 +0800 Subject: [PATCH 3/9] lint Signed-off-by: zirain --- .github/workflows/build_and_test.yaml | 202 +++++++++++++------------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 198dbf046bc..0558e257279 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -2,98 +2,98 @@ name: Build and Test on: push: branches: - - "main" - - "release/v*" + - "main" + - "release/v*" paths-ignore: - - "**/*.png" + - "**/*.png" pull_request: branches: - - "main" - - "release/v*" + - "main" + - "release/v*" paths-ignore: - - "**/*.png" + - "**/*.png" jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - # Generate the install manifests first so it can checked - # for errors while running `make -k lint` - - run: make generate-manifests - - run: make lint-deps - - run: make -k lint + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + # Generate the install manifests first so it can checked + # for errors while running `make -k lint` + - run: make generate-manifests + - run: make lint-deps + - run: make -k lint gen-check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - run: make -k gen-check + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + - run: make -k gen-check license-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - run: make -k licensecheck + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + - run: make -k licensecheck coverage-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - # test - - name: Run Coverage Tests - run: make go.test.coverage - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - fail_ci_if_error: true - files: ./coverage.xml - name: codecov-envoy-gateway - verbose: true + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + # test + - name: Run Coverage Tests + run: make go.test.coverage + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: true + files: ./coverage.xml + name: codecov-envoy-gateway + verbose: true build: runs-on: ubuntu-latest needs: [lint, gen-check, license-check, coverage-test] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps - - name: Build EG Multiarch Binaries - run: make build-multiarch PLATFORMS="linux_amd64 linux_arm64" + - name: Build EG Multiarch Binaries + run: make build-multiarch PLATFORMS="linux_amd64 linux_arm64" - - name: Upload EG Binaries - uses: actions/upload-artifact@v3 - with: - name: envoy-gateway - path: bin/ + - name: Upload EG Binaries + uses: actions/upload-artifact@v3 + with: + name: envoy-gateway + path: bin/ conformance-test: runs-on: ubuntu-latest needs: [build] strategy: matrix: - version: [ v1.26.6, v1.27.3, v1.28.0 ] + version: [v1.26.6, v1.27.3, v1.28.0] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps - # conformance - - name: Run Standard Conformance Tests - env: - KIND_NODE_TAG: ${{ matrix.version }} - IMAGE_PULL_POLICY: IfNotPresent - run: make conformance + # conformance + - name: Run Standard Conformance Tests + env: + KIND_NODE_TAG: ${{ matrix.version }} + IMAGE_PULL_POLICY: IfNotPresent + run: make conformance crd-validation: runs-on: ubuntu-latest - needs: [ build ] + needs: [build] strategy: matrix: - version: [ v1.26.6, v1.27.3, v1.28.0 ] + version: [v1.26.6, v1.27.3, v1.28.0] steps: - uses: actions/checkout@v4 - uses: ./tools/github-actions/setup-deps @@ -110,59 +110,59 @@ jobs: needs: [build] strategy: matrix: - version: [ v1.26.6, v1.27.3, v1.28.0 ] + version: [v1.26.6, v1.27.3, v1.28.0] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps - # E2E - - name: Run E2E Tests - env: - KIND_NODE_TAG: ${{ matrix.version }} - IMAGE_PULL_POLICY: IfNotPresent - run: make e2e + # E2E + - name: Run E2E Tests + env: + KIND_NODE_TAG: ${{ matrix.version }} + IMAGE_PULL_POLICY: IfNotPresent + run: make e2e publish: runs-on: ubuntu-latest needs: [conformance-test, e2e-test] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - - name: Download EG Binaries - uses: actions/download-artifact@v3 - with: - name: envoy-gateway - path: bin/ - - - name: Give Privileges To EG Binaries - run: | - chmod +x bin/linux/amd64/envoy-gateway - chmod +x bin/linux/arm64/envoy-gateway - - # build and push image - - name: Login to DockerHub - if: github.event_name == 'push' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Setup Multiarch Environment - if: github.event_name == 'push' - run: make image.multiarch.setup - - - name: Build and Push EG Commit Image - if: github.event_name == 'push' - # tag is set to the short SHA of the commit - run: make image.push.multiarch PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev - - - name: Build and Push EG Latest Image - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # tag is set to `latest` when pushing to main branch - run: make image.push.multiarch TAG=latest PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev - - - name: Build and Push EG Latest Helm Chart - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # use `0.0.0` as the default latest version. - run: OCI_REGISTRY=oci://docker.io/envoyproxy CHART_VERSION=v0.0.0-latest TAG=latest make helm-package helm-push + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + - name: Download EG Binaries + uses: actions/download-artifact@v3 + with: + name: envoy-gateway + path: bin/ + + - name: Give Privileges To EG Binaries + run: | + chmod +x bin/linux/amd64/envoy-gateway + chmod +x bin/linux/arm64/envoy-gateway + + # build and push image + - name: Login to DockerHub + if: github.event_name == 'push' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Setup Multiarch Environment + if: github.event_name == 'push' + run: make image.multiarch.setup + + - name: Build and Push EG Commit Image + if: github.event_name == 'push' + # tag is set to the short SHA of the commit + run: make image.push.multiarch PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev + + - name: Build and Push EG Latest Image + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # tag is set to `latest` when pushing to main branch + run: make image.push.multiarch TAG=latest PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev + + - name: Build and Push EG Latest Helm Chart + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # use `0.0.0` as the default latest version. + run: OCI_REGISTRY=oci://docker.io/envoyproxy CHART_VERSION=v0.0.0-latest TAG=latest make helm-package helm-push From 8d2b017a1ab9bc036ef61a0e7213179724703b87 Mon Sep 17 00:00:00 2001 From: zirain Date: Wed, 25 Oct 2023 06:15:58 +0800 Subject: [PATCH 4/9] remove XValidation Signed-off-by: zirain --- api/v1alpha1/envoyproxy_types.go | 1 - .../crds/generated/gateway.envoyproxy.io_envoyproxies.yaml | 3 --- 2 files changed, 4 deletions(-) diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index f5f09cdf3c9..76da4babce1 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -97,7 +97,6 @@ type EnvoyProxyProvider struct { // optional auxiliary control planes. Supported types are "Kubernetes". // // +unionDiscriminator - // +kubebuilder:validation:XValidation:message="type must be one of ['Kubernetes']",rule="self == 'Kubernetes'" Type ProviderType `json:"type"` // Kubernetes defines the desired state of the Kubernetes resource provider. // Kubernetes provides infrastructure resources for running the data plane, diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 6411a3b8887..33aed66b3a6 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -5291,9 +5291,6 @@ spec: enum: - Kubernetes type: string - x-kubernetes-validations: - - message: type must be one of ['Kubernetes'] - rule: self == 'Kubernetes' required: - type type: object From 5cf018fa2dca896ab742c4954d34dcd905efe145 Mon Sep 17 00:00:00 2001 From: zirain Date: Wed, 25 Oct 2023 06:46:56 +0800 Subject: [PATCH 5/9] reuse kube-test Signed-off-by: zirain --- .github/workflows/build_and_test.yaml | 4 +-- test/validation/envoyproxy_test.go | 2 +- test/validation/main_test.go | 43 ++++++++++++++++++++------- tools/make/kube.mk | 12 ++------ 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 0558e257279..dc108bd7497 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -88,7 +88,7 @@ jobs: IMAGE_PULL_POLICY: IfNotPresent run: make conformance - crd-validation: + kube-test: runs-on: ubuntu-latest needs: [build] strategy: @@ -103,7 +103,7 @@ jobs: env: KIND_NODE_TAG: ${{ matrix.version }} IMAGE_PULL_POLICY: IfNotPresent - run: make validation + run: make kube-test e2e-test: runs-on: ubuntu-latest diff --git a/test/validation/envoyproxy_test.go b/test/validation/envoyproxy_test.go index 4a6a0fbe6c5..0a24772e56d 100644 --- a/test/validation/envoyproxy_test.go +++ b/test/validation/envoyproxy_test.go @@ -52,7 +52,7 @@ func TestEnvoyProxyProvider(t *testing.T) { }, } }, - wantErrors: []string{"type must be one of ['Kubernetes']"}, + wantErrors: []string{"Unsupported value: \"foo\": supported values: \"Kubernetes\""}, }, } diff --git a/test/validation/main_test.go b/test/validation/main_test.go index 3206012160a..ceccb1d8db0 100644 --- a/test/validation/main_test.go +++ b/test/validation/main_test.go @@ -9,13 +9,18 @@ package validation import ( + "context" "fmt" "os" - "path" + "path/filepath" "testing" - "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" ) @@ -23,17 +28,21 @@ import ( var c client.Client func TestMain(m *testing.M) { - kc := os.Getenv("KUBECONFIG") - if kc == "" { - kc = path.Join(os.Getenv("HOME"), ".kube/config") - } - - rest, err := clientcmd.BuildConfigFromFlags("", kc) + // Setup the test environment. + testEnv, restCfg, err := startEnv() if err != nil { - panic(fmt.Sprintf("Failed to build config from BuildConfigFromFlags: %v", err)) + panic(fmt.Sprintf("Failed to start testenv: %v", err)) } - c, err = client.New(rest, client.Options{}) + _, cancel := context.WithCancel(ctrl.SetupSignalHandler()) + defer func() { + cancel() + if err := testEnv.Stop(); err != nil { + panic(fmt.Sprintf("Failed to stop testenv: %v", err)) + } + }() + + c, err = client.New(restCfg, client.Options{}) if err != nil { panic(fmt.Sprintf("Error initializing client: %v", err)) } @@ -41,3 +50,17 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } + +func startEnv() (*envtest.Environment, *rest.Config, error) { + log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true))) + egAPIs := filepath.Join("..", "..", "charts", "gateway-helm", "crds", "generated") + + env := &envtest.Environment{ + CRDDirectoryPaths: []string{egAPIs}, + } + cfg, err := env.Start() + if err != nil { + return env, nil, err + } + return env, cfg, nil +} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 5218443ad31..3db2d5a35ef 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -49,6 +49,7 @@ kube-generate: $(tools/controller-gen) ## Generate code containing DeepCopy, Dee kube-test: manifests generate $(tools/setup-envtest) ## Run Kubernetes provider tests. @$(LOG_TARGET) KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test --tags=integration ./... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test --tags=validation ./... -coverprofile cover.out ##@ Kubernetes Deployment @@ -209,13 +210,4 @@ generate-egctl-releases: ## Generate egctl releases curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_darwin_amd64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_darwin_amd64.tar.gz curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_darwin_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_darwin_arm64.tar.gz curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_amd64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_amd64.tar.gz - curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_arm64.tar.gz - -.PHONY: validation -validation: create-cluster kube-install-image kube-deploy run-validation delete-cluster - -.PHONY: run-validation -run-validation: ## Run validation tests. - @$(LOG_TARGET) - kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available - go test -v -tags validation ./test/validation + curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_arm64.tar.gzß From ee80bc8ff118d225b1f1e26748f4f33eb843150a Mon Sep 17 00:00:00 2001 From: zirain Date: Wed, 25 Oct 2023 09:01:35 +0800 Subject: [PATCH 6/9] revert nits Signed-off-by: zirain --- tools/make/kube.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 3db2d5a35ef..1acd86a6f11 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -210,4 +210,4 @@ generate-egctl-releases: ## Generate egctl releases curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_darwin_amd64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_darwin_amd64.tar.gz curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_darwin_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_darwin_arm64.tar.gz curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_amd64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_amd64.tar.gz - curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_arm64.tar.gzß + curl -sSL https://github.com/envoyproxy/gateway/releases/download/latest/egctl_latest_linux_arm64.tar.gz -o $(OUTPUT_DIR)/egctl_$(TAG)_linux_arm64.tar.gz From e6b4af1b538b09b4a3bf0856245501ea105b51eb Mon Sep 17 00:00:00 2001 From: zirain Date: Wed, 25 Oct 2023 10:14:37 +0800 Subject: [PATCH 7/9] rename to celvalidation Signed-off-by: zirain --- test/{validation => cel-validation}/envoyproxy_test.go | 2 +- test/{validation => cel-validation}/main_test.go | 6 +++--- tools/make/kube.mk | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename test/{validation => cel-validation}/envoyproxy_test.go (98%) rename test/{validation => cel-validation}/main_test.go (95%) diff --git a/test/validation/envoyproxy_test.go b/test/cel-validation/envoyproxy_test.go similarity index 98% rename from test/validation/envoyproxy_test.go rename to test/cel-validation/envoyproxy_test.go index 0a24772e56d..919a1ec9241 100644 --- a/test/validation/envoyproxy_test.go +++ b/test/cel-validation/envoyproxy_test.go @@ -6,7 +6,7 @@ //go:build validation // +build validation -package validation +package cel_validation import ( "context" diff --git a/test/validation/main_test.go b/test/cel-validation/main_test.go similarity index 95% rename from test/validation/main_test.go rename to test/cel-validation/main_test.go index ceccb1d8db0..64915388439 100644 --- a/test/validation/main_test.go +++ b/test/cel-validation/main_test.go @@ -3,10 +3,10 @@ // The full text of the Apache license is available in the LICENSE file at // the root of the repo. -//go:build validation -// +build validation +//go:build celvalidation +// +build celvalidation -package validation +package celvalidation import ( "context" diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 1acd86a6f11..a12aa103270 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -49,7 +49,7 @@ kube-generate: $(tools/controller-gen) ## Generate code containing DeepCopy, Dee kube-test: manifests generate $(tools/setup-envtest) ## Run Kubernetes provider tests. @$(LOG_TARGET) KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test --tags=integration ./... -coverprofile cover.out - KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test --tags=validation ./... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test --tags=celvalidation ./... -coverprofile cover.out ##@ Kubernetes Deployment From b87783cbebe9a30629a9ee65487d03da5fea22c5 Mon Sep 17 00:00:00 2001 From: zirain Date: Thu, 26 Oct 2023 08:16:52 +0800 Subject: [PATCH 8/9] make kube-test be part of go.test.coverage Signed-off-by: zirain --- .github/workflows/build_and_test.yaml | 41 +++++++++++++++------------ tools/make/golang.mk | 2 +- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index dc108bd7497..bc28ed0861c 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -76,11 +76,22 @@ jobs: needs: [build] strategy: matrix: - version: [v1.26.6, v1.27.3, v1.28.0] + version: [ v1.26.6, v1.27.3, v1.28.0 ] steps: - uses: actions/checkout@v4 - uses: ./tools/github-actions/setup-deps + - name: Download EG Binaries + uses: actions/download-artifact@v3 + with: + name: envoy-gateway + path: bin/ + + - name: Give Privileges To EG Binaries + run: | + chmod +x bin/linux/amd64/envoy-gateway + chmod +x bin/linux/arm64/envoy-gateway + # conformance - name: Run Standard Conformance Tests env: @@ -88,32 +99,26 @@ jobs: IMAGE_PULL_POLICY: IfNotPresent run: make conformance - kube-test: + e2e-test: runs-on: ubuntu-latest needs: [build] strategy: matrix: - version: [v1.26.6, v1.27.3, v1.28.0] + version: [ v1.26.6, v1.27.3, v1.28.0 ] steps: - uses: actions/checkout@v4 - uses: ./tools/github-actions/setup-deps - # CRD Validation Tests - - name: Run Validation Tests - env: - KIND_NODE_TAG: ${{ matrix.version }} - IMAGE_PULL_POLICY: IfNotPresent - run: make kube-test + - name: Download EG Binaries + uses: actions/download-artifact@v3 + with: + name: envoy-gateway + path: bin/ - e2e-test: - runs-on: ubuntu-latest - needs: [build] - strategy: - matrix: - version: [v1.26.6, v1.27.3, v1.28.0] - steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps + - name: Give Privileges To EG Binaries + run: | + chmod +x bin/linux/amd64/envoy-gateway + chmod +x bin/linux/arm64/envoy-gateway # E2E - name: Run E2E Tests diff --git a/tools/make/golang.mk b/tools/make/golang.mk index 4cf7d4e3bbb..b0142e7e180 100644 --- a/tools/make/golang.mk +++ b/tools/make/golang.mk @@ -52,7 +52,7 @@ go.testdata.complete: ## Override test ouputdata go test -timeout 30s github.com/envoyproxy/gateway/internal/gatewayapi --override-testdata=true .PHONY: go.test.coverage -go.test.coverage: $(tools/setup-envtest) ## Run go unit and integration tests in GitHub Actions +go.test.coverage: kube-test $(tools/setup-envtest) ## Run go unit and integration tests in GitHub Actions @$(LOG_TARGET) KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... --tags=integration -race -coverprofile=coverage.xml -covermode=atomic From a2a5fdd5108db844ac102e16f1386c7b016e984a Mon Sep 17 00:00:00 2001 From: zirain Date: Thu, 26 Oct 2023 09:29:17 +0800 Subject: [PATCH 9/9] revert changes Signed-off-by: zirain --- .github/workflows/build_and_test.yaml | 242 +++++++++++++------------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index bc28ed0861c..c7265d52d15 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -2,74 +2,74 @@ name: Build and Test on: push: branches: - - "main" - - "release/v*" + - "main" + - "release/v*" paths-ignore: - - "**/*.png" + - "**/*.png" pull_request: branches: - - "main" - - "release/v*" + - "main" + - "release/v*" paths-ignore: - - "**/*.png" + - "**/*.png" jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - # Generate the install manifests first so it can checked - # for errors while running `make -k lint` - - run: make generate-manifests - - run: make lint-deps - - run: make -k lint + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + # Generate the install manifests first so it can checked + # for errors while running `make -k lint` + - run: make generate-manifests + - run: make lint-deps + - run: make -k lint gen-check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - run: make -k gen-check + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + - run: make -k gen-check license-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - run: make -k licensecheck + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + - run: make -k licensecheck coverage-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - # test - - name: Run Coverage Tests - run: make go.test.coverage - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - fail_ci_if_error: true - files: ./coverage.xml - name: codecov-envoy-gateway - verbose: true + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + # test + - name: Run Coverage Tests + run: make go.test.coverage + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: true + files: ./coverage.xml + name: codecov-envoy-gateway + verbose: true build: runs-on: ubuntu-latest needs: [lint, gen-check, license-check, coverage-test] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps - - name: Build EG Multiarch Binaries - run: make build-multiarch PLATFORMS="linux_amd64 linux_arm64" + - name: Build EG Multiarch Binaries + run: make build-multiarch PLATFORMS="linux_amd64 linux_arm64" - - name: Upload EG Binaries - uses: actions/upload-artifact@v3 - with: - name: envoy-gateway - path: bin/ + - name: Upload EG Binaries + uses: actions/upload-artifact@v3 + with: + name: envoy-gateway + path: bin/ conformance-test: runs-on: ubuntu-latest @@ -78,26 +78,26 @@ jobs: matrix: version: [ v1.26.6, v1.27.3, v1.28.0 ] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - - name: Download EG Binaries - uses: actions/download-artifact@v3 - with: - name: envoy-gateway - path: bin/ - - - name: Give Privileges To EG Binaries - run: | - chmod +x bin/linux/amd64/envoy-gateway - chmod +x bin/linux/arm64/envoy-gateway - - # conformance - - name: Run Standard Conformance Tests - env: - KIND_NODE_TAG: ${{ matrix.version }} - IMAGE_PULL_POLICY: IfNotPresent - run: make conformance + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + - name: Download EG Binaries + uses: actions/download-artifact@v3 + with: + name: envoy-gateway + path: bin/ + + - name: Give Privileges To EG Binaries + run: | + chmod +x bin/linux/amd64/envoy-gateway + chmod +x bin/linux/arm64/envoy-gateway + + # conformance + - name: Run Standard Conformance Tests + env: + KIND_NODE_TAG: ${{ matrix.version }} + IMAGE_PULL_POLICY: IfNotPresent + run: make conformance e2e-test: runs-on: ubuntu-latest @@ -106,68 +106,68 @@ jobs: matrix: version: [ v1.26.6, v1.27.3, v1.28.0 ] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - - name: Download EG Binaries - uses: actions/download-artifact@v3 - with: - name: envoy-gateway - path: bin/ - - - name: Give Privileges To EG Binaries - run: | - chmod +x bin/linux/amd64/envoy-gateway - chmod +x bin/linux/arm64/envoy-gateway - - # E2E - - name: Run E2E Tests - env: - KIND_NODE_TAG: ${{ matrix.version }} - IMAGE_PULL_POLICY: IfNotPresent - run: make e2e + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + - name: Download EG Binaries + uses: actions/download-artifact@v3 + with: + name: envoy-gateway + path: bin/ + + - name: Give Privileges To EG Binaries + run: | + chmod +x bin/linux/amd64/envoy-gateway + chmod +x bin/linux/arm64/envoy-gateway + + # E2E + - name: Run E2E Tests + env: + KIND_NODE_TAG: ${{ matrix.version }} + IMAGE_PULL_POLICY: IfNotPresent + run: make e2e publish: runs-on: ubuntu-latest needs: [conformance-test, e2e-test] steps: - - uses: actions/checkout@v4 - - uses: ./tools/github-actions/setup-deps - - - name: Download EG Binaries - uses: actions/download-artifact@v3 - with: - name: envoy-gateway - path: bin/ - - - name: Give Privileges To EG Binaries - run: | - chmod +x bin/linux/amd64/envoy-gateway - chmod +x bin/linux/arm64/envoy-gateway - - # build and push image - - name: Login to DockerHub - if: github.event_name == 'push' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Setup Multiarch Environment - if: github.event_name == 'push' - run: make image.multiarch.setup - - - name: Build and Push EG Commit Image - if: github.event_name == 'push' - # tag is set to the short SHA of the commit - run: make image.push.multiarch PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev - - - name: Build and Push EG Latest Image - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # tag is set to `latest` when pushing to main branch - run: make image.push.multiarch TAG=latest PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev - - - name: Build and Push EG Latest Helm Chart - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # use `0.0.0` as the default latest version. - run: OCI_REGISTRY=oci://docker.io/envoyproxy CHART_VERSION=v0.0.0-latest TAG=latest make helm-package helm-push + - uses: actions/checkout@v4 + - uses: ./tools/github-actions/setup-deps + + - name: Download EG Binaries + uses: actions/download-artifact@v3 + with: + name: envoy-gateway + path: bin/ + + - name: Give Privileges To EG Binaries + run: | + chmod +x bin/linux/amd64/envoy-gateway + chmod +x bin/linux/arm64/envoy-gateway + + # build and push image + - name: Login to DockerHub + if: github.event_name == 'push' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Setup Multiarch Environment + if: github.event_name == 'push' + run: make image.multiarch.setup + + - name: Build and Push EG Commit Image + if: github.event_name == 'push' + # tag is set to the short SHA of the commit + run: make image.push.multiarch PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev + + - name: Build and Push EG Latest Image + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # tag is set to `latest` when pushing to main branch + run: make image.push.multiarch TAG=latest PLATFORMS="linux_amd64 linux_arm64" IMAGE=envoyproxy/gateway-dev + + - name: Build and Push EG Latest Helm Chart + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # use `0.0.0` as the default latest version. + run: OCI_REGISTRY=oci://docker.io/envoyproxy CHART_VERSION=v0.0.0-latest TAG=latest make helm-package helm-push