From 5598417dfa1482bacf4fe85a8c5d2279b7f5b098 Mon Sep 17 00:00:00 2001 From: Nathan Fudenberg Date: Mon, 2 Dec 2024 15:44:18 -0500 Subject: [PATCH] Feat/large validation (#10417) Co-authored-by: Ryan Old Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: changelog-bot --- Makefile | 6 +- .../v1.18.0-rc3/validate-large-configs.yaml | 9 + .../projects/gloo/api/v1/settings.proto.sk.md | 2 +- projects/envoyinit/pkg/runner/run.go | 12 +- projects/gloo/api/v1/settings.proto | 4 + projects/gloo/pkg/api/v1/settings.pb.go | 4 + test/kube2e/README.md | 7 +- test/kube2e/gateway/gateway_suite_test.go | 4 +- test/kube2e/gloo/gloo_suite_test.go | 4 +- .../validation/full_envoy_validation/suite.go | 20 + .../valid-resources/large-configuration.yaml | 1008 +++++++++++++++++ .../e2e/features/validation/types.go | 3 + .../validation_reject_invalid/suite.go | 4 +- .../manifests/full-envoy-validation-helm.yaml | 4 +- 14 files changed, 1080 insertions(+), 11 deletions(-) create mode 100644 changelog/v1.18.0-rc3/validate-large-configs.yaml create mode 100644 test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml diff --git a/Makefile b/Makefile index c890561040f..416e5ed06fb 100644 --- a/Makefile +++ b/Makefile @@ -287,10 +287,9 @@ run-hashicorp-e2e-tests: GINKGO_FLAGS += --label-filter="end-to-end && !performa run-hashicorp-e2e-tests: test .PHONY: run-kube-e2e-tests -run-kube-e2e-tests: TEST_PKG = ./test/kube2e/$(KUBE2E_TESTS) ## Run the Kubernetes E2E Tests in the {KUBE2E_TESTS} package +run-kube-e2e-tests: TEST_PKG = ./test/kube2e/$(KUBE2E_TESTS) ## Run the legacy Kubernetes E2E Tests in the {KUBE2E_TESTS} package run-kube-e2e-tests: test - #---------------------------------------------------------------------------------- # Go Tests #---------------------------------------------------------------------------------- @@ -1080,6 +1079,9 @@ endif # distroless images CLUSTER_NAME ?= kind INSTALL_NAMESPACE ?= gloo-system +kind-setup: + VERSION=${VERSION} CLUSTER_NAME=${CLUSTER_NAME} ./ci/kind/setup-kind.sh + kind-load-%-distroless: kind load docker-image $(IMAGE_REGISTRY)/$*:$(VERSION)-distroless --name $(CLUSTER_NAME) diff --git a/changelog/v1.18.0-rc3/validate-large-configs.yaml b/changelog/v1.18.0-rc3/validate-large-configs.yaml new file mode 100644 index 00000000000..928d194900f --- /dev/null +++ b/changelog/v1.18.0-rc3/validate-large-configs.yaml @@ -0,0 +1,9 @@ +changelog: + - type: FIX + issueLink: https://github.com/solo-io/solo-projects/issues/7089 + resolvesIssue: false + description: >- + Fix the validation of large configurations when using envoy validation. + This was rarely seen in practice but occurred more often with the new fullEnvoyConfig validation. + Previously if the configuration grew too large translation would be blocked. + \ No newline at end of file diff --git a/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md b/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md index d82918ad4de..2bb1511e4c3 100644 --- a/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md +++ b/docs/content/reference/api/github.com/solo-io/gloo/projects/gloo/api/v1/settings.proto.sk.md @@ -948,7 +948,7 @@ options for configuring admission control / validation | `validationServerGrpcMaxSizeBytes` | [.google.protobuf.Int32Value](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/int-32-value) | By default, gRPC validation messages between gateway and gloo pods have a max message size of 100 MB. Setting this value sets the gRPC max message size in bytes for the gloo validation server. This should only be changed if necessary. If not included, the gRPC max message size will be the default of 100 MB. | | `serverEnabled` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | By providing the validation field (parent of this object) the user is implicitly opting into validation. This field allows the user to opt out of the validation server, while still configuring pre-existing fields such as `warn_route_short_circuiting` and `disable_transformation_validation`. If not included, the validation server will be enabled. | | `warnMissingTlsSecret` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | Allows configuring validation to report a missing TLS secret referenced by a SslConfig or UpstreamSslConfig as a warning instead of an error. This will allow for eventually consistent workloads, but will also permit the accidental deletion of secrets being referenced, which would cause disruption in traffic. | -| `fullEnvoyValidation` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | Configures the Gloo translation loop to send the final product of translation through Envoy validation mode. This has an negative impact on the total translation throughput, but it helps ensure the configuration will not be nacked when served to Envoy. This feature is disabled by default and is not recommended for production deployments unless the performance implications are well understood and acceptable. | +| `fullEnvoyValidation` | [.google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/bool-value) | Configures the Gloo translation loop to send the final product of translation through Envoy validation mode. This has an negative impact on the total translation throughput, but it helps ensure the configuration will not be nacked when served to Envoy. This feature is disabled by default and is not recommended for production deployments unless the performance implications are well understood and acceptable. Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout. When enabling this feature, consider increasing the timeout for the validating webhook (`.Values.gateway.validation.webhook.timeoutSeconds`). | diff --git a/projects/envoyinit/pkg/runner/run.go b/projects/envoyinit/pkg/runner/run.go index 1fe96f24383..5726efa90d8 100644 --- a/projects/envoyinit/pkg/runner/run.go +++ b/projects/envoyinit/pkg/runner/run.go @@ -6,6 +6,7 @@ import ( "log" "os" "syscall" + "time" "github.com/rotisserie/eris" "github.com/solo-io/gloo/pkg/utils/cmdutils" @@ -30,8 +31,15 @@ const ( func RunEnvoyValidate(ctx context.Context, envoyExecutable, bootstrapConfig string) error { logger := contextutils.LoggerFrom(ctx) - validateCmd := cmdutils.Command(ctx, envoyExecutable, "--mode", "validate", "--config-yaml", bootstrapConfig, "-l", "critical", "--log-format", "%v") - if err := validateCmd.Run(); err != nil { + validateCmd := cmdutils.Command(ctx, envoyExecutable, "--mode", "validate", "--config-path", "/dev/fd/0", + "-l", "critical", "--log-format", "%v") + validateCmd = validateCmd.WithStdin(bytes.NewBufferString(bootstrapConfig)) + + start := time.Now() + err := validateCmd.Run() + logger.Debugf("envoy validation of %d size completed in %s", len(bootstrapConfig), time.Since(start)) + + if err != nil { if os.IsNotExist(err) { // log a warning and return nil; will allow users to continue to run Gloo locally without // relying on the Gloo container with Envoy already published to the expected directory diff --git a/projects/gloo/api/v1/settings.proto b/projects/gloo/api/v1/settings.proto index 945b35cebe3..519bb7217f9 100644 --- a/projects/gloo/api/v1/settings.proto +++ b/projects/gloo/api/v1/settings.proto @@ -920,6 +920,10 @@ message GatewayOptions { // // This feature is disabled by default and is not recommended for production deployments unless // the performance implications are well understood and acceptable. + // + // Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout. + // When enabling this feature, consider increasing the timeout for the validating webhook + // (`.Values.gateway.validation.webhook.timeoutSeconds`). google.protobuf.BoolValue full_envoy_validation = 14; } diff --git a/projects/gloo/pkg/api/v1/settings.pb.go b/projects/gloo/pkg/api/v1/settings.pb.go index b6e64cfadad..9b787e80b59 100644 --- a/projects/gloo/pkg/api/v1/settings.pb.go +++ b/projects/gloo/pkg/api/v1/settings.pb.go @@ -3445,6 +3445,10 @@ type GatewayOptions_ValidationOptions struct { // // This feature is disabled by default and is not recommended for production deployments unless // the performance implications are well understood and acceptable. + // + // Large configurations can take more than 10 seconds to validate, causing the validating webhook to timeout. + // When enabling this feature, consider increasing the timeout for the validating webhook + // (`.Values.gateway.validation.webhook.timeoutSeconds`). FullEnvoyValidation *wrapperspb.BoolValue `protobuf:"bytes,14,opt,name=full_envoy_validation,json=fullEnvoyValidation,proto3" json:"full_envoy_validation,omitempty"` } diff --git a/test/kube2e/README.md b/test/kube2e/README.md index bacb04c8b16..65e4dbf13d2 100644 --- a/test/kube2e/README.md +++ b/test/kube2e/README.md @@ -2,6 +2,10 @@ > This directory houses legacy tests. All new tests should instead be added to the `test/kubernetes/e2e` directory. # Kubernetes End-to-End tests + +> These are our legacy Kubernetes E2E tests. We are migrating them to `../kubernetes/e2e`. Create new E2E tests there +> using the new framework. + See the [developer kube-e2e testing guide](/devel/testing/kube-e2e-tests.md) for more information about the philosophy of these tests. *Note: All commands should be run from the root directory of the Gloo repository* @@ -68,7 +72,7 @@ To run the regression tests, your kubeconfig file must point to a running Kubern Use the same command that CI relies on: ```bash -KUBE2E_TESTS= make run-kube-e2e-tests +CLUSTER_NAME=solo-test-cluster KUBE2E_TESTS= make run-kube-e2e-tests ``` #### Test Environment Variables @@ -81,6 +85,7 @@ The below table contains the environment variables that can be used to configure | WAIT_ON_FAIL | 0 | Set to 1 to prevent Ginkgo from cleaning up the Gloo Edge installation in case of failure. Useful to exec into inspect resources created by the test. A command to resume the test run (and thus clean up resources) will be logged to the output. | | TEAR_DOWN | false | Set to true to uninstall Gloo after the test suite completes | | RELEASED_VERSION | '' | Used by nightlies to tests a specific released version. 'LATEST' will find the latest release | +| CLUSTER_NAME | kind | Used to control which Kind cluster to run the tests inside | #### Common Test Errors `getting Helm chart version: expected a single entry with name [gloo], found: 5`\ diff --git a/test/kube2e/gateway/gateway_suite_test.go b/test/kube2e/gateway/gateway_suite_test.go index 660e8f7812e..6290830237f 100644 --- a/test/kube2e/gateway/gateway_suite_test.go +++ b/test/kube2e/gateway/gateway_suite_test.go @@ -23,6 +23,7 @@ import ( "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kube2e" "github.com/solo-io/gloo/test/kube2e/helper" + testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" skhelpers "github.com/solo-io/solo-kit/test/helpers" . "github.com/onsi/ginkgo/v2" @@ -77,7 +78,8 @@ func StartTestHelper() { } // We rely on the "new" kubernetes/e2e setup code, since it incorporates controller-runtime logging setup - clusterContext := cluster.MustKindContext("kind") + runtimeContext := testruntime.NewContext() + clusterContext := cluster.MustKindContext(runtimeContext.ClusterName) resourceClientset, err = kube2e.NewKubeResourceClientSet(ctx, clusterContext.RestConfig) Expect(err).NotTo(HaveOccurred(), "can create kube resource client set") diff --git a/test/kube2e/gloo/gloo_suite_test.go b/test/kube2e/gloo/gloo_suite_test.go index 069f39cf51d..91941a5ea6a 100644 --- a/test/kube2e/gloo/gloo_suite_test.go +++ b/test/kube2e/gloo/gloo_suite_test.go @@ -23,6 +23,7 @@ import ( "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kube2e" "github.com/solo-io/gloo/test/kube2e/helper" + testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" glootestutils "github.com/solo-io/gloo/test/testutils" "github.com/solo-io/go-utils/testutils" @@ -76,7 +77,8 @@ var _ = BeforeSuite(func() { } // We rely on the "new" kubernetes/e2e setup code, since it incorporates controller-runtime logging setup - clusterContext := cluster.MustKindContext("kind") + runtimeContext := testruntime.NewContext() + clusterContext := cluster.MustKindContext(runtimeContext.ClusterName) resourceClientset, err = kube2e.NewKubeResourceClientSet(ctx, clusterContext.RestConfig) Expect(err).NotTo(HaveOccurred(), "can create kube resource client set") diff --git a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go index e37422f36c8..79e026912cc 100644 --- a/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go +++ b/test/kubernetes/e2e/features/validation/full_envoy_validation/suite.go @@ -2,6 +2,7 @@ package full_envoy_validation import ( "context" + "fmt" "github.com/solo-io/gloo/test/kubernetes/e2e" testdefaults "github.com/solo-io/gloo/test/kubernetes/e2e/defaults" @@ -77,3 +78,22 @@ func (s *testingSuite) TestRejectInvalidTransformation() { s.Assert().Contains(output, "Failed to parse response template: Failed to parse "+ "header template ':status': [inja.exception.parser_error] (at 1:92) expected statement close, got '%'") } + +// TestLargeConfiguration checks webhook accepts large configuration when fullEnvoyValidation=true +func (s *testingSuite) TestLargeConfiguration() { + s.T().Cleanup(func() { + err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, validation.LargeConfiguration, "-n", + s.testInstallation.Metadata.InstallNamespace) + s.Assertions.NoError(err, "can delete large configuration") + + err = s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, validation.ExampleUpstream) + s.Assertions.NoError(err, "can delete example upstream") + }) + + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.ExampleUpstream, "-n", s.testInstallation.Metadata.InstallNamespace) + s.Assert().NoError(err) + + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, validation.LargeConfiguration, "-n", + s.testInstallation.Metadata.InstallNamespace) + fmt.Println(err) +} diff --git a/test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml b/test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml new file mode 100644 index 00000000000..641a7cc2364 --- /dev/null +++ b/test/kubernetes/e2e/features/validation/testdata/valid-resources/large-configuration.yaml @@ -0,0 +1,1008 @@ + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: httpbin + namespace: full-envoy-validation-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin + namespace: full-envoy-validation-test +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + version: v1 + template: + metadata: + labels: + app: httpbin + version: v1 + spec: + serviceAccountName: httpbin + containers: + - name: httpbin + image: docker.io/mccutchen/go-httpbin:v2.6.0 + imagePullPolicy: IfNotPresent + command: [ go-httpbin ] + args: + - "-port" + - "8080" + - "-max-duration" + - "600s" # override default 10s + ports: + - containerPort: 8080 + # Include curl container for e2e testing, allows sending traffic mediated by the proxy sidecar + - name: curl + image: curlimages/curl:7.83.1 + resources: + requests: + cpu: "100m" + limits: + cpu: "200m" + imagePullPolicy: IfNotPresent + command: + - "tail" + - "-f" + - "/dev/null" + - name: hey + image: gcr.io/solo-public/docs/hey:0.1.4 + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: Service +metadata: + name: httpbin + namespace: full-envoy-validation-test + labels: + app: httpbin + service: httpbin +spec: + ports: + - name: http + port: 8000 + targetPort: 8080 + selector: + app: httpbin +--- +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: nginx-upstream + namespace: full-envoy-validation-test +spec: + static: + hosts: + - addr: nginx-upstream.com + port: 80 +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualHostOption +metadata: + name: jwt-validation-company + namespace: full-envoy-validation-test +spec: + options: +--- +apiVersion: gateway.solo.io/v1 +kind: RouteOption +metadata: + name: jwt-route-ip + namespace: full-envoy-validation-test +spec: + options: + autoHostRewrite: true + prefixRewrite: /get + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header("server") == "Google Frontend" %}{{ body() }}{% else %}{% if header(":status") == "401" %}{"error":"Invalid Token","errorCode":"INVALID_TOKEN","message":"Invalid Token","statusCode":403}{% else if header(":status") == "429" %}{"status":"fail","data":{"error":"QUOTA_EXCEEDED","path": "{{ request_header(":path") }}" }}{% else %}{{ body() }}{% endif %}{% endif %}' + headers: + :status: + text: '{% if header("server") == "Google Frontend" %}{{ header(":status") }}{% else %}{% if header(":status") == "401" %}403{% else if header(":status") == "429" %}450{% else %}{{ header(":status") }}{% endif %}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualHostOption +metadata: + name: cors-company + namespace: full-envoy-validation-test +spec: + options: + cors: + allowCredentials: true + allowHeaders: + - origin, + - x-requested-with, + - accept, + - content-type, + - authorization, + - x-something-api-key, + - x-something-device-type, + - x-something-platform, + - x-something-app-version, + - x-something-platform-version, + - x-something-default-language, + - x-something-country-code-override, + - x-something-user-token, + - x-something-install-id, + - x-something-profile-id, + - x-something-install-date, + - x-something-context-override, + - x-something-is-kid-profile, + - x-something-cosed, + - x-px-block-error, + - x-something-debug, + - x-something-token-key-id + allowMethods: + - GET + - PUT + - POST + - DELETE + - PATCH + allowOrigin: + - http://localhost:3000 + allowOriginRegex: + - https://*.company.com + exposeHeaders: + - '*' + maxAge: 3628800s +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualHostOption +metadata: + name: jwt-decode-company + namespace: full-envoy-validation-test +spec: + options: + stagedTransformations: + early: + requestTransforms: + - clearRouteCache: true + requestTransformation: + transformationTemplate: + advancedTemplates: true + extractors: + bearer: + header: authorization + regex: Bearer.(.*)\.(.*)\.(.*) + subgroup: 2 + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + body: + text: '{% if existsIn(context(), "video")%}{"video":{% set seriesMediaIdValue="default" + %}{% set nextEpisodeMediaIdValue="default" %}{% set page = at(context(), + "video") %}{ {% for key, value in page %} {% if at(loop, "is_first") + and key != "seriesMediaId" and key != "mediaId" and key != "nextEpisodeMediaId" + %}{% set first_key=key %}{% set first_value=value %}{% endif %}{% + if key == "mediaId" and substring(value, 0, 21) == "transmission:matchid:" + %}"mediaId":"video:mcp:unexpected-live-match",{% else %}{% if + key == "mediaId" %}"{{key}}":{% if not isNumber(value) %}"{% endif + %}{{value}}{% if not isNumber(value) %}"{% endif %},{% endif %}{% + endif %}{% if key == "seriesMediaId" %}{% set seriesMediaIdValue=value + %}{% endif %}{% if key == "nextEpisodeMediaId" %}{% set nextEpisodeMediaIdValue=value + %}{% endif %}{% if key != "seriesMediaId" and key != "nextEpisodeMediaId" + and key != "mediaId" %}"{{key}}":{% if not isNumber(value) %}"{% + endif %}{{value}}{% if not isNumber(value) %}"{% endif %},{% endif + %}{% endfor %}"{{first_key}}":{% if not isNumber(first_value) + %}"{% endif %}{{first_value}}{% if not isNumber(first_value) %}"{% + endif %}{% if seriesMediaIdValue != "default" and seriesMediaIdValue + != "" %},"seriesMediaId":{% if not isNumber(seriesMediaIdValue)%}"{% + endif %}{{seriesMediaIdValue}}{% if not isNumber(seriesMediaIdValue) + %}"{% endif %}{% endif %}{% if nextEpisodeMediaIdValue != "default" + and seriesMediaIdValue != "default" and seriesMediaIdValue != + "" %},"nextEpisodeMediaId":{% if not isNumber(nextEpisodeMediaIdValue)%}"{% + endif %}{{nextEpisodeMediaIdValue}}{% if not isNumber(nextEpisodeMediaIdValue) + %}"{% endif %}{% endif %} }} {% else %}{{ context() }}{% endif + %}' + extractors: + country: + header: x-something-country-code + regex: (AR|BO|CL|CO|CR|DO|EC|GT|HN|MX|NI|PA|PE|PR|PY|SV|US|UY|VE) + subgroup: 1 + profile: + header: profile-id + regex: .*?"id.{3}(\w*[-_\w*]*).* + subgroup: 1 + profile-extractor: + header: profile-extended + regex: .*?"id.{3}(\w*[-_\w*]*).* + subgroup: 1 + sub: + header: sub-claim + regex: ^(.*\|)?(.*)$ + subgroup: 2 + subscription-id: + header: x-something-subscription-info + regex: .*"subscriptionId".*?"([^"]+)".* + subgroup: 1 + subscription-id-user-token: + header: x-something-subscription-info-user-token + regex: .*"subscriptionId".*?"([^"]+)".* + subgroup: 1 + subscription-tier: + header: x-something-subscription-info-user-token + regex: .*"subscriptionTier.{3}(\w*).* + subgroup: 1 + headers: + x-something-country-blocked: + text: '{% if extraction("country") == "" %}country_not_allowed{% + endif %}' + x-something-install-id: + text: '{% if request_header("x-something-install-id") != "" %}{{ request_header("x-something-install-id") + }}{% else %}{% if header("x-something-install-id-claim") != "" %}{{ + header("x-something-install-id-claim") }}{% else %}asdfasdf{% + endif %}{% endif %}' + x-something-plan-group: + text: '{% if extraction("subscription-id") == "" %}default{% else + %}{%if extraction("subscription-id") in ["something-sv-web-prepaid-7d", + "something-gt-web-prepaid-7d", "something-gt-web-prepaid-15d", "something-gt-web-prepaid-30d", + "something-hd-web-prepaid-7d", "something-hd-web-prepaid-15d", "something-hd-web-prepaid-30d", + "something-ng-web-prepaid-7d", "something-ng-web-prepaid-15d","something-ng-web-prepaid-30d", + "something-pn-web-prepaid-7d", "something-hn-web-prepaid-standalone-7d"] + %}something-restricted{% else %}default{% endif %}{% endif%}' + x-something-plan-ids: + text: '{% if extraction("subscription-id") == "" %}{{ extraction("subscription-id-user-token") + }}{% else %}{{ extraction("subscription-id") }}{% endif %}' + x-something-profile-id: + text: '{% if request_header("x-something-profile-id") != "" %}{% set + a = false %}{% for num in range(length(header("x-something-available-profile-ids")) + / 36 ) %}{% if substring(header("x-something-available-profile-ids"), + num * 36 + num , 36) == request_header("x-something-profile-id") %}{{ + request_header("x-something-profile-id") }}{% set a = true %}{% endif + %}{% endfor %}{% if a == false %}{{ request_header("x-something-profile-id") + }}{% endif %}{% else %} {% if header("x-iss") == "identity-api.self.something.com"%} {{ + extraction("profile") }}{% else %}{% if extraction("profile-extractor") + != "" %}{{ extraction("profile-extractor") }}{% else %}{{ extraction("user-token-profile-id") + }}{% endif%}{% endif %}{% endif %}' + x-something-subscription-plan-tier: + text: '{{ extraction("subscription-tier") }}' + x-something-user-id: + text: '{{ extraction("sub") }}' + headersToRemove: + - authorization + - x-something-subscription-info + - x-something-install-id-claim + - profile-extended + - x-something-subscription-info-user-token +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-1 + namespace: full-envoy-validation-test +spec: + virtualHost: + domains: + - httpbin-1.example.io + optionsConfigRefs: + delegateOptions: + - name: cors-company + namespace: full-envoy-validation-test + - name: jwt-validation-company + namespace: full-envoy-validation-test + - name: jwt-decode-company + namespace: full-envoy-validation-test + routes: + - directResponseAction: + body: '{"status":"fail","data":{"error":"COUNTRY_BLOCKED"}' + status: 451 + matchers: + - headers: + - name: x-something-country-blocked + value: country_not_allowed + prefix: / + - matchers: + - exact: /gql/v2/healthcheck + options: + autoHostRewrite: true + prefixRewrite: /healthcheck + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-user-token + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /ip + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /get + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /headers + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /status + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /user-agent + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /cookies + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /base64 + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - invertMatch: true + name: x-something-api-key + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_TOKEN"}}{% + else %}{{ body() }}{% endif%}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + namespace: full-envoy-validation-test + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: ^$ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + namespace: full-envoy-validation-test + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: .+ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_API_KEY"}{% + else %}{{ context() }}{% endif %}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + headers: + x-something-profile-id: + text: web-app-ssr + x-something-user-id: + text: web-app-ssr + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + namespace: full-envoy-validation-test +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-2 +spec: + virtualHost: + domains: + - httpbin-2.example.io + optionsConfigRefs: + delegateOptions: + - name: cors-company + namespace: full-envoy-validation-test + - name: jwt-validation-company + namespace: full-envoy-validation-test + - name: jwt-decode-company + namespace: full-envoy-validation-test + routes: + - directResponseAction: + body: '{"status":"fail","data":{"error":"COUNTRY_BLOCKED"}' + status: 451 + matchers: + - headers: + - name: x-something-country-blocked + value: country_not_allowed + prefix: / + - matchers: + - exact: /gql/v2/healthcheck + options: + autoHostRewrite: true + prefixRewrite: /healthcheck + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-user-token + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /ip + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /get + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /headers + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /status + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /user-agent + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /cookies + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /base64 + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - invertMatch: true + name: x-something-api-key + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_TOKEN"}}{% + else %}{{ body() }}{% endif%}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: ^$ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: .+ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_API_KEY"}{% + else %}{{ context() }}{% endif %}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + headers: + x-something-profile-id: + text: web-app-ssr + x-something-user-id: + text: web-app-ssr + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream +--- +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-3 +spec: + virtualHost: + domains: + - httpbin-3.example.io + optionsConfigRefs: + delegateOptions: + - name: cors-company + namespace: full-envoy-validation-test + - name: jwt-validation-company + namespace: full-envoy-validation-test + - name: jwt-decode-company + namespace: full-envoy-validation-test + routes: + - directResponseAction: + body: '{"status":"fail","data":{"error":"COUNTRY_BLOCKED"}' + status: 451 + matchers: + - headers: + - name: x-something-country-blocked + value: country_not_allowed + prefix: / + - matchers: + - exact: /gql/v2/healthcheck + options: + autoHostRewrite: true + prefixRewrite: /healthcheck + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-user-token + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Google Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /ip + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /get + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /headers + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /status + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /user-agent + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /cookies + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - prefix: /base64 + name: jwt-options-route + optionsConfigRefs: + delegateOptions: + - name: jwt-route-ip + namespace: full-envoy-validation-test + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - invertMatch: true + name: x-something-api-key + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_TOKEN"}}{% + else %}{{ body() }}{% endif%}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: ^$ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + logRequestResponseInfo: true + transformationTemplate: + body: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}{{ body() }}{% else if header(":status") + != "401" %}{{ body() }}{% else %}{"status":"fail","data":{"error":"INVALID_TOKEN"}{% + endif %}' + headers: + :status: + text: '{% if header(":status") == "401" and header("server") + == "Frontend" %}401{% else if header(":status") != + "401" %}{{ header(":status") }}{% else %}403{% endif %}' + ignoreErrorOnParse: true + inheritTransformation: true + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream + - matchers: + - headers: + - name: x-something-api-key + regex: true + value: .+ + prefix: /gql/v2 + options: + autoHostRewrite: true + prefixRewrite: / + stagedTransformations: + early: + responseTransforms: + - responseTransformation: + transformationTemplate: + body: + text: '{% if header(":status") == "401" %}{"status":"fail","data":{"error":"INVALID_API_KEY"}{% + else %}{{ context() }}{% endif %}' + headers: + :status: + text: '{% if header(":status") == "401" %}403{% else %}{{ + header(":status") }}{% endif %}' + regular: + requestTransforms: + - clearRouteCache: true + requestTransformation: + logRequestResponseInfo: true + transformationTemplate: + advancedTemplates: true + headers: + x-something-profile-id: + text: web-app-ssr + x-something-user-id: + text: web-app-ssr + timeout: 31s + routeAction: + single: + upstream: + name: nginx-upstream \ No newline at end of file diff --git a/test/kubernetes/e2e/features/validation/types.go b/test/kubernetes/e2e/features/validation/types.go index 8a6138aecac..87a4d6e2c76 100644 --- a/test/kubernetes/e2e/features/validation/types.go +++ b/test/kubernetes/e2e/features/validation/types.go @@ -46,6 +46,9 @@ var ( VSTransformationHeaderText = filepath.Join(util.MustGetThisDir(), "testdata", "transformation", "vs-transform-header-text.yaml") VSTransformationSingleReplace = filepath.Join(util.MustGetThisDir(), "testdata", "transformation", "vs-transform-single-replace.yaml") + // Valid resources + LargeConfiguration = filepath.Join(util.MustGetThisDir(), "testdata", "valid-resources", "large-configuration.yaml") + // Split webhook validation BasicUpstream = filepath.Join(util.MustGetThisDir(), "testdata", "split-webhook", "basic-upstream.yaml") diff --git a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go index 39f41541563..e48bab98c8c 100644 --- a/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go +++ b/test/kubernetes/e2e/features/validation/validation_reject_invalid/suite.go @@ -228,13 +228,13 @@ func (s *testingSuite) TestRejectTransformation() { // this should be rejected output, err = s.testInstallation.Actions.Kubectl().ApplyFileWithOutput(s.ctx, validation.VSTransformationExtractors, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().Error(err) - s.Assert().Contains(output, "envoy validation mode output: error initializing configuration '': Failed to parse response template: group 1 requested for regex with only 0 sub groups") + s.Assert().Contains(output, "Failed to parse response template: group 1 requested for regex with only 0 sub groups") // Single replace mode -- rejects invalid subgroup in transformation // note that the regex has no subgroups, but we are trying to extract the first subgroup // this should be rejected output, err = s.testInstallation.Actions.Kubectl().ApplyFileWithOutput(s.ctx, validation.VSTransformationSingleReplace, "-n", s.testInstallation.Metadata.InstallNamespace) s.Assert().Error(err) - s.Assert().Contains(output, "envoy validation mode output: error initializing configuration '': Failed to parse response template: group 1 requested for regex with only 0 sub groups") + s.Assert().Contains(output, "Failed to parse response template: group 1 requested for regex with only 0 sub groups") } diff --git a/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml b/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml index 2c13d97feb5..41bbdbc3f03 100644 --- a/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/full-envoy-validation-helm.yaml @@ -1,7 +1,9 @@ gateway: validation: failurePolicy: Fail # For "strict" validation mode, fail the validation if webhook server is not available - allowWarnings: false # For "strict" validation mode, webhook will also reject warnings + allowWarnings: false # transformation validation is disabled because full envoy validation is enabled. disableTransformationValidation: true + webhook: + timeoutSeconds: 30 # We are seeing Envoy take 10s of seconds to validate some of the larger configurations fullEnvoyValidation: true