feat: data plane & envoyproxy resilience test suite

.github/workflows/build_and_test.yaml

@@ -176,6 +176,19 @@ jobs:
 
     - name: Read Benchmark report
       run: cat test/benchmark/benchmark_report/benchmark_report.md
+  resilience-test:
+    runs-on: ubuntu-latest
+    if: ${{ ! startsWith(github.event_name, 'push') }}
+    needs: [build]
+    steps:
+    - uses: actions/[email protected]
+    - uses: ./tools/github-actions/setup-deps
+    - name: Resilience Test
+      env:
+        KIND_NODE_TAG: v1.28.13
+        IMAGE_PULL_POLICY: IfNotPresent
+        CUSTOM_CNI: "true"
+      run: make resilience
 
   publish:
     runs-on: ubuntu-latest

test/resilience/embed.go

@@ -0,0 +1,13 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build resilience
+
+package resilience
+
+import "embed"
+
+//go:embed testdata/*.yaml
+var Manifests embed.FS

test/resilience/resilience_test.go

@@ -0,0 +1,40 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build resilience
+
+package resilience
+
+import (
+	"flag"
+	"io/fs"
+	"os"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/gateway-api/conformance/utils/flags"
+	"testing"
+
+	"github.com/envoyproxy/gateway/test/resilience/suite"
+	"github.com/envoyproxy/gateway/test/resilience/tests"
+	kubetest "github.com/envoyproxy/gateway/test/utils/kubernetes"
+)
+
+func TestResilience(t *testing.T) {
+	cli, _ := kubetest.NewClient(t)
+	// Parse benchmark options.
+	flag.Parse()
+	log.SetLogger(zap.New(zap.WriteTo(os.Stderr), zap.UseDevMode(true)))
+	bSuite, err := suite.NewResilienceTestSuite(
+		cli,
+		*suite.ReportSaveDir,
+		[]fs.FS{Manifests},
+		*flags.GatewayClassName,
+	)
+	if err != nil {
+		t.Fatalf("Failed to create the resillience test suit: %v", err)
+	}
+	t.Logf("Running %d resilience tests", len(tests.ResilienceTests))
+	bSuite.Run(t, tests.ResilienceTests)
+}

test/resilience/suite/flags.go

@@ -0,0 +1,14 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build resilience
+
+package suite
+
+import "flag"
+
+var (
+	ReportSaveDir = flag.String("report-save-dir", "benchmark_report", "The dir where to save the benchmark test report.")
+)

test/resilience/suite/suite.go

@@ -0,0 +1,115 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build resilience
+
+package suite
+
+import (
+	"context"
+	"github.com/envoyproxy/gateway/test/utils/kubernetes"
+	"io/fs"
+	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
+	"testing"
+	"time"
+
+	opt "github.com/envoyproxy/gateway/internal/cmd/options"
+	kube "github.com/envoyproxy/gateway/internal/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/gateway-api/conformance/utils/config"
+)
+
+const (
+	BenchmarkTestScaledKey = "benchmark-test/scaled"
+	BenchmarkTestClientKey = "benchmark-test/client"
+	DefaultControllerName  = "gateway.envoyproxy.io/gatewayclass-controller"
+)
+
+type ResilienceTest struct {
+	ShortName   string
+	Description string
+	Test        func(*testing.T, *ResilienceTestSuite)
+}
+
+type ResilienceTestSuite struct {
+	Client         client.Client
+	TimeoutConfig  config.TimeoutConfig
+	ControllerName string
+	ReportSaveDir  string
+	KubeActions    *kubernetes.KubeActions
+	// Labels
+	scaledLabels map[string]string // indicate which resources are scaled
+
+	// Clients that for internal usage.
+	kubeClient       kube.CLIClient // required for getting logs from pod\
+	ManifestFS       []fs.FS
+	GatewayClassName string
+	RoundTripper     roundtripper.RoundTripper
+}
+
+func NewResilienceTestSuite(client client.Client, reportDir string, manifestFS []fs.FS, gcn string) (*ResilienceTestSuite, error) {
+	var (
+		timeoutConfig = config.TimeoutConfig{}
+	)
+
+	// Reset some timeout config for the benchmark test.
+	config.SetupTimeoutConfig(&timeoutConfig)
+	timeoutConfig.RouteMustHaveParents = 180 * time.Second
+	roundTripper := &roundtripper.DefaultRoundTripper{Debug: true, TimeoutConfig: timeoutConfig}
+	// Initial various client.
+	kubeClient, err := kube.NewCLIClient(opt.DefaultConfigFlags.ToRawKubeConfigLoader())
+	if err != nil {
+		return nil, err
+	}
+	KubeActions := kubernetes.NewKubeHelper(client, kubeClient)
+	return &ResilienceTestSuite{
+		Client:           client,
+		ManifestFS:       manifestFS,
+		TimeoutConfig:    timeoutConfig,
+		ControllerName:   DefaultControllerName,
+		ReportSaveDir:    reportDir,
+		GatewayClassName: gcn,
+		scaledLabels: map[string]string{
+			BenchmarkTestScaledKey: "true",
+		},
+		KubeActions:  KubeActions,
+		kubeClient:   kubeClient,
+		RoundTripper: roundTripper,
+	}, nil
+}
+
+func (rts *ResilienceTestSuite) WithResCleanUp(ctx context.Context, t *testing.T, f func() (client.Object, error)) error {
+	res, err := f()
+	t.Cleanup(func() {
+		t.Logf("Start to cleanup resilsence test resources")
+		if deleteErr := rts.Client.Delete(ctx, res); deleteErr != nil {
+		}
+
+		t.Logf("Clean up complete!")
+	})
+	return err
+}
+
+func (rts *ResilienceTestSuite) Kube() *kubernetes.KubeActions {
+	return rts.KubeActions
+}
+
+func (rts *ResilienceTestSuite) Run(t *testing.T, tests []ResilienceTest) {
+	t.Logf("Running %d resilience tests", len(tests))
+	for _, test := range tests {
+		t.Logf("Running resilience test: %s", test.ShortName)
+		test.Test(t, rts)
+	}
+}
+
+func (rts *ResilienceTestSuite) RegisterCleanup(t *testing.T, ctx context.Context, object client.Object) {
+	t.Cleanup(func() {
+		t.Logf("Start to cleanup resilsence test resources")
+		if deleteErr := rts.Client.Delete(ctx, object); deleteErr != nil {
+		}
+
+		t.Logf("Clean up complete!")
+	})
+}

test/resilience/testdata/base.yaml

@@ -0,0 +1,110 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gateway-resilience
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  namespace: gateway-resilience
+  name: envoy-gateway
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: custom-proxy-config
+    namespace: gateway-resilience
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: all-namespaces
+  namespace: gateway-resilience
+spec:
+  gatewayClassName: "{GATEWAY_CLASS_NAME}"
+  listeners:
+    - name: http
+      port: 80
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: All
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend
+  namespace: gateway-resilience
+spec:
+  selector:
+    app: backend
+  ports:
+    - protocol: TCP
+      port: 8080
+      name: http11
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend
+  namespace: gateway-resilience
+  labels:
+    app: backend
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: backend
+  template:
+    metadata:
+      labels:
+        app: backend
+    spec:
+      containers:
+        - name: backend
+          # From https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/echo-basic/echo-basic.go
+          image: gcr.io/k8s-staging-gateway-api/echo-basic:v20231214-v1.0.0-140-gf544a46e
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SERVICE_NAME
+              value: backend
+          resources:
+            requests:
+              cpu: 10m
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: backend
+  namespace: gateway-resilience
+spec:
+  parentRefs:
+    - name: all-namespaces
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /welcome
+      backendRefs:
+        - name: backend
+          port: 8080
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: custom-proxy-config
+  namespace: gateway-resilience
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+

test/resilience/testdata/route_changes.yaml

@@ -0,0 +1,16 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: backend
+  namespace: gateway-resilience
+spec:
+  parentRefs:
+    - name: all-namespaces
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /route-change
+      backendRefs:
+        - name: backend
+          port: 8080