From fdbe6681fbcd7a669c40a44cecee4994cfcba4d3 Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Mon, 27 Nov 2023 12:43:14 -0600 Subject: [PATCH] Add basic system test with utilities (#1274) Problem: In order to test a full NGF system running in k8s, we need a framework that can easily deploy apps and send traffic. Solution: Enhance the framework with functions to create apps and send traffic using port forwarding. Also added a basic test to utilize these functions as a proof of concept. --- Makefile | 2 +- tests/Makefile | 2 +- tests/README.md | 119 +++++++++ tests/framework/portforward.go | 91 +++++++ tests/framework/request.go | 50 ++++ tests/framework/resourcemanager.go | 313 +++++++++++++++++++++++ tests/framework/timeout.go | 31 +++ tests/suite/manifests/hello/gateway.yaml | 11 + tests/suite/manifests/hello/hello.yaml | 32 +++ tests/suite/manifests/hello/route.yaml | 18 ++ tests/suite/sample_test.go | 47 ++++ tests/{ => suite}/system_suite_test.go | 55 ++-- 12 files changed, 753 insertions(+), 18 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/framework/portforward.go create mode 100644 tests/framework/request.go create mode 100644 tests/framework/resourcemanager.go create mode 100644 tests/framework/timeout.go create mode 100644 tests/suite/manifests/hello/gateway.yaml create mode 100644 tests/suite/manifests/hello/hello.yaml create mode 100644 tests/suite/manifests/hello/route.yaml create mode 100644 tests/suite/sample_test.go rename tests/{ => suite}/system_suite_test.go (75%) diff --git a/Makefile b/Makefile index 304decea1f..3a126989dd 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ lint: ## Run golangci-lint against code .PHONY: unit-test unit-test: ## Run unit tests for the go code - go test ./... -tags unit -race -coverprofile cover.out + go test ./internal/... -race -coverprofile cover.out go tool cover -html=cover.out -o cover.html .PHONY: njs-unit-test diff --git a/tests/Makefile b/tests/Makefile index 528e8dabc7..a19276b00d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -22,7 +22,7 @@ load-images: ## Load NGF and NGINX images on configured kind cluster kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG) test: ## Run the system tests against your default k8s cluster - go test -v . -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \ + go test -v ./suite -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \ --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \ --k8s-version=$(K8S_VERSION) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..15cd3f6b15 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,119 @@ +# System Testing + +The tests in this directory are meant to be run on a live Kubernetes environment to verify a real system. These +are similar to the existing [conformance tests](../conformance/README.md), but will verify things such as: + +- NGF-specific functionality +- Non-Functional requirements testing (such as performance, scale, etc.) + +When running, the tests create a port-forward from your NGF Pod to localhost using a port chosen by the +test framework. Traffic is sent over this port. + +Directory structure is as follows: + +- `framework`: contains utility functions for running the tests +- `suite`: contains the test files + +**Note**: Existing NFR tests will be migrated into this testing `suite` and results stored in a `results` directory. + +## Prerequisites + +- Kubernetes cluster. +- Docker. +- Golang. + +**Note**: all commands in steps below are executed from the `tests` directory + +```shell +make +``` + +```text +build-images Build NGF and NGINX images +create-kind-cluster Create a kind cluster +delete-kind-cluster Delete kind cluster +help Display this help +load-images Load NGF and NGINX images on configured kind cluster +test Run the system tests against your default k8s cluster +``` + +**Note:** The following variables are configurable when running the below `make` commands: + +| Variable | Default | Description | +|----------|---------|-------------| +| TAG | edge | tag for the locally built NGF images | +| PREFIX | nginx-gateway-fabric | prefix for the locally built NGF image | +| NGINX_PREFIX | nginx-gateway-fabric/nginx | prefix for the locally built NGINX image | +| PULL_POLICY | Never | NGF image pull policy | +| GW_API_VERSION | 1.0.0 | version of Gateway API resources to install | +| K8S_VERSION | latest | version of k8s that the tests are run on | + +## Step 1 - Create a Kubernetes cluster + +This can be done in a cloud provider of choice, or locally using `kind`: + +```makefile +make create-kind-cluster +``` + +> Note: The default kind cluster deployed is the latest available version. You can specify a different version by +> defining the kind image to use through the KIND_IMAGE variable, e.g. + +```makefile +make create-kind-cluster KIND_IMAGE=kindest/node:v1.27.3 +``` + +## Step 2 - Build and Load Images + +Loading the images only applies to a `kind` cluster. If using a cloud provider, you will need to tag and push +your images to a registry that is accessible from that cloud provider. + +```makefile +make build-images load-images TAG=$(whoami) +``` + +## Step 3 - Run the tests + +```makefile +make test TAG=$(whoami) +``` + +To run a specific test, you can "focus" it by adding the `F` prefix to the name. For example: + +```go +It("runs some test", func(){ + ... +}) +``` + +becomes: + +```go +FIt("runs some test", func(){ + ... +}) +``` + +This can also be done at higher levels like `Context`. + +To disable a specific test, add the `X` prefix to it, similar to the previous example: + +```go +It("runs some test", func(){ + ... +}) +``` + +becomes: + +```go +XIt("runs some test", func(){ + ... +}) +``` + +## Step 4 - Delete kind cluster + +```makefile +make delete-kind-cluster +``` diff --git a/tests/framework/portforward.go b/tests/framework/portforward.go new file mode 100644 index 0000000000..5e7378cc02 --- /dev/null +++ b/tests/framework/portforward.go @@ -0,0 +1,91 @@ +package framework + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/url" + "path" + "time" + + core "k8s.io/api/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetNGFPodName returns the name of the NGF Pod. +func GetNGFPodName( + k8sClient client.Client, + namespace, + releaseName string, + timeout time.Duration, +) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + var podList core.PodList + if err := k8sClient.List( + ctx, + &podList, + client.InNamespace(namespace), + client.MatchingLabels{ + "app.kubernetes.io/instance": releaseName, + }, + ); err != nil { + return "", fmt.Errorf("error getting list of Pods: %w", err) + } + + if len(podList.Items) > 0 { + return podList.Items[0].Name, nil + } + + return "", errors.New("unable to find NGF Pod") +} + +// PortForward starts a port-forward to the specified Pod and returns the local port being forwarded. +func PortForward(config *rest.Config, namespace, podName string, stopCh chan struct{}) (int, error) { + roundTripper, upgrader, err := spdy.RoundTripperFor(config) + if err != nil { + return 0, fmt.Errorf("error creating roundtripper: %w", err) + } + + serverURL, err := url.Parse(config.Host) + if err != nil { + return 0, fmt.Errorf("error parsing rest config host: %w", err) + } + + serverURL.Path = path.Join( + "api", "v1", + "namespaces", namespace, + "pods", podName, + "portforward", + ) + + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL) + + readyCh := make(chan struct{}, 1) + out, errOut := new(bytes.Buffer), new(bytes.Buffer) + + forwarder, err := portforward.New(dialer, []string{":80"}, stopCh, readyCh, out, errOut) + if err != nil { + return 0, fmt.Errorf("error creating port forwarder: %w", err) + } + + go func() { + if err := forwarder.ForwardPorts(); err != nil { + panic(err) + } + }() + + <-readyCh + ports, err := forwarder.GetPorts() + if err != nil { + return 0, fmt.Errorf("error getting ports being forwarded: %w", err) + } + + return int(ports[0].Local), nil +} diff --git a/tests/framework/request.go b/tests/framework/request.go new file mode 100644 index 0000000000..bf68fd0519 --- /dev/null +++ b/tests/framework/request.go @@ -0,0 +1,50 @@ +package framework + +import ( + "bytes" + "context" + "fmt" + "net" + "net/http" + "strings" + "time" +) + +// Get sends a GET request to the specified url. +// It resolves to localhost (where the NGF port-forward is running) instead of using DNS. +// The status and body of the response is returned, or an error. +func Get(url string, timeout time.Duration) (int, string, error) { + dialer := &net.Dialer{} + + http.DefaultTransport.(*http.Transport).DialContext = func( + ctx context.Context, + network, + addr string, + ) (net.Conn, error) { + split := strings.Split(addr, ":") + port := split[len(split)-1] + return dialer.DialContext(ctx, network, fmt.Sprintf("127.0.0.1:%s", port)) + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return 0, "", err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + + body := new(bytes.Buffer) + _, err = body.ReadFrom(resp.Body) + if err != nil { + return resp.StatusCode, "", err + } + + return resp.StatusCode, body.String(), nil +} diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go new file mode 100644 index 0000000000..ab2f26de18 --- /dev/null +++ b/tests/framework/resourcemanager.go @@ -0,0 +1,313 @@ +// Utility functions for managing resources in Kubernetes. Inspiration and methods used from +// https://github.com/kubernetes-sigs/gateway-api/tree/main/conformance/utils. + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "context" + "embed" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + core "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + v1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// ResourceManager handles creating/updating/deleting Kubernetes resources. +type ResourceManager struct { + K8sClient client.Client + FS embed.FS + TimeoutConfig TimeoutConfig +} + +// Apply creates or updates Kubernetes resources defined as Go objects. +func (rm *ResourceManager) Apply(resources []client.Object) error { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) + defer cancel() + + for _, resource := range resources { + if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), resource); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error getting resource: %w", err) + } + + if err := rm.K8sClient.Create(ctx, resource); err != nil { + return fmt.Errorf("error creating resource: %w", err) + } + + continue + } + + if err := rm.K8sClient.Update(ctx, resource); err != nil { + return fmt.Errorf("error updating resource: %w", err) + } + } + + return nil +} + +// ApplyFromFiles creates or updates Kubernetes resources defined within the provided YAML files. +func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string) error { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) + defer cancel() + + handlerFunc := func(obj unstructured.Unstructured) error { + obj.SetNamespace(namespace) + nsName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} + fetchedObj := obj.DeepCopy() + if err := rm.K8sClient.Get(ctx, nsName, fetchedObj); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("error getting resource: %w", err) + } + + if err := rm.K8sClient.Create(ctx, &obj); err != nil { + return fmt.Errorf("error creating resource: %w", err) + } + + return nil + } + + obj.SetResourceVersion(fetchedObj.GetResourceVersion()) + if err := rm.K8sClient.Update(ctx, &obj); err != nil { + return fmt.Errorf("error updating resource: %w", err) + } + + return nil + } + + return rm.readAndHandleObjects(handlerFunc, files) +} + +// Delete deletes Kubernetes resources defined as Go objects. +func (rm *ResourceManager) Delete(resources []client.Object) error { + for _, resource := range resources { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) + defer cancel() + + if err := rm.K8sClient.Delete(ctx, resource); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting resource: %w", err) + } + } + + return nil +} + +// DeleteFromFile deletes Kubernetes resources defined within the provided YAML files. +func (rm *ResourceManager) DeleteFromFiles(files []string, namespace string) error { + handlerFunc := func(obj unstructured.Unstructured) error { + obj.SetNamespace(namespace) + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) + defer cancel() + + if err := rm.K8sClient.Delete(ctx, &obj); err != nil && !apierrors.IsNotFound(err) { + return err + } + + return nil + } + + return rm.readAndHandleObjects(handlerFunc, files) +} + +func (rm *ResourceManager) readAndHandleObjects( + handle func(unstructured.Unstructured) error, + files []string, +) error { + for _, file := range files { + data, err := rm.getFileContents(file) + if err != nil { + return err + } + + decoder := yaml.NewYAMLOrJSONDecoder(data, 4096) + for { + obj := unstructured.Unstructured{} + if err := decoder.Decode(&obj); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("error decoding resource: %w", err) + } + + if len(obj.Object) == 0 { + continue + } + + if err := handle(obj); err != nil { + return err + } + } + } + + return nil +} + +// getFileContents takes a string that can either be a local file +// path or an https:// URL to YAML manifests and provides the contents. +func (rm *ResourceManager) getFileContents(file string) (*bytes.Buffer, error) { + if strings.HasPrefix(file, "http://") { + return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + } else if strings.HasPrefix(file, "https://") { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.ManifestFetchTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, file, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + } + + manifests := new(bytes.Buffer) + count, err := manifests.ReadFrom(resp.Body) + if err != nil { + return nil, err + } + + if resp.ContentLength != -1 && count != resp.ContentLength { + return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + } + return manifests, nil + } + + if !strings.HasPrefix(file, "manifests/") { + file = "manifests/" + file + } + + b, err := rm.FS.ReadFile(file) + if err != nil { + return nil, err + } + + return bytes.NewBuffer(b), nil +} + +// WaitForAppsToBeReady waits for all apps in the specified namespace to be ready, +// or until the ctx timeout is reached. +func (rm *ResourceManager) WaitForAppsToBeReady(namespace string) error { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) + defer cancel() + + if err := rm.waitForPodsToBeReady(ctx, namespace); err != nil { + return err + } + + if err := rm.waitForRoutesToBeReady(ctx, namespace); err != nil { + return err + } + + return rm.waitForGatewaysToBeReady(ctx, namespace) +} + +func (rm *ResourceManager) waitForPodsToBeReady(ctx context.Context, namespace string) error { + return wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var podList core.PodList + if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + return false, err + } + + var podsReady int + for _, pod := range podList.Items { + for _, cond := range pod.Status.Conditions { + if cond.Type == core.PodReady && cond.Status == core.ConditionTrue { + podsReady++ + } + } + } + + return podsReady == len(podList.Items), nil + }, + ) +} + +func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespace string) error { + return wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var gatewayList v1.GatewayList + if err := rm.K8sClient.List(ctx, &gatewayList, client.InNamespace(namespace)); err != nil { + return false, err + } + + for _, gw := range gatewayList.Items { + for _, cond := range gw.Status.Conditions { + if cond.Type == string(v1.GatewayConditionProgrammed) && cond.Status == metav1.ConditionTrue { + return true, nil + } + } + } + + return false, nil + }, + ) +} + +func (rm *ResourceManager) waitForRoutesToBeReady(ctx context.Context, namespace string) error { + return wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + var routeList v1.HTTPRouteList + if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + return false, err + } + + var numParents, readyCount int + for _, route := range routeList.Items { + numParents += len(route.Status.Parents) + for _, parent := range route.Status.Parents { + for _, cond := range parent.Conditions { + if cond.Type == string(v1.RouteConditionAccepted) && cond.Status == metav1.ConditionTrue { + readyCount++ + } + } + } + } + + return numParents == readyCount, nil + }, + ) +} diff --git a/tests/framework/timeout.go b/tests/framework/timeout.go new file mode 100644 index 0000000000..d49e988a50 --- /dev/null +++ b/tests/framework/timeout.go @@ -0,0 +1,31 @@ +package framework + +import "time" + +type TimeoutConfig struct { + // CreateTimeout represents the maximum time for a Kubernetes object to be created. + CreateTimeout time.Duration + + // DeleteTimeout represents the maximum time for a Kubernetes object to be deleted. + DeleteTimeout time.Duration + + // GetTimeout represents the maximum time to get a Kubernetes object. + GetTimeout time.Duration + + // ManifestFetchTimeout represents the maximum time for getting content from a https:// URL. + ManifestFetchTimeout time.Duration + + // RequestTimeout represents the maximum time for making an HTTP Request with the roundtripper. + RequestTimeout time.Duration +} + +// DefaultTimeoutConfig populates a TimeoutConfig with the default values. +func DefaultTimeoutConfig() TimeoutConfig { + return TimeoutConfig{ + CreateTimeout: 60 * time.Second, + DeleteTimeout: 10 * time.Second, + GetTimeout: 10 * time.Second, + ManifestFetchTimeout: 10 * time.Second, + RequestTimeout: 10 * time.Second, + } +} diff --git a/tests/suite/manifests/hello/gateway.yaml b/tests/suite/manifests/hello/gateway.yaml new file mode 100644 index 0000000000..e6507f613b --- /dev/null +++ b/tests/suite/manifests/hello/gateway.yaml @@ -0,0 +1,11 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: "*.example.com" diff --git a/tests/suite/manifests/hello/hello.yaml b/tests/suite/manifests/hello/hello.yaml new file mode 100644 index 0000000000..1d566b2159 --- /dev/null +++ b/tests/suite/manifests/hello/hello.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello +spec: + replicas: 1 + selector: + matchLabels: + app: hello + template: + metadata: + labels: + app: hello + spec: + containers: + - name: hello + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: hello diff --git a/tests/suite/manifests/hello/route.yaml b/tests/suite/manifests/hello/route.yaml new file mode 100644 index 0000000000..e70cce260f --- /dev/null +++ b/tests/suite/manifests/hello/route.yaml @@ -0,0 +1,18 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: hello +spec: + parentRefs: + - name: gateway + sectionName: http + hostnames: + - "hello.example.com" + rules: + - matches: + - path: + type: Exact + value: /hello + backendRefs: + - name: hello + port: 80 diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go new file mode 100644 index 0000000000..64e51d54c9 --- /dev/null +++ b/tests/suite/sample_test.go @@ -0,0 +1,47 @@ +package suite + +import ( + "fmt" + "net/http" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nginxinc/nginx-gateway-fabric/tests/framework" +) + +var _ = Describe("Basic test example", func() { + files := []string{ + "hello/hello.yaml", + "hello/gateway.yaml", + "hello/route.yaml", + } + ns := &core.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hello", + }, + } + + BeforeEach(func() { + Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + }) + + AfterEach(func() { + Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) + Expect(resourceManager.Delete([]client.Object{ns})).To(Succeed()) + }) + + It("sends traffic", func() { + url := fmt.Sprintf("http://hello.example.com:%s/hello", strconv.Itoa(portFwdPort)) + status, body, err := framework.Get(url, timeoutConfig.RequestTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(status).To(Equal(http.StatusOK)) + Expect(body).To(ContainSubstring("URI: /hello")) + }) +}) diff --git a/tests/system_suite_test.go b/tests/suite/system_suite_test.go similarity index 75% rename from tests/system_suite_test.go rename to tests/suite/system_suite_test.go index 0a770f0f4f..c98ad554a8 100644 --- a/tests/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -1,9 +1,7 @@ -//go:build !unit -// +build !unit - -package tests +package suite import ( + "embed" "flag" "path" "path/filepath" @@ -18,6 +16,7 @@ import ( k8sRuntime "k8s.io/apimachinery/pkg/runtime" ctlr "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + v1 "sigs.k8s.io/gateway-api/apis/v1" "github.com/nginxinc/nginx-gateway-fabric/tests/framework" ) @@ -32,7 +31,25 @@ func TestNGF(t *testing.T) { RunSpecs(t, "NGF System Tests") } -var k8sClient client.Client +var ( + gatewayAPIVersion = flag.String("gateway-api-version", "", "Version of Gateway API to install") + k8sVersion = flag.String("k8s-version", "latest", "Version of k8s being tested on") + // Configurable NGF installation variables. Helm values will be used as defaults if not specified. + ngfImageRepository = flag.String("ngf-image-repo", "", "Image repo for NGF control plane") + nginxImageRepository = flag.String("nginx-image-repo", "", "Image repo for NGF data plane") + imageTag = flag.String("image-tag", "", "Image tag for NGF images") + imagePullPolicy = flag.String("pull-policy", "", "Image pull policy for NGF images") +) + +var ( + //go:embed manifests/* + manifests embed.FS + k8sClient client.Client + resourceManager framework.ResourceManager + portForwardStopCh = make(chan struct{}, 1) + portFwdPort int + timeoutConfig framework.TimeoutConfig +) var _ = BeforeSuite(func() { k8sConfig := ctlr.GetConfigOrDie() @@ -40,6 +57,7 @@ var _ = BeforeSuite(func() { Expect(core.AddToScheme(scheme)).To(Succeed()) Expect(apps.AddToScheme(scheme)).To(Succeed()) Expect(apiext.AddToScheme(scheme)).To(Succeed()) + Expect(v1.AddToScheme(scheme)).To(Succeed()) options := client.Options{ Scheme: scheme, @@ -49,8 +67,15 @@ var _ = BeforeSuite(func() { k8sClient, err = client.New(k8sConfig, options) Expect(err).ToNot(HaveOccurred()) + timeoutConfig = framework.DefaultTimeoutConfig() + resourceManager = framework.ResourceManager{ + K8sClient: k8sClient, + FS: manifests, + TimeoutConfig: timeoutConfig, + } + _, file, _, _ := runtime.Caller(0) - fileDir := path.Join(path.Dir(file)) + fileDir := path.Join(path.Dir(file), "../") basepath := filepath.Dir(fileDir) cfg := framework.InstallationConfig{ @@ -69,9 +94,17 @@ var _ = BeforeSuite(func() { output, err = framework.InstallNGF(cfg) Expect(err).ToNot(HaveOccurred(), string(output)) + + podName, err := framework.GetNGFPodName(k8sClient, cfg.Namespace, cfg.ReleaseName, timeoutConfig.CreateTimeout) + Expect(err).ToNot(HaveOccurred()) + + portFwdPort, err = framework.PortForward(k8sConfig, cfg.Namespace, podName, portForwardStopCh) + Expect(err).ToNot(HaveOccurred()) }) var _ = AfterSuite(func() { + portForwardStopCh <- struct{}{} + cfg := framework.InstallationConfig{ ReleaseName: "ngf-test", Namespace: "nginx-gateway", @@ -83,13 +116,3 @@ var _ = AfterSuite(func() { output, err = framework.UninstallGatewayAPI(*gatewayAPIVersion, *k8sVersion) Expect(err).ToNot(HaveOccurred(), string(output)) }) - -var ( - gatewayAPIVersion = flag.String("gateway-api-version", "", "Version of Gateway API to install") - k8sVersion = flag.String("k8s-version", "latest", "Version of k8s being tested on") - // Configurable NGF installation variables. Helm values will be used as defaults if not specified. - ngfImageRepository = flag.String("ngf-image-repo", "", "Image repo for NGF control plane") - nginxImageRepository = flag.String("nginx-image-repo", "", "Image repo for NGF data plane") - imageTag = flag.String("image-tag", "", "Image tag for NGF images") - imagePullPolicy = flag.String("pull-policy", "", "Image pull policy for NGF images") -)