Skip to content

Commit

Permalink
Automate scale test (#1926)
Browse files Browse the repository at this point in the history 
Problem:
Non-functional scale test needs to be run manually.

Solution:
- Automate scale test.
- Use in-cluster Prometheus to collect CPU, memory and NGF metrics.
- Use Kubernetes API server to get NGF logs.

For development and troubleshooting, it is possible to run scale test
locally in Kind cluster. However, it is necessary to bring down
the number of HTTPRoutes to 50 or less (roughly).

Testing:
- Ran this test locally with 64 listeners, 50 routes and 50 upstreams
  with NGINX OSS.
- Ran this test locally with 64 listeners, 10 routes and 10 upstreams with NGINX Plus.
- Ran this test on GKE with the default configuration with NGINX OSS.

Out of scope: ensuring this test runs successfully via GitHub pipeline.

Closes #1368

Largely based on work by Ciara in
#1804

Co-authored-by: Ciara Stacke <[email protected]>
  • Loading branch information
@pleshakov @ciarams87
pleshakov and ciarams87 authored May 17, 2024
1 parent 2cd63e7 commit ee4319c
Show file tree
Hide file tree
Showing 84 changed files with 1,533 additions and 962 deletions.
2 changes: 1 addition & 1 deletion tests/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ test: ## Runs the functional tests on your default k8s cluster
--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \
--gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \
--image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) \
--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
--pull-policy=$(PULL_POLICY) --k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) \
--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)

Expand Down
183 changes: 111 additions & 72 deletions tests/scale/generate_manifests.go → tests/framework/generate_manifests.go
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
//go:build scale
// +build scale

package scale
package framework

import (
"bytes"
"errors"
"fmt"
"os"
"path/filepath"
"io"
"text/template"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/yaml"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var gwTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
const gwTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
name: gateway
Expand All @@ -33,7 +34,7 @@ spec:
{{- end -}}
{{- end -}}`

var hrTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
const hrTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
name: {{ .Name }}
Expand All @@ -53,7 +54,7 @@ spec:
port: 80`

// nolint:all
var secretTmplTxt = `apiVersion: v1
const secretTmplTxt = `apiVersion: v1
kind: Secret
metadata:
name: {{ . }}
Expand All @@ -63,8 +64,7 @@ data:
tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzZtTnJSdUZ2WXZoSE4KbXI3c1FvNUtKSUVDN3N6TFVrNExFeklSNS9yMEVaUjQ2RnRTaGJQd0ZuaXAwMFBxekhpVkhKYy92TjdkQTVLeApQS1VmdFJuQ1J6YldVaTZBZzJpRU93bXF6WUhGbVNpZkFlVjk0RlAxOGtSbjl1ckV3OEpiRXJIUncrVW51L25tCmFMRHF1eGpFTVBweGhuRklCSnYwK1R3djNEVGx6TjNwUlV6dnpidGZvZCtEVTZBSmR6N3Rid1dTNmR6MHc1Z2kKbW9RelZnbFpnVDBJek9FZkV3NVpWMnRMZllHZWRlRVJ1VjhtR041c09va3R2aGxsMU1udHRaMkZNVHgySmVjUQo3K0xBRm9YVnBTS2NjbUFVZ1JBM0xOOHdVZXBVTHZZdFhiUm1QTFc4SjFINmhFeHJHTHBiTERZNmpzbGxBNlZpCk0xMjVjU0hsQWdNQkFBRUNnZ0VBQnpaRE50bmVTdWxGdk9HZlFYaHRFWGFKdWZoSzJBenRVVVpEcUNlRUxvekQKWlV6dHdxbkNRNlJLczUyandWNTN4cU9kUU94bTNMbjNvSHdNa2NZcEliWW82MjJ2dUczYnkwaVEzaFlsVHVMVgpqQmZCcS9UUXFlL2NMdngvSkczQWhFNmJxdFRjZFlXeGFmTmY2eUtpR1dzZk11WVVXTWs4MGVJVUxuRmZaZ1pOCklYNTlSOHlqdE9CVm9Sa3hjYTVoMW1ZTDFsSlJNM3ZqVHNHTHFybmpOTjNBdWZ3ZGRpK1VDbGZVL2l0K1EvZkUKV216aFFoTlRpNVFkRWJLVStOTnYvNnYvb2JvandNb25HVVBCdEFTUE05cmxFemIralQ1WHdWQjgvLzRGY3VoSwoyVzNpcjhtNHVlQ1JHSVlrbGxlLzhuQmZ0eVhiVkNocVRyZFBlaGlPM1FLQmdRRGlrR3JTOTc3cjg3Y1JPOCtQClpoeXltNXo4NVIzTHVVbFNTazJiOTI1QlhvakpZL2RRZDVTdFVsSWE4OUZKZnNWc1JRcEhHaTFCYzBMaTY1YjIKazR0cE5xcVFoUmZ1UVh0UG9GYXRuQzlPRnJVTXJXbDVJN0ZFejZnNkNQMVBXMEg5d2hPemFKZUdpZVpNYjlYTQoybDdSSFZOcC9jTDlYbmhNMnN0Q1lua2Iwd0tCZ1FEUzF4K0crakEyUVNtRVFWNXA1RnRONGcyamsyZEFjMEhNClRIQ2tTazFDRjhkR0Z2UWtsWm5ZbUt0dXFYeXNtekJGcnZKdmt2eUhqbUNYYTducXlpajBEdDZtODViN3BGcVAKQWxtajdtbXI3Z1pUeG1ZMXBhRWFLMXY4SDNINGtRNVl3MWdrTWRybVJHcVAvaTBGaDVpaGtSZS9DOUtGTFVkSQpDcnJjTzhkUVp3S0JnSHA1MzRXVWNCMVZibzFlYStIMUxXWlFRUmxsTWlwRFM2TzBqeWZWSmtFb1BZSEJESnp2ClIrdzZLREJ4eFoyWmJsZ05LblV0YlhHSVFZd3lGelhNcFB5SGxNVHpiZkJhYmJLcDFyR2JVT2RCMXpXM09PRkgKcmppb21TUm1YNmxhaDk0SjRHU0lFZ0drNGw1SHhxZ3JGRDZ2UDd4NGRjUktJWFpLZ0w2dVJSSUpBb0dCQU1CVApaL2p5WStRNTBLdEtEZHUrYU9ORW4zaGxUN3hrNXRKN3NBek5rbWdGMU10RXlQUk9Xd1pQVGFJbWpRbk9qbHdpCldCZ2JGcXg0M2ZlQ1Z4ZXJ6V3ZEM0txaWJVbWpCTkNMTGtYeGh3ZEVteFQwVit2NzZGYzgwaTNNYVdSNnZZR08KditwVVovL0F6UXdJcWZ6dlVmV2ZxdStrMHlhVXhQOGNlcFBIRyt0bEFvR0FmQUtVVWhqeFU0Ym5vVzVwVUhKegpwWWZXZXZ5TW54NWZyT2VsSmRmNzlvNGMvMHhVSjh1eFBFWDFkRmNrZW96dHNpaVFTNkN6MENRY09XVWxtSkRwCnVrdERvVzM3VmNSQU1BVjY3NlgxQVZlM0UwNm5aL2g2Tkd4Z28rT042Q3pwL0lkMkJPUm9IMFAxa2RjY1NLT3kKMUtFZlNnb1B0c1N1eEpBZXdUZmxDMXc9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
`

var appTmplTxt = `apiVersion: v1
apiVersion: apps/v1
const appTmplTxt = `apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ . }}
Expand Down Expand Up @@ -105,25 +105,55 @@ var (
appTmpl = template.Must(template.New("app").Parse(appTmplTxt))
)

type Listener struct {
type listener struct {
Name string
HostnamePrefix string
SecretName string
}

type Route struct {
type route struct {
Name string
ListenerName string
HostnamePrefix string
BackendName string
}

func getPrereqDirName(manifestDir string) string {
return filepath.Join(manifestDir, "prereqs")
// ScaleObjects contains objects for scale testing.
type ScaleObjects struct {
// BaseObjects contains objects that are common to all scale iterations.
BaseObjects []client.Object
// ScaleIterationGroups contains objects for each scale iteration.
ScaleIterationGroups [][]client.Object
}

func generateScaleListenerManifests(numListeners int, manifestDir string, tls bool) error {
listeners := make([]Listener, 0)
func decodeObjects(reader io.Reader) ([]client.Object, error) {
var objects []client.Object

decoder := yaml.NewYAMLOrJSONDecoder(reader, 4096)
for {
obj := unstructured.Unstructured{}
if err := decoder.Decode(&obj); err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("error decoding resource: %w", err)
}

if len(obj.Object) == 0 {
continue
}

objects = append(objects, &obj)
}

return objects, nil
}

// GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test.
func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) {
var result ScaleObjects

listeners := make([]listener, 0)
backends := make([]string, 0)
secrets := make([]string, 0)

Expand All @@ -138,13 +168,13 @@ func generateScaleListenerManifests(numListeners int, manifestDir string, tls bo
secrets = append(secrets, secretName)
}

listeners = append(listeners, Listener{
listeners = append(listeners, listener{
Name: listenerName,
HostnamePrefix: hostnamePrefix,
SecretName: secretName,
})

route := Route{
r := route{
Name: fmt.Sprintf("route-%d", i),
ListenerName: listenerName,
HostnamePrefix: hostnamePrefix,
Expand All @@ -153,80 +183,101 @@ func generateScaleListenerManifests(numListeners int, manifestDir string, tls bo

backends = append(backends, backendName)

if err := generateManifests(manifestDir, i, listeners, []Route{route}); err != nil {
return err
objects, err := generateManifests(listeners, []route{r})
if err != nil {
return ScaleObjects{}, err
}

result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects)
}

if err := generateSecrets(getPrereqDirName(manifestDir), secrets); err != nil {
return err
secretObjects, err := generateSecrets(secrets)
if err != nil {
return ScaleObjects{}, err
}

return generateBackendAppManifests(getPrereqDirName(manifestDir), backends)
}
result.BaseObjects = append(result.BaseObjects, secretObjects...)

func generateSecrets(secretsDir string, secrets []string) error {
err := os.Mkdir(secretsDir, 0o750)
if err != nil && !os.IsExist(err) {
return err
backendObjects, err := generateBackendAppObjects(backends)
if err != nil {
return ScaleObjects{}, err
}

result.BaseObjects = append(result.BaseObjects, backendObjects...)

return result, nil
}

func generateSecrets(secrets []string) ([]client.Object, error) {
objects := make([]client.Object, 0, len(secrets))

for _, secret := range secrets {
var buf bytes.Buffer

if err = secretTmpl.Execute(&buf, secret); err != nil {
return err
if err := secretTmpl.Execute(&buf, secret); err != nil {
return nil, err
}

path := filepath.Join(secretsDir, fmt.Sprintf("%s.yaml", secret))

fmt.Println("Writing", path)
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
return err
objs, err := decodeObjects(&buf)
if err != nil {
return nil, err
}

objects = append(objects, objs...)
}

return nil
return objects, nil
}

func generateScaleHTTPRouteManifests(numRoutes int, manifestDir string) error {
l := Listener{
// GenerateScaleHTTPRouteObjects generates objects for a given number of routes for the scale test.
func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) {
var result ScaleObjects

l := listener{
Name: "listener",
HostnamePrefix: "*",
}

backendName := "backend"

for i := 0; i < numRoutes; i++ {

route := Route{
r := route{
Name: fmt.Sprintf("route-%d", i),
HostnamePrefix: fmt.Sprintf("%d", i),
ListenerName: "listener",
BackendName: backendName,
}

var listeners []Listener
var listeners []listener
if i == 0 {
// only generate a Gateway on the first iteration
listeners = []Listener{l}
listeners = []listener{l}
}

if err := generateManifests(manifestDir, i, listeners, []Route{route}); err != nil {
return err
objects, err := generateManifests(listeners, []route{r})
if err != nil {
return ScaleObjects{}, err
}

result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects)
}

backendObjects, err := generateBackendAppObjects([]string{backendName})
if err != nil {
return ScaleObjects{}, err
}

return generateBackendAppManifests(getPrereqDirName(manifestDir), []string{backendName})
result.BaseObjects = backendObjects

return result, nil
}

func generateManifests(outDir string, version int, listeners []Listener, routes []Route) error {
func generateManifests(listeners []listener, routes []route) ([]client.Object, error) {
var buf bytes.Buffer

if len(listeners) > 0 {
if err := gwTmpl.Execute(&buf, listeners); err != nil {
return err
return nil, err
}
}

Expand All @@ -236,42 +287,30 @@ func generateManifests(outDir string, version int, listeners []Listener, routes
}

if err := hrTmpl.Execute(&buf, r); err != nil {
return err
return nil, err
}
}

err := os.Mkdir(outDir, 0o750)
if err != nil && !os.IsExist(err) {
return err
}

filename := fmt.Sprintf("manifest-%d.yaml", version)
path := filepath.Join(outDir, filename)

fmt.Println("Writing", path)
return os.WriteFile(path, buf.Bytes(), 0o600)
return decodeObjects(&buf)
}

func generateBackendAppManifests(outDir string, backends []string) error {
err := os.Mkdir(outDir, 0o750)
if err != nil && !os.IsExist(err) {
return err
}
func generateBackendAppObjects(backends []string) ([]client.Object, error) {
objects := make([]client.Object, 0, 2*len(backends))

for _, backend := range backends {
var buf bytes.Buffer

if err = appTmpl.Execute(&buf, backend); err != nil {
return err
if err := appTmpl.Execute(&buf, backend); err != nil {
return nil, err
}

path := filepath.Join(outDir, fmt.Sprintf("%s.yaml", backend))

fmt.Println("Writing", path)
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
return err
objs, err := decodeObjects(&buf)
if err != nil {
return nil, err
}

objects = append(objects, objs...)
}

return nil
return objects, nil
}
Loading

0 comments on commit ee4319c

Please sign in to comment.