From a15ad570a5f0234875810ddb6caf73d0d4ddc754 Mon Sep 17 00:00:00 2001 From: mleader Date: Thu, 14 Sep 2023 18:17:53 -0400 Subject: [PATCH 1/2] start storage stress scenario fix bugs in definitions missing gc in workload bug move workload remove gc key add regexp for storage provisioner fix lint errors iterate on storage provisioner option remove dead code add input variable for storage claim size add container image template injection to pod use metrics for workload fix lint error add default sc remove cpu-kubelet metric check --- .../ocp-config/pvc-density/pod.yml | 34 ++++++++++ .../ocp-config/pvc-density/pvc-density.yml | 55 ++++++++++++++++ .../ocp-config/pvc-density/pvc.yml | 12 ++++ .../ocp-config/pvc-density/storageclass.yml | 7 ++ cmd/kube-burner/ocp.go | 1 + pkg/workloads/pvc-density.go | 64 +++++++++++++++++++ pkg/workloads/workloads.go | 1 + test/test-ocp.bats | 8 +++ 8 files changed, 182 insertions(+) create mode 100644 cmd/kube-burner/ocp-config/pvc-density/pod.yml create mode 100644 cmd/kube-burner/ocp-config/pvc-density/pvc-density.yml create mode 100644 cmd/kube-burner/ocp-config/pvc-density/pvc.yml create mode 100644 cmd/kube-burner/ocp-config/pvc-density/storageclass.yml create mode 100644 pkg/workloads/pvc-density.go diff --git a/cmd/kube-burner/ocp-config/pvc-density/pod.yml b/cmd/kube-burner/ocp-config/pvc-density/pod.yml new file mode 100644 index 000000000..33002675f --- /dev/null +++ b/cmd/kube-burner/ocp-config/pvc-density/pod.yml @@ -0,0 +1,34 @@ +kind: Pod +apiVersion: v1 +metadata: + labels: + app: pvc-density-{{.Iteration}} + name: pod-{{.Iteration}} +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: Exists + - key: node-role.kubernetes.io/infra + operator: DoesNotExist + - key: node-role.kubernetes.io/workload + operator: DoesNotExist + tolerations: + - key: os + value: Windows + effect: NoSchedule + volumes: + - name: storage-stress-vlm-{{.Iteration}} + persistentVolumeClaim: + claimName: pvc-{{.Iteration}} + containers: + - image: {{.containerImage}} + name: pvc-density-container + resources: + requests: + memory: "10Mi" + cpu: "10m" + imagePullPolicy: IfNotPresent diff --git a/cmd/kube-burner/ocp-config/pvc-density/pvc-density.yml b/cmd/kube-burner/ocp-config/pvc-density/pvc-density.yml new file mode 100644 index 000000000..b1bbadba3 --- /dev/null +++ b/cmd/kube-burner/ocp-config/pvc-density/pvc-density.yml @@ -0,0 +1,55 @@ +--- +global: + gc: {{.GC}} + indexerConfig: + esServers: ["{{.ES_SERVER}}"] + insecureSkipVerify: true + defaultIndex: {{.ES_INDEX}} + type: {{.INDEXING_TYPE}} + measurements: + - name: podLatency + +jobs: + - name: add-default-storage + namespace: pvc-density + jobIterations: 1 + namespacedIterations: false + podWait: false + waitWhenFinished: true + namespaceLabels: + security.openshift.io/scc.podSecurityLabelSync: false + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + objects: + + - objectTemplate: storageclass.yml + inputVars: + storageProvisioner: {{.STORAGE_PROVISIONER}} + replicas: 1 + + - name: pvc-density + namespace: pvc-density + jobIterations: {{.JOB_ITERATIONS}} + cleanup: true + namespacedIterations: false + podWait: false + waitWhenFinished: true + preLoadImages: true + preLoadPeriod: 10s + namespaceLabels: + security.openshift.io/scc.podSecurityLabelSync: false + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + objects: + + - objectTemplate: pvc.yml + replicas: 1 + inputVars: + claimSize: {{.CLAIM_SIZE}} + + - objectTemplate: pod.yml + replicas: 1 + inputVars: + containerImage: registry.k8s.io/pause:3.1 diff --git a/cmd/kube-burner/ocp-config/pvc-density/pvc.yml b/cmd/kube-burner/ocp-config/pvc-density/pvc.yml new file mode 100644 index 000000000..6c9f7195c --- /dev/null +++ b/cmd/kube-burner/ocp-config/pvc-density/pvc.yml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + labels: + app: pvc-density-{{.Iteration}} + name: pvc-{{.Iteration}} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{.claimSize}} \ No newline at end of file diff --git a/cmd/kube-burner/ocp-config/pvc-density/storageclass.yml b/cmd/kube-burner/ocp-config/pvc-density/storageclass.yml new file mode 100644 index 000000000..0e7d11276 --- /dev/null +++ b/cmd/kube-burner/ocp-config/pvc-density/storageclass.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: pvc-density-sc + annotations: + storageclass.kubernetes.io/is-default-class: 'true' +provisioner: {{.storageProvisioner}} \ No newline at end of file diff --git a/cmd/kube-burner/ocp.go b/cmd/kube-burner/ocp.go index 201a0e90a..0b8338dd4 100644 --- a/cmd/kube-burner/ocp.go +++ b/cmd/kube-burner/ocp.go @@ -100,6 +100,7 @@ func openShiftCmd() *cobra.Command { workloads.NewNodeDensityHeavy(&wh), workloads.NewNodeDensityCNI(&wh), workloads.NewIndex(&wh.MetricsEndpoint, &wh.Metadata, &wh.OcpMetaAgent), + workloads.NewPVCDensity(&wh), ) return ocpCmd } diff --git a/pkg/workloads/pvc-density.go b/pkg/workloads/pvc-density.go new file mode 100644 index 000000000..1b82a0b49 --- /dev/null +++ b/pkg/workloads/pvc-density.go @@ -0,0 +1,64 @@ +package workloads + +import ( + "fmt" + "os" + "regexp" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var dynamicStorageProvisioners = map[string]string{ + "aws": "kubernetes.io/aws-ebs", + "cinder": "kuberenetes.io/cinder", + "azure-disk": "kubernetes.io/azure-disk", + "azure-file": "kubernetes.io/azure-file", + "gce": "kubernetes.io/gce-pd", + "ibm": "powervs.csi.ibm.com", + "vsphere": "kubernetes.io/vsphere-volume", +} + +// NewPVCDensity holds pvc-density workload +func NewPVCDensity(wh *WorkloadHelper) *cobra.Command { + + var iterations int + var storageProvisioners []string + var claimSize string + var containerImage string + provisioner := "aws" + + cmd := &cobra.Command{ + Use: "pvc-density", + Short: "Runs pvc-density workload", + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, args []string) { + wh.Metadata.Benchmark = cmd.Name() + os.Setenv("JOB_ITERATIONS", fmt.Sprint(iterations)) + os.Setenv("CONTAINER_IMAGE", containerImage) + os.Setenv("CLAIM_SIZE", fmt.Sprint(claimSize)) + + for key := range dynamicStorageProvisioners { + storageProvisioners = append(storageProvisioners, key) + } + re := regexp.MustCompile(`(?sm)^(cinder|azure\-disk|azure\-file|gce|ibm|vsphere|aws)$`) + if !re.MatchString(provisioner) { + log.Fatal(fmt.Errorf("%s does not match one of %s", provisioner, storageProvisioners)) + } + + os.Setenv("STORAGE_PROVISIONER", fmt.Sprint(dynamicStorageProvisioners[provisioner])) + }, + Run: func(cmd *cobra.Command, args []string) { + wh.run(cmd.Name(), MetricsProfileMap[cmd.Name()]) + }, + } + + cmd.Flags().IntVar(&iterations, "iterations", 0, fmt.Sprintf("%v iterations", iterations)) + cmd.Flags().StringVar(&provisioner, "provisioner", provisioner, fmt.Sprintf( + "[%s]", strings.Join(storageProvisioners, " "))) + cmd.Flags().StringVar(&claimSize, "claim-size", "256Mi", "claim-size=256Mi") + cmd.Flags().StringVar(&containerImage, "container-image", "gcr.io/google_containers/pause:3.1", "Container image") + + return cmd +} diff --git a/pkg/workloads/workloads.go b/pkg/workloads/workloads.go index a5f505f5e..0ffd5cabc 100644 --- a/pkg/workloads/workloads.go +++ b/pkg/workloads/workloads.go @@ -11,4 +11,5 @@ var MetricsProfileMap = map[string]string{ "networkpolicy-multitenant": "metrics.yml", "networkpolicy-matchlabels": "metrics.yml", "networkpolicy-matchexpressions": "metrics.yml", + "pvc-density": "metrics.yml", } diff --git a/test/test-ocp.bats b/test/test-ocp.bats index 0e6b5edb1..5f988c228 100755 --- a/test/test-ocp.bats +++ b/test/test-ocp.bats @@ -93,3 +93,11 @@ teardown_file() { run kube-burner ocp networkpolicy-multitenant --iterations 5 ${COMMON_FLAGS} [ "$status" -eq 0 ] } + +@test "pvc-density" { + # Since 'aws' is the chosen storage provisioner, this will only execute successfully if the ocp environment is aws + run kube-burner ocp pvc-density --iterations=2 --provisioner=aws + [ "$status" -eq 0 ] + run check_metric_value clusterMetadata jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement + [ "$status" -eq 0 ] +} From a04ca0238a54117d48da48fd1449c9d024cffc39 Mon Sep 17 00:00:00 2001 From: mleader Date: Tue, 10 Oct 2023 09:46:12 -0400 Subject: [PATCH 2/2] add common flags --- test/test-ocp.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-ocp.bats b/test/test-ocp.bats index 5f988c228..84cddadfd 100755 --- a/test/test-ocp.bats +++ b/test/test-ocp.bats @@ -96,7 +96,7 @@ teardown_file() { @test "pvc-density" { # Since 'aws' is the chosen storage provisioner, this will only execute successfully if the ocp environment is aws - run kube-burner ocp pvc-density --iterations=2 --provisioner=aws + run kube-burner ocp pvc-density --iterations=2 --provisioner=aws ${COMMON_FLAGS} [ "$status" -eq 0 ] run check_metric_value clusterMetadata jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement [ "$status" -eq 0 ]