Kube-burner-ocp

Signed-off-by: Raul Sevilla <[email protected]>

cluster-density.go

@@ -0,0 +1,62 @@
+// Copyright 2022 The Kube-burner Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ocp
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/cloud-bulldozer/kube-burner/pkg/workloads"
+	"github.com/spf13/cobra"
+)
+
+// NewClusterDensity holds cluster-density workload
+func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Command {
+	var iterations, churnPercent int
+	var churn bool
+	var churnDelay, churnDuration time.Duration
+	var churnDeletionStrategy string
+	var podReadyThreshold time.Duration
+	cmd := &cobra.Command{
+		Use:   variant,
+		Short: fmt.Sprintf("Runs %v workload", variant),
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if verifyContainerRegistry(wh.RestConfig) {
+				os.Exit(1)
+			}
+			wh.Metadata.Benchmark = cmd.Name()
+			os.Setenv("JOB_ITERATIONS", fmt.Sprint(iterations))
+			os.Setenv("CHURN", fmt.Sprint(churn))
+			os.Setenv("CHURN_DURATION", fmt.Sprintf("%v", churnDuration))
+			os.Setenv("CHURN_DELAY", fmt.Sprintf("%v", churnDelay))
+			os.Setenv("CHURN_PERCENT", fmt.Sprint(churnPercent))
+			os.Setenv("CHURN_DELETION_STRATEGY", churnDeletionStrategy)
+			os.Setenv("POD_READY_THRESHOLD", fmt.Sprintf("%v", podReadyThreshold))
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			wh.Run(cmd.Name(), MetricsProfileMap[cmd.Name()])
+		},
+	}
+	cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 2*time.Minute, "Pod ready timeout threshold")
+	cmd.Flags().IntVar(&iterations, "iterations", 0, fmt.Sprintf("%v iterations", variant))
+	cmd.Flags().BoolVar(&churn, "churn", true, "Enable churning")
+	cmd.Flags().DurationVar(&churnDuration, "churn-duration", 1*time.Hour, "Churn duration")
+	cmd.Flags().DurationVar(&churnDelay, "churn-delay", 2*time.Minute, "Time to wait between each churn")
+	cmd.Flags().IntVar(&churnPercent, "churn-percent", 10, "Percentage of job iterations that kube-burner will churn each round")
+	cmd.Flags().StringVar(&churnDeletionStrategy, "churn-deletion-strategy", "default", "Churn deletion strategy to use")
+	cmd.MarkFlagRequired("iterations")
+	return cmd
+}

cmd/config/alerts.yml

@@ -0,0 +1,61 @@
+# etcd
+
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[10m:]) > 0.01
+  description: 10 minutes avg. 99th etcd fsync latency on {{$labels.pod}} higher than 10ms. {{$value}}s
+  severity: warning
+
+- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m]))[10m:]) > 0.03
+  description: 10 minutes avg. 99th etcd commit latency on {{$labels.pod}} higher than 30ms. {{$value}}s
+  severity: warning
+
+- expr: rate(etcd_server_leader_changes_seen_total[2m]) > 0
+  description: etcd leader changes observed
+  severity: warning
+
+# API server
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"POST|PUT|DELETE|PATCH", subresource!~"log|exec|portforward|attach|proxy"}[2m])) by (le, resource, verb))[10m:]) > 1
+  description: 10 minutes avg. 99th mutating API call latency for {{$labels.verb}}/{{$labels.resource}} higher than 1 second. {{$value}}s
+  severity: warning
+
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"LIST|GET", subresource!~"log|exec|portforward|attach|proxy", scope="resource"}[2m])) by (le, resource, verb, scope))[5m:]) > 1
+  description: 5 minutes avg. 99th read-only API call latency for {{$labels.verb}}/{{$labels.resource}} in scope {{$labels.scope}} higher than 1 second. {{$value}}s
+  severity: warning
+
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"LIST|GET", subresource!~"log|exec|portforward|attach|proxy", scope="namespace"}[2m])) by (le, resource, verb, scope))[5m:]) > 5
+  description: 5 minutes avg. 99th read-only API call latency for {{$labels.verb}}/{{$labels.resource}} in scope {{$labels.scope}} higher than 5 seconds. {{$value}}s
+  severity: warning
+
+- expr: avg_over_time(histogram_quantile(0.99, sum(irate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb=~"LIST|GET", subresource!~"log|exec|portforward|attach|proxy", scope="cluster"}[2m])) by (le, resource, verb, scope))[5m:]) > 30
+  description: 5 minutes avg. 99th read-only API call latency for {{$labels.verb}}/{{$labels.resource}} in scope {{$labels.scope}} higher than 30 seconds. {{$value}}s
+  severity: warning
+
+# Control plane pods
+- expr: up{apiserver=~"kube-apiserver|openshift-apiserver"} == 0
+  description: "{{$labels.apiserver}} {{$labels.instance}} down"
+  severity: warning
+
+- expr: up{namespace=~"openshift-etcd"} == 0
+  description: "{{$labels.namespace}}/{{$labels.pod}} down"
+  severity: warning
+
+- expr: up{namespace=~"openshift-.*(kube-controller-manager|scheduler|controller-manager|sdn|ovn-kubernetes|dns)"} == 0
+  description: "{{$labels.namespace}}/{{$labels.pod}} down"
+  severity: warning
+
+- expr: up{job=~"crio|kubelet"} == 0
+  description: "{{$labels.node}}/{{$labels.job}} down"
+  severity: warning
+
+- expr: up{job="ovnkube-node"} == 0
+  description: "{{$labels.instance}}/{{$labels.pod}} {{$labels.job}} down"
+  severity: warning
+
+# Service sync latency
+- expr: histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[2m])) by (le)) > 10
+  description: 99th Kubeproxy network programming latency higher than 10 seconds. {{$value}}s 
+  severity: warning
+
+# Prometheus alerts
+- expr: ALERTS{severity="critical", alertstate="firing"} > 0
+  description: Critical prometheus alert. {{$labels.alertname}}
+  severity: error

cmd/config/cluster-density-ms/cluster-density-ms.yml

@@ -0,0 +1,57 @@
+---
+global:
+  gc: {{.GC}}
+  gcMetrics: {{.GC_METRICS}}
+  indexerConfig:
+    esServers: ["{{.ES_SERVER}}"]
+    insecureSkipVerify: true
+    defaultIndex: {{.ES_INDEX}}
+    type: {{.INDEXING_TYPE}}
+  measurements:
+    - name: podLatency
+      thresholds:
+        - conditionType: Ready
+          metric: P99
+          threshold: {{.POD_READY_THRESHOLD}}
+jobs:
+  - name: cluster-density-ms
+    namespace: cluster-density-ms
+    jobIterations: {{.JOB_ITERATIONS}}
+    qps: {{.QPS}}
+    burst: {{.BURST}}
+    namespacedIterations: true
+    podWait: false
+    waitWhenFinished: true
+    preLoadImages: true
+    preLoadPeriod: 10s
+    churn: {{.CHURN}}
+    churnDuration: {{.CHURN_DURATION}}
+    churnPercent: {{.CHURN_PERCENT}}
+    churnDelay: {{.CHURN_DELAY}}
+    churnDeletionStrategy: {{.CHURN_DELETION_STRATEGY}}
+    namespaceLabels:
+      security.openshift.io/scc.podSecurityLabelSync: false
+      pod-security.kubernetes.io/enforce: privileged
+      pod-security.kubernetes.io/audit: privileged
+      pod-security.kubernetes.io/warn: privileged
+    objects:
+
+      - objectTemplate: imagestream.yml
+        replicas: 1
+
+      - objectTemplate: deployment.yml
+        replicas: 4
+        inputVars:
+          podReplicas: 2
+
+      - objectTemplate: service.yml
+        replicas: 2
+
+      - objectTemplate: route.yml
+        replicas: 1
+
+      - objectTemplate: secret.yml
+        replicas: 20
+
+      - objectTemplate: configmap.yml
+        replicas: 10

cmd/config/cluster-density-ms/configmap.yml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{.JobName}}-{{.Replica}}
+data:
+  key1: "{{randAlphaNum 2048}}"

cmd/config/cluster-density-ms/deployment.yml

@@ -0,0 +1,109 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: cluster-density-{{.Replica}}
+spec:
+  replicas: {{.podReplicas}}
+  selector:
+    matchLabels:
+      name: cluster-density-{{.Replica}}
+  template:
+    metadata:
+      labels:
+        name: cluster-density-{{.Replica}}
+        app: cluster-density-ms
+    spec:
+      topologySpreadConstraints:
+      - maxSkew: 1 
+        topologyKey: kubernetes.io/hostname
+        whenUnsatisfiable: ScheduleAnyway 
+        labelSelector: 
+          matchLabels:
+            app: cluster-density-ms
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/worker
+                operator: Exists
+              - key: node-role.kubernetes.io/infra
+                operator: DoesNotExist
+              - key: node-role.kubernetes.io/workload
+                operator: DoesNotExist
+      containers:
+      - args:
+        - sleep
+        - infinity
+        image: registry.k8s.io/pause:3.1
+        resources:
+          requests:
+            memory: "10Mi"
+            cpu: "10m"
+        volumeMounts:
+        - name: secret-1
+          mountPath: /secret1
+        - name: secret-2
+          mountPath: /secret2
+        - name: secret-3
+          mountPath: /secret3
+        - name: secret-4
+          mountPath: /secret4
+        - name: configmap-1
+          mountPath: /configmap1
+        - name: configmap-2
+          mountPath: /configmap2
+        - name: configmap-3
+          mountPath: /configmap3
+        - name: configmap-4
+          mountPath: /configmap4
+        - name: podinfo
+          mountPath: /etc/podlabels
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+        - containerPort: 8443
+          protocol: TCP
+        name: cluster-density
+        env:
+        - name: ENVVAR1
+          value: "{{randAlphaNum 250}}"
+        - name: ENVVAR2
+          value: "{{randAlphaNum 250}}"
+        - name: ENVVAR3
+          value: "{{randAlphaNum 250}}"
+        - name: ENVVAR4
+          value: "{{randAlphaNum 250}}"
+      volumes:
+      - name: secret-1
+        secret:
+          secretName: {{.JobName}}-1
+      - name: secret-2
+        secret:
+          secretName: {{.JobName}}-2
+      - name: secret-3
+        secret:
+          secretName: {{.JobName}}-3
+      - name: secret-4
+        secret:
+          secretName: {{.JobName}}-4
+      - name: configmap-1
+        configMap:
+          name: {{.JobName}}-1
+      - name: configmap-2
+        configMap:
+          name: {{.JobName}}-2
+      - name: configmap-3
+        configMap:
+          name: {{.JobName}}-3
+      - name: configmap-4
+        configMap:
+          name: {{.JobName}}-4
+      - name: podinfo
+        downwardAPI:
+          items:
+            - path: "labels"
+              fieldRef:
+                fieldPath: metadata.labels

cmd/config/cluster-density-ms/imagestream.yml

@@ -0,0 +1,5 @@
+---
+kind: ImageStream
+apiVersion: image.openshift.io/v1
+metadata:
+  name: cluster-density-{{.Replica}}

cmd/config/cluster-density-ms/route.yml

@@ -0,0 +1,11 @@
+---
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+  name: cluster-density-{{.Replica}}
+spec:
+  to:
+    kind: Service
+    name: cluster-density-{{.Replica}}
+  tls:
+    termination: edge

cmd/config/cluster-density-ms/secret.yml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{.JobName}}-{{.Replica}}
+data:
+  top-secret: "{{randAlphaNum 2048}}"

cmd/config/cluster-density-ms/service.yml

@@ -0,0 +1,18 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: cluster-density-{{.Replica}}
+spec:
+  selector:
+    name: cluster-density-{{.Replica}}
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    targetPort: 8080
+  - name: https
+    protocol: TCP
+    port: 443
+    targetPort: 8443
+  type: ClusterIP

cmd/config/cluster-density-v2/build.yml

@@ -0,0 +1,28 @@
+---
+kind: Build
+apiVersion: build.openshift.io/v1
+metadata:
+  name: {{.JobName}}-{{.Replica}}
+spec:
+  resources:
+    requests:
+      cpu: 70m
+      memory: "10Mi"
+  nodeSelector:
+    node-role.kubernetes.io/worker: ""
+  serviceAccount: builder
+  source:
+    dockerfile: |-
+      FROM registry.fedoraproject.org/fedora-minimal:latest
+      RUN touch $(date +%s)
+    type: Dockerfile
+  strategy:
+    dockerStrategy:
+      from:
+        kind: DockerImage
+        name: registry.fedoraproject.org/fedora-minimal:latest
+    type: Source
+  output:
+    to:
+      kind: ImageStreamTag
+      name: cluster-density-{{.Replica}}:latest