External Metrics w/E2E testing, prometheus, & KIND.

kubernetes · Aug 28, 2023 · 034632a · 034632a
1 parent 3c9aecb
commit 034632a
Show file tree

Hide file tree

Showing 33 changed files with 3,078 additions and 28 deletions.
diff --git a/vertical-pod-autoscaler/hack/deploy-for-e2e-locally.sh b/vertical-pod-autoscaler/hack/deploy-for-e2e-locally.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+
+function print_help {
+  echo "ERROR! Usage: deploy-for-e2e-locally.sh [suite]*"
+  echo "<suite> should be one of:"
+  echo " - recommender"
+  echo " - recommender-externalmetrics"
+}
+
+if [ $# -eq 0 ]; then
+  print_help
+  exit 1
+fi
+
+if [ $# -gt 1 ]; then
+  print_help
+  exit 1
+fi
+
+SUITE=$1
+
+case ${SUITE} in
+  recommender|recommender-externalmetrics)
+    COMPONENTS="${SUITE}"
+    ;;
+  *)
+    print_help
+    exit 1
+    ;;
+esac
+
+# Local KIND-hosted registry
+export REGISTRY=${REGISTRY:-localhost:5001}
+export TAG=${TAG:-latest}
+
+rm -f ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.yaml
+patch -c ${SCRIPT_ROOT}/deploy/vpa-rbac.yaml -i ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.diff -o ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.yaml
+kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/vpa-rbac.yaml
+# Other-versioned CRDs are irrelevant as we're running a modern-ish cluster.
+kubectl apply -f ${SCRIPT_ROOT}/deploy/vpa-v1-crd-gen.yaml
+kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/k8s-metrics-server.yaml
+
+for i in ${COMPONENTS}; do
+  if [ $i == recommender-externalmetrics ] ; then
+    i=recommender
+  fi
+  ALL_ARCHITECTURES=amd64 make --directory ${SCRIPT_ROOT}/pkg/${i} release REGISTRY=${REGISTRY} TAG=${TAG}
+  kind load docker-image ${REGISTRY}/vpa-${i}-amd64:${TAG}
+done
+
+
+for i in ${COMPONENTS}; do
+  if [ $i == recommender-externalmetrics ] ; then
+     kubectl delete namespace monitoring --ignore-not-found=true
+     kubectl create namespace monitoring
+     kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/prometheus.yaml
+     kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/prometheus-adapter.yaml
+     kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/metrics-pump.yaml
+     kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/${i}-deployment.yaml
+  else
+     REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh  ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl apply -f -
+  fi
+done
diff --git a/vertical-pod-autoscaler/hack/e2e/Dockerfile.externalmetrics-writer b/vertical-pod-autoscaler/hack/e2e/Dockerfile.externalmetrics-writer
@@ -0,0 +1,18 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.10-slim
+RUN pip3 install kubernetes argparse requests
+
+COPY emit-metrics.py /
diff --git a/vertical-pod-autoscaler/hack/e2e/k8s-metrics-server.yaml b/vertical-pod-autoscaler/hack/e2e/k8s-metrics-server.yaml
@@ -0,0 +1,244 @@
+---
+# Source: metrics-server/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-metrics-server
+  namespace: default
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+---
+# Source: metrics-server/templates/clusterrole-aggregated-reader.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server-aggregated-reader
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+rules:
+  - apiGroups:
+      - metrics.k8s.io
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Source: metrics-server/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:local-metrics-server
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+rules:
+  - apiGroups:
+    - ""
+    resources:
+    - nodes/metrics
+    verbs:
+    - get
+  - apiGroups:
+    - ""
+    resources:
+      - pods
+      - nodes
+      - namespaces
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Source: metrics-server/templates/clusterrolebinding-auth-delegator.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-metrics-server:system:auth-delegator
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: local-metrics-server
+    namespace: default
+---
+# Source: metrics-server/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:local-metrics-server
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:local-metrics-server
+subjects:
+  - kind: ServiceAccount
+    name: local-metrics-server
+    namespace: default
+---
+# Source: metrics-server/templates/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: local-metrics-server-auth-reader
+  namespace: default
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: local-metrics-server
+    namespace: default
+---
+# Source: metrics-server/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: local-metrics-server
+  namespace: default
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  type: ClusterIP
+  ports:
+    - name: https
+      port: 443
+      protocol: TCP
+      targetPort: https
+  selector:
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+---
+# Source: metrics-server/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-metrics-server
+  namespace: default
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: metrics-server
+      app.kubernetes.io/instance: local-metrics-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: metrics-server
+        app.kubernetes.io/instance: local-metrics-server
+    spec:
+      schedulerName: 
+      serviceAccountName: local-metrics-server
+      priorityClassName: "system-cluster-critical"
+      containers:
+        - name: metrics-server
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 1000
+          image: registry.k8s.io/metrics-server/metrics-server:v0.6.2
+          imagePullPolicy: IfNotPresent
+          args:
+            - --secure-port=10250
+            - --cert-dir=/tmp
+            - --kubelet-insecure-tls
+            - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+            - --kubelet-use-node-status-port
+            - --metric-resolution=15s
+          ports:
+          - name: https
+            protocol: TCP
+            containerPort: 10250
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /livez
+              port: https
+              scheme: HTTPS
+            initialDelaySeconds: 0
+            periodSeconds: 10
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /readyz
+              port: https
+              scheme: HTTPS
+            initialDelaySeconds: 20
+            periodSeconds: 10
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+---
+# Source: metrics-server/templates/apiservice.yaml
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+  labels:
+    helm.sh/chart: metrics-server-3.8.3
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/instance: local-metrics-server
+    app.kubernetes.io/version: "0.6.2"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: local-metrics-server
+    namespace: default
+    port: 443
+  version: v1beta1
+  versionPriority: 100