From 6a1263513ab65d3d8155420735c8af2e69a6578c Mon Sep 17 00:00:00 2001 From: Ang Gao Date: Thu, 15 Oct 2020 12:14:59 +0100 Subject: [PATCH 1/4] Add Argo Rollouts e2e test Signed-off-by: Ang Gao --- tests/README.md | 6 + tests/package.json | 4 +- tests/scalers/argo-rollouts.test.ts | 243 ++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 tests/scalers/argo-rollouts.test.ts diff --git a/tests/README.md b/tests/README.md index 91a440879d4..df415189304 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,6 +11,12 @@ npm install npm test --verbose ``` +### Run on test file: + +``` +npx ava scalers/prometheus.test.ts +``` + ## E2E test setup The test script will run 3 phases: diff --git a/tests/package.json b/tests/package.json index 0d657bd12c7..1d4d504fc12 100644 --- a/tests/package.json +++ b/tests/package.json @@ -11,7 +11,9 @@ "ts-node/register" ] }, - "scripts": {}, + "scripts": { + "test": "ava" + }, "dependencies": { "@kubernetes/client-node": "^0.10.3", "@types/async": "^3.0.3", diff --git a/tests/scalers/argo-rollouts.test.ts b/tests/scalers/argo-rollouts.test.ts new file mode 100644 index 00000000000..a8636fcf7f0 --- /dev/null +++ b/tests/scalers/argo-rollouts.test.ts @@ -0,0 +1,243 @@ +import * as async from 'async' +import * as fs from 'fs' +import * as sh from 'shelljs' +import * as tmp from 'tmp' +import test from 'ava' + +const testNamespace = 'argo-rollouts-test' +const prometheusNamespace = 'monitoring' +const prometheusDeploymentFile = 'scalers/prometheus-deployment.yaml' +const argoRolloutsNamespace = 'argo-rollouts' +const argoRolloutsYamlFile = tmp.fileSync() + +test.before(t => { + // install prometheus + sh.exec(`kubectl create namespace ${prometheusNamespace}`) + t.is(0, sh.exec(`kubectl apply --namespace ${prometheusNamespace} -f ${prometheusDeploymentFile}`).code, 'creating a Prometheus deployment should work.') + // wait for prometheus to load + let prometheusReadyReplicaCount = '0' + for (let i = 0; i < 30; i++) { + prometheusReadyReplicaCount = sh.exec(`kubectl get deploy/prometheus-server -n ${prometheusNamespace} -o jsonpath='{.status.readyReplicas}'`).stdout + if (prometheusReadyReplicaCount != '1') { + sh.exec('sleep 2s') + } + } + t.is('1', prometheusReadyReplicaCount, 'Prometheus is not in a ready state') + + // install argo-rollouts + sh.exec(`kubectl create namespace ${argoRolloutsNamespace}`) + const argoRolloutsYaml = sh.exec(`curl -L https://raw.githubusercontent.com/argoproj/argo-rollouts/stable/manifests/install.yaml`).stdout + fs.writeFileSync(argoRolloutsYamlFile.name, argoRolloutsYaml) + t.is( + 0, + sh.exec(`kubectl apply -f ${argoRolloutsYamlFile.name} --namespace ${argoRolloutsNamespace}`).code, + 'Deploying argo-rollouts should work.' + ) + + sh.config.silent = true + // create rollouts - there are two rollouts - both using the same image but one rollout + // is directly tied to the KEDA HPA while the other is isolated that can be used for metrics + // even when the KEDA deployment is at zero - the service points to both rollouts + const tmpFile = tmp.fileSync() + fs.writeFileSync(tmpFile.name, rollout.replace('{{PROMETHEUS_NAMESPACE}}', prometheusNamespace)) + sh.exec(`kubectl create namespace ${testNamespace}`) + t.is( + 0, + sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, + 'creating a rollouts should work.' + ) + for (let i = 0; i < 10; i++) { + const readyReplicaCount = sh.exec(`kubectl get rollouts.argoproj.io/test-app --namespace ${testNamespace} -o jsonpath="{.status.readyReplicas}`).stdout + if (readyReplicaCount != '1') { + sh.exec('sleep 2s') + } + } +}) + +test.serial('Rollouts should have 0 replicas on start', t => { + const replicaCount = sh.exec( + `kubectl get rollouts.argoproj.io/keda-test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` + ).stdout + t.is(replicaCount, '0', 'replica count should start out as 0') +}) + +test.serial(`Rollouts should scale to 5 (the max) with HTTP Requests exceeding in the rate then back to 0`, t => { + // generate a large number of HTTP requests (using Apache Bench) that will take some time + // so prometheus has some time to scrape it + const tmpFile = tmp.fileSync() + fs.writeFileSync(tmpFile.name, generateRequestsYaml.replace('{{NAMESPACE}}', testNamespace)) + t.is( + 0, + sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, + 'creating job should work.' + ) + + t.is( + '2', + sh.exec( + `kubectl get rollouts.argoproj.io/test-app --namespace ${testNamespace} -o jsonpath="{.status.readyReplicas}"` + ).stdout, + 'There should be 2 replica for the test-app rollout' + ) + + // keda based rollout should start scaling up with http requests issued + let replicaCount = '0' + for (let i = 0; i < 60 && replicaCount !== '5'; i++) { + t.log(`Waited ${5 * i} seconds for prometheus-based rollout to scale up`) + const jobLogs = sh.exec(`kubectl logs -l job-name=generate-requests -n ${testNamespace}`).stdout + t.log(`Logs from the generate requests: ${jobLogs}`) + + replicaCount = sh.exec( + `kubectl get rollouts.argoproj.io/keda-test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` + ).stdout + if (replicaCount !== '5') { + sh.exec('sleep 5s') + } + } + + t.is('5', replicaCount, 'Replica count should be maxed at 5') + + for (let i = 0; i < 50 && replicaCount !== '0'; i++) { + replicaCount = sh.exec( + `kubectl get rollouts.argoproj.io/keda-test-app --namespace ${testNamespace} -o jsonpath="{.spec.replicas}"` + ).stdout + if (replicaCount !== '0') { + sh.exec('sleep 5s') + } + } + + t.is('0', replicaCount, 'Replica count should be 0 after 3 minutes') +}) + +test.after.always.cb('clean up argo-rollouts testing deployment', t => { + const resources = [ + 'scaledobject.keda.sh/prometheus-scaledobject', + 'rollouts.argoproj.io/test-app', + 'rollouts.argoproj.io/keda-test-app', + 'service/test-app', + 'job/generate-requests', + ] + + for (const resource of resources) { + sh.exec(`kubectl delete ${resource} --namespace ${testNamespace}`) + } + sh.exec(`kubectl delete namespace ${testNamespace}`) + + // uninstall prometheus + sh.exec(`kubectl delete --namespace ${prometheusNamespace} -f ${prometheusDeploymentFile}`) + sh.exec(`kubectl delete namespace ${prometheusNamespace}`) + + // uninstall argo-rollouts + sh.exec(`kubectl delete --namespace ${argoRolloutsNamespace} -f ${argoRolloutsYamlFile}`) + sh.exec(`kubectl delete namespace ${argoRolloutsNamespace}`) + + t.end() +}) + +const rollout = `apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + labels: + app: test-app + name: test-app +spec: + replicas: 2 + strategy: + canary: + steps: + - setWeight: 50 + - pause: {duration: 10} + selector: + matchLabels: + app: test-app + template: + metadata: + labels: + app: test-app + type: keda-testing + spec: + containers: + - name: prom-test-app + image: tbickford/simple-web-app-prometheus:a13ade9 + imagePullPolicy: IfNotPresent +--- +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + labels: + app: keda-test-app + name: keda-test-app +spec: + replicas: 0 + strategy: + canary: + steps: + - setWeight: 50 + - pause: {duration: 10} + selector: + matchLabels: + app: keda-test-app + template: + metadata: + labels: + app: keda-test-app + type: keda-testing + spec: + containers: + - name: prom-test-app + image: tbickford/simple-web-app-prometheus:a13ade9 + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: test-app + annotations: + prometheus.io/scrape: "true" + name: test-app +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + type: keda-testing +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: prometheus-scaledobject +spec: + scaleTargetRef: + apiVersion: argoproj.io/v1alpha1 + kind: Rollout + name: keda-test-app + minReplicaCount: 0 + maxReplicaCount: 5 + pollingInterval: 5 + cooldownPeriod: 10 + triggers: + - type: prometheus + metadata: + serverAddress: http://prometheus-server.{{PROMETHEUS_NAMESPACE}}.svc + metricName: http_requests_total + threshold: '100' + query: sum(rate(http_requests_total{app="test-app"}[2m]))` + +const generateRequestsYaml = `apiVersion: batch/v1 +kind: Job +metadata: + name: generate-requests +spec: + template: + spec: + containers: + - image: jordi/ab + name: test + command: ["/bin/sh"] + args: ["-c", "for i in $(seq 1 60);do echo $i;ab -c 5 -n 1000 -v 2 http://test-app.{{NAMESPACE}}.svc/;sleep 1;done"] + restartPolicy: Never + activeDeadlineSeconds: 120 + backoffLimit: 2` From 921128d9e8f5b59a5c3c5a759976846626df3e30 Mon Sep 17 00:00:00 2001 From: Ang Gao Date: Thu, 15 Oct 2020 12:18:44 +0100 Subject: [PATCH 2/4] Update Changelog Signed-off-by: Ang Gao --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1f726c8127..9634168cde5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - Add Metrics API Scaler ([#1026](https://github.com/kedacore/keda/pull/1026)) - Add cpu/memory Scaler ([#1215](https://github.com/kedacore/keda/pull/1215)) - Add Scaling Strategy for ScaledJob ([#1227](https://github.com/kedacore/keda/pull/1227)) +- Add Argo Rollouts e2e test ([#1234](https://github.com/kedacore/keda/issues/1234)) ### Improvements From 732d169237f9fbc8d38126760bffd11a6c4c1f4a Mon Sep 17 00:00:00 2001 From: Ang Gao Date: Thu, 15 Oct 2020 12:20:52 +0100 Subject: [PATCH 3/4] Fix typo Signed-off-by: Ang Gao --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index df415189304..cd13587a7f0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,7 +11,7 @@ npm install npm test --verbose ``` -### Run on test file: +### Run one test file: ``` npx ava scalers/prometheus.test.ts From 8bf995c69d963de21cd8ffcebb6fae5a3b926cc7 Mon Sep 17 00:00:00 2001 From: Ang Gao Date: Thu, 15 Oct 2020 12:54:45 +0100 Subject: [PATCH 4/4] Move issue to other section in Changelog Signed-off-by: Ang Gao --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9634168cde5..2c6e02ce87f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,6 @@ - Add Metrics API Scaler ([#1026](https://github.com/kedacore/keda/pull/1026)) - Add cpu/memory Scaler ([#1215](https://github.com/kedacore/keda/pull/1215)) - Add Scaling Strategy for ScaledJob ([#1227](https://github.com/kedacore/keda/pull/1227)) -- Add Argo Rollouts e2e test ([#1234](https://github.com/kedacore/keda/issues/1234)) ### Improvements @@ -60,6 +59,7 @@ - Change API optional structs to pointers to conform with k8s guide ([#1170](https://github.com/kedacore/keda/issues/1170)) - Update Operator SDK and k8s deps ([#1007](https://github.com/kedacore/keda/pull/1007),[#870](https://github.com/kedacore/keda/issues/870),[#1180](https://github.com/kedacore/keda/pull/1180)) - Change Metrics Server image name from `keda-metrics-adapter` to `keda-metrics-apiserver` ([#1105](https://github.com/kedacore/keda/issues/1105)) +- Add Argo Rollouts e2e test ([#1234](https://github.com/kedacore/keda/issues/1234)) ## v1.5.0