diff --git a/Makefile b/Makefile index 0c85c0bee4..b0531967c3 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ chainsaw-integration-test: kubectl apply -f ./lifecycle-operator/config/crd/bases # chainsaw test --test-dir ./test/chainsaw/integration/ chainsaw test --test-dir ./test/chainsaw/testmetrics/ -# chainsaw test --test-dir ./test/chainsaw/testanalysis/ + chainsaw test --test-dir ./test/chainsaw/testanalysis/ chainsaw test --test-dir ./test/chainsaw/testcertificate/ .PHONY: chainsaw-integration-test-local #these tests should run on a real cluster! @@ -73,7 +73,7 @@ chainsaw-integration-test-local: kubectl apply -f ./lifecycle-operator/config/crd/bases # chainsaw test --test-dir ./test/chainsaw/integration/ --config ./.chainsaw-local.yaml chainsaw test --test-dir ./test/chainsaw/testmetrics/ --config ./.chainsaw-local.yaml -# chainsaw test --test-dir ./test/chainsaw/testanalysis/ --config ./.chainsaw-local.yaml + chainsaw test --test-dir ./test/chainsaw/testanalysis/ --config ./.chainsaw-local.yaml chainsaw test --test-dir ./test/chainsaw/testcertificate/ --config ./.chainsaw-local.yaml .PHONY: chainsaw-integration-test-scheduling-gates #these tests should run on a real cluster! diff --git a/test/chainsaw/testanalysis/analysis-controller-existing-status/00-install.yaml b/test/chainsaw/testanalysis/analysis-controller-existing-status/00-install.yaml new file mode 100644 index 0000000000..0f8d08a5f9 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-existing-status/00-install.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: testy +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: ready + namespace: testy +spec: + provider: + name: my-mocked-provider + namespace: testy + query: 'sum(kube_pod_container_status_ready{namespace="{{.ns}}"})' +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisDefinition +metadata: + name: ed-my-proj-dev-svc1 + namespace: testy +spec: + objectives: + - analysisValueTemplateRef: + name: ready + namespace: testy + target: + failure: + lessThan: + fixedValue: 2 + warning: + lessThan: + fixedValue: 3 + weight: 1 + keyObjective: false + totalScore: + passPercentage: 90 + warningPercentage: 75 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample + namespace: testy +spec: + timeframe: + from: 2023-09-14T07:33:19Z + to: 2023-09-14T08:33:19Z + args: + "ns": "keptn-system" + analysisDefinition: + name: ed-my-proj-dev-svc1 + namespace: testy +status: + storedValues: + my-provider-query-1: + objectiveReference: + name: objective-template-1 + value: 1 + errMsg: "" + +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: KeptnMetricsProvider +metadata: + name: my-mocked-provider + namespace: testy +spec: + type: prometheus + targetServer: "http://mockserver.testy.svc.cluster.local:1080" diff --git a/test/chainsaw/testanalysis/analysis-controller-existing-status/01-install.yaml b/test/chainsaw/testanalysis/analysis-controller-existing-status/01-install.yaml new file mode 100644 index 0000000000..cd82cd0761 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-existing-status/01-install.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: Service +metadata: + name: mockserver + namespace: testy +spec: + ports: + - name: serviceport + port: 1080 + protocol: TCP + targetPort: serviceport + selector: + app: mockserver + sessionAffinity: None + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: mockserver + name: mockserver + namespace: testy +spec: + replicas: 1 + selector: + matchLabels: + app: mockserver + template: + metadata: + labels: + app: mockserver + name: mockserver + spec: + containers: + - env: + - name: MOCKSERVER_LOG_LEVEL + value: INFO + - name: SERVER_PORT + value: "1080" + image: mockserver/mockserver:mockserver-5.13.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + name: mockserver + ports: + - containerPort: 1080 + name: serviceport + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 2 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + volumeMounts: + - mountPath: /config + name: config-volume + - mountPath: /libs + name: libs-volume + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: config-volume + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: libs-volume +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: mockserver-config + namespace: testy +data: + initializerJson.json: |- + [ + { + "httpRequest": { + "path": "/api/v1/query_range", + "method": "POST" + }, + "httpResponse": { + "body": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "metric-name", + "job": "", + "instance": "" + }, + "values": [[1669714193.275, "4"]] + } + ] + } + }, + "statusCode": 200 + } + } + ] + mockserver.properties: |- + ############################### + # MockServer & Proxy Settings # + ############################### + # Socket & Port Settings + # socket timeout in milliseconds (default 120000) + mockserver.maxSocketTimeout=120000 + # Certificate Generation + # dynamically generated CA key pair (if they don't already exist in + specified directory) + mockserver.dynamicallyCreateCertificateAuthorityCertificate=true + # save dynamically generated CA key pair in working directory + mockserver.directoryToSaveDynamicSSLCertificate=. + # certificate domain name (default "localhost") + mockserver.sslCertificateDomainName=localhost + # comma separated list of ip addresses for Subject Alternative Name domain + names (default empty list) + mockserver.sslSubjectAlternativeNameDomains=www.example.com,www.another.com + # comma separated list of ip addresses for Subject Alternative Name ips + (default empty list) + mockserver.sslSubjectAlternativeNameIps=127.0.0.1 + # CORS + # enable CORS for MockServer REST API + mockserver.enableCORSForAPI=true + # enable CORS for all responses + mockserver.enableCORSForAllResponses=true + # Json Initialization + mockserver.initializationJsonPath=/config/initializerJson.json diff --git a/test/chainsaw/testanalysis/analysis-controller-existing-status/assert-1.yaml b/test/chainsaw/testanalysis/analysis-controller-existing-status/assert-1.yaml new file mode 100755 index 0000000000..3351d60ee3 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-existing-status/assert-1.yaml @@ -0,0 +1,14 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample + namespace: testy +spec: + analysisDefinition: + name: ed-my-proj-dev-svc1 +status: + storedValues: + ready-testy: + objectiveReference: + name: ready + namespace: testy diff --git a/test/chainsaw/testanalysis/analysis-controller-existing-status/assert-2.yaml b/test/chainsaw/testanalysis/analysis-controller-existing-status/assert-2.yaml new file mode 100755 index 0000000000..2acaa8dc7a --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-existing-status/assert-2.yaml @@ -0,0 +1,12 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample + namespace: testy +spec: + analysisDefinition: + name: ed-my-proj-dev-svc1 +status: + pass: true + # yamllint disable-line rule:line-length + raw: '{"objectiveResults":[{"result":{"failResult":{"operator":{"lessThan":{"fixedValue":"2"}},"fulfilled":false},"warnResult":{"operator":{"lessThan":{"fixedValue":"3"}},"fulfilled":false},"warning":false,"pass":true},"objective":{"analysisValueTemplateRef":{"name":"ready","namespace":"testy"},"target":{"failure":{"lessThan":{"fixedValue":"2"}},"warning":{"lessThan":{"fixedValue":"3"}}},"weight":1},"value":4,"query":"sum(kube_pod_container_status_ready{namespace=\"keptn-system\"})","score":1}],"totalScore":1,"maximumScore":1,"pass":true,"warning":false}' diff --git a/test/chainsaw/testanalysis/analysis-controller-existing-status/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysis-controller-existing-status/chainsaw-test.yaml new file mode 100755 index 0000000000..fc40c2c995 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-existing-status/chainsaw-test.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysis-controller-existing-status +spec: + steps: + - name: step-00 + try: + - apply: + file: 00-install.yaml + - assert: + file: assert-1.yaml + catch: + - script: + content: kubectl logs -l control-plane=metrics-operator -n keptn-system + - script: + content: kubectl describe Analysis -n testy + - name: step-01 + try: + - apply: + file: 01-install.yaml + - assert: + file: assert-2.yaml + catch: + - script: + content: kubectl logs -l control-plane=metrics-operator -n keptn-system + - script: + content: kubectl describe Analysis -n testy diff --git a/test/chainsaw/testanalysis/analysis-controller-multiple-providers/assert-1.yaml b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/assert-1.yaml new file mode 100755 index 0000000000..e27c4d9bd4 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/assert-1.yaml @@ -0,0 +1,11 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample +spec: + analysisDefinition: + name: ed-my-proj-dev-svc1 +status: + pass: true + # yamllint disable-line rule:line-length + raw: '{"objectiveResults":[{"result":{"failResult":{"operator":{"lessThan":{"fixedValue":"5"}},"fulfilled":false},"warnResult":{"operator":{"lessThan":{"fixedValue":"4"}},"fulfilled":false},"warning":false,"pass":true},"objective":{"analysisValueTemplateRef":{"name":"value-1"},"target":{"failure":{"lessThan":{"fixedValue":"5"}},"warning":{"lessThan":{"fixedValue":"4"}}},"weight":2},"value":11,"query":"query-1","score":2},{"result":{"failResult":{"operator":{"greaterThan":{"fixedValue":"20"}},"fulfilled":false},"warnResult":{"operator":{"greaterThan":{"fixedValue":"15"}},"fulfilled":true},"warning":true,"pass":false},"objective":{"analysisValueTemplateRef":{"name":"value-2"},"target":{"failure":{"greaterThan":{"fixedValue":"20"}},"warning":{"greaterThan":{"fixedValue":"15"}}},"weight":1},"value":20,"query":"query-2","score":0.5},{"result":{"failResult":{"operator":{"notInRange":{"lowBound":"25","highBound":"35"}},"fulfilled":false},"warnResult":{"operator":{},"fulfilled":false},"warning":false,"pass":true},"objective":{"analysisValueTemplateRef":{"name":"value-3"},"target":{"failure":{"notInRange":{"lowBound":"25","highBound":"35"}}},"weight":1},"value":30,"query":"query-3","score":1}],"totalScore":3.5,"maximumScore":4,"pass":true,"warning":false}' diff --git a/test/chainsaw/testanalysis/analysis-controller-multiple-providers/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/chainsaw-test.yaml new file mode 100755 index 0000000000..54545a86d0 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/chainsaw-test.yaml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysis-controller-multiple-providers +spec: + steps: + - name: step-00 + try: + - script: + content: | + envsubst < mock-server.yaml | kubectl apply -f - + - script: + content: | + envsubst < install.yaml | kubectl apply -f - -n $NAMESPACE + - name: step-01 + try: + - assert: + file: assert-1.yaml + catch: + - script: + content: kubectl logs -l control-plane=metrics-operator -n keptn-system + - script: + content: kubectl describe Analysis -n $NAMESPACE diff --git a/test/chainsaw/testanalysis/analysis-controller-multiple-providers/install.yaml b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/install.yaml new file mode 100644 index 0000000000..073f549287 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/install.yaml @@ -0,0 +1,104 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: value-1 +spec: + provider: + name: my-first-mocked-provider + query: 'query-1' +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: value-2 +spec: + provider: + name: my-second-mocked-provider + query: 'query-2' +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: value-3 +spec: + provider: + name: my-third-mocked-provider + query: 'query-3' +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisDefinition +metadata: + name: ed-my-proj-dev-svc1 +spec: + objectives: + - analysisValueTemplateRef: + name: value-1 + target: + failure: + lessThan: + fixedValue: 5 + warning: + lessThan: + fixedValue: 4 + weight: 2 + keyObjective: false + - analysisValueTemplateRef: + name: value-2 + target: + failure: + greaterThan: + fixedValue: 20 + warning: + greaterThan: + fixedValue: 15 + weight: 1 + keyObjective: false + - analysisValueTemplateRef: + name: value-3 + target: + failure: + notInRange: + lowBound: 25 + highBound: 35 + weight: 1 + keyObjective: false + totalScore: + passPercentage: 75 + warningPercentage: 50 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample +spec: + timeframe: + from: 2023-09-14T07:33:19Z + to: 2023-09-14T08:33:19Z + args: + "ns": "keptn-system" + analysisDefinition: + name: ed-my-proj-dev-svc1 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: KeptnMetricsProvider +metadata: + name: my-first-mocked-provider +spec: + type: prometheus + targetServer: "http://mockserver.$NAMESPACE.svc.cluster.local:1080" +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: KeptnMetricsProvider +metadata: + name: my-second-mocked-provider +spec: + type: prometheus + targetServer: "http://mockserver.$NAMESPACE.svc.cluster.local:1080" +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: KeptnMetricsProvider +metadata: + name: my-third-mocked-provider +spec: + type: prometheus + targetServer: "http://mockserver.$NAMESPACE.svc.cluster.local:1080" diff --git a/test/chainsaw/testanalysis/analysis-controller-multiple-providers/mock-server.yaml b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/mock-server.yaml new file mode 100644 index 0000000000..d437e62b58 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-multiple-providers/mock-server.yaml @@ -0,0 +1,211 @@ +apiVersion: v1 +kind: Service +metadata: + name: mockserver + namespace: $NAMESPACE +spec: + ports: + - name: serviceport + port: 1080 + protocol: TCP + targetPort: serviceport + selector: + app: mockserver + sessionAffinity: None + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: mockserver + name: mockserver + namespace: $NAMESPACE +spec: + replicas: 1 + selector: + matchLabels: + app: mockserver + template: + metadata: + labels: + app: mockserver + name: mockserver + spec: + containers: + - env: + - name: MOCKSERVER_LOG_LEVEL + value: INFO + - name: SERVER_PORT + value: "1080" + image: mockserver/mockserver:mockserver-5.13.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + name: mockserver + ports: + - containerPort: 1080 + name: serviceport + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 2 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + volumeMounts: + - mountPath: /config + name: config-volume + - mountPath: /libs + name: libs-volume + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: config-volume + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: libs-volume +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: mockserver-config + namespace: $NAMESPACE +data: + initializerJson.json: |- + [ + { + "httpRequest": { + "path": "/api/v1/query_range", + "method": "POST", + "body" : { + "type" : "PARAMETERS", + "parameters" : { + "query" : [ "query-1" ] + } + } + }, + "httpResponse": { + "body": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "metric-name", + "job": "", + "instance": "" + }, + "values": [[1669714193.275, "11"]] + } + ] + } + }, + "statusCode": 200 + } + }, + { + "httpRequest": { + "path": "/api/v1/query_range", + "method": "POST", + "body" : { + "type" : "PARAMETERS", + "parameters" : { + "query" : [ "query-2" ] + } + } + }, + "httpResponse": { + "body": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "metric-name", + "job": "", + "instance": "" + }, + "values": [[1669714193.275, "20"]] + } + ] + } + }, + "statusCode": 200 + } + }, + { + "httpRequest": { + "path": "/api/v1/query_range", + "method": "POST", + "body" : { + "type" : "PARAMETERS", + "parameters" : { + "query" : [ "query-3" ] + } + } + }, + "httpResponse": { + "body": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "metric-name", + "job": "", + "instance": "" + }, + "values": [[1669714193.275, "30"]] + } + ] + } + }, + "statusCode": 200 + } + } + ] + mockserver.properties: |- + ############################### + # MockServer & Proxy Settings # + ############################### + # Socket & Port Settings + # socket timeout in milliseconds (default 120000) + mockserver.maxSocketTimeout=120000 + # Certificate Generation + # dynamically generated CA key pair (if they don't already exist in + specified directory) + mockserver.dynamicallyCreateCertificateAuthorityCertificate=true + # save dynamically generated CA key pair in working directory + mockserver.directoryToSaveDynamicSSLCertificate=. + # certificate domain name (default "localhost") + mockserver.sslCertificateDomainName=localhost + # comma separated list of ip addresses for Subject Alternative Name domain + names (default empty list) + mockserver.sslSubjectAlternativeNameDomains=www.example.com,www.another.com + # comma separated list of ip addresses for Subject Alternative Name ips + (default empty list) + mockserver.sslSubjectAlternativeNameIps=127.0.0.1 + # CORS + # enable CORS for MockServer REST API + mockserver.enableCORSForAPI=true + # enable CORS for all responses + mockserver.enableCORSForAllResponses=true + # Json Initialization + mockserver.initializationJsonPath=/config/initializerJson.json diff --git a/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/assert-1.yaml b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/assert-1.yaml new file mode 100755 index 0000000000..8c8c27bdbf --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/assert-1.yaml @@ -0,0 +1,12 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample +spec: + analysisDefinition: + name: ed-my-proj-dev-svc1 +status: + pass: true + # yamllint disable-line rule:line-length + raw: '{"objectiveResults":[{"result":{"failResult":{"operator":{"lessThan":{"fixedValue":"2"}},"fulfilled":false},"warnResult":{"operator":{"lessThan":{"fixedValue":"3"}},"fulfilled":false},"warning":false,"pass":true},"objective":{"analysisValueTemplateRef":{"name":"ready"},"target":{"failure":{"lessThan":{"fixedValue":"2"}},"warning":{"lessThan":{"fixedValue":"3"}}},"weight":1},"value":4,"query":"sum(kube_pod_container_status_ready{namespace=\"keptn-system\"})","score":1}],"totalScore":1,"maximumScore":1,"pass":true,"warning":false}' + state: Completed diff --git a/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/chainsaw-test.yaml new file mode 100755 index 0000000000..aad617dcf0 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/chainsaw-test.yaml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysis-controller-with-duration-timeframe +spec: + steps: + - name: step-00 + try: + - script: + content: | + kubectl apply -f mock-server.yaml -n $NAMESPACE + - script: + content: | + envsubst < install.yaml | kubectl apply -f - -n $NAMESPACE + - name: step-01 + try: + - assert: + file: assert-1.yaml + catch: + - script: + content: kubectl logs -l control-plane=metrics-operator -n keptn-system + - script: + content: kubectl describe Analysis -n $NAMESPACE diff --git a/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/install.yaml b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/install.yaml new file mode 100644 index 0000000000..6d82e7abac --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/install.yaml @@ -0,0 +1,49 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: ready +spec: + provider: + name: my-mocked-provider + query: 'sum(kube_pod_container_status_ready{namespace="{{.ns}}"})' +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisDefinition +metadata: + name: ed-my-proj-dev-svc1 +spec: + objectives: + - analysisValueTemplateRef: + name: ready + target: + failure: + lessThan: + fixedValue: 2 + warning: + lessThan: + fixedValue: 3 + weight: 1 + keyObjective: false + totalScore: + passPercentage: 90 + warningPercentage: 75 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample +spec: + timeframe: + recent: 5m + args: + "ns": "keptn-system" + analysisDefinition: + name: ed-my-proj-dev-svc1 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: KeptnMetricsProvider +metadata: + name: my-mocked-provider +spec: + type: prometheus + targetServer: "http://mockserver.$NAMESPACE.svc.cluster.local:1080" diff --git a/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/mock-server.yaml b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/mock-server.yaml new file mode 100644 index 0000000000..b01cda4fb3 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller-with-duration-timeframe/mock-server.yaml @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: Service +metadata: + name: mockserver +spec: + ports: + - name: serviceport + port: 1080 + protocol: TCP + targetPort: serviceport + selector: + app: mockserver + sessionAffinity: None + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: mockserver + name: mockserver +spec: + replicas: 1 + selector: + matchLabels: + app: mockserver + template: + metadata: + labels: + app: mockserver + name: mockserver + spec: + containers: + - env: + - name: MOCKSERVER_LOG_LEVEL + value: INFO + - name: SERVER_PORT + value: "1080" + image: mockserver/mockserver:mockserver-5.13.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + name: mockserver + ports: + - containerPort: 1080 + name: serviceport + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 2 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + volumeMounts: + - mountPath: /config + name: config-volume + - mountPath: /libs + name: libs-volume + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: config-volume + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: libs-volume +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: mockserver-config +data: + initializerJson.json: |- + [ + { + "httpRequest": { + "path": "/api/v1/query_range", + "method": "POST" + }, + "httpResponse": { + "body": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "metric-name", + "job": "", + "instance": "" + }, + "values": [[1669714193.275, "4"]] + } + ] + } + }, + "statusCode": 200 + } + } + ] + mockserver.properties: |- + ############################### + # MockServer & Proxy Settings # + ############################### + # Socket & Port Settings + # socket timeout in milliseconds (default 120000) + mockserver.maxSocketTimeout=120000 + # Certificate Generation + # dynamically generated CA key pair (if they don't already exist in + specified directory) + mockserver.dynamicallyCreateCertificateAuthorityCertificate=true + # save dynamically generated CA key pair in working directory + mockserver.directoryToSaveDynamicSSLCertificate=. + # certificate domain name (default "localhost") + mockserver.sslCertificateDomainName=localhost + # comma separated list of ip addresses for Subject Alternative Name domain + names (default empty list) + mockserver.sslSubjectAlternativeNameDomains=www.example.com,www.another.com + # comma separated list of ip addresses for Subject Alternative Name ips + (default empty list) + mockserver.sslSubjectAlternativeNameIps=127.0.0.1 + # CORS + # enable CORS for MockServer REST API + mockserver.enableCORSForAPI=true + # enable CORS for all responses + mockserver.enableCORSForAllResponses=true + # Json Initialization + mockserver.initializationJsonPath=/config/initializerJson.json diff --git a/test/chainsaw/testanalysis/analysis-controller/assert-1.yaml b/test/chainsaw/testanalysis/analysis-controller/assert-1.yaml new file mode 100755 index 0000000000..8c8c27bdbf --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller/assert-1.yaml @@ -0,0 +1,12 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample +spec: + analysisDefinition: + name: ed-my-proj-dev-svc1 +status: + pass: true + # yamllint disable-line rule:line-length + raw: '{"objectiveResults":[{"result":{"failResult":{"operator":{"lessThan":{"fixedValue":"2"}},"fulfilled":false},"warnResult":{"operator":{"lessThan":{"fixedValue":"3"}},"fulfilled":false},"warning":false,"pass":true},"objective":{"analysisValueTemplateRef":{"name":"ready"},"target":{"failure":{"lessThan":{"fixedValue":"2"}},"warning":{"lessThan":{"fixedValue":"3"}}},"weight":1},"value":4,"query":"sum(kube_pod_container_status_ready{namespace=\"keptn-system\"})","score":1}],"totalScore":1,"maximumScore":1,"pass":true,"warning":false}' + state: Completed diff --git a/test/chainsaw/testanalysis/analysis-controller/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysis-controller/chainsaw-test.yaml new file mode 100755 index 0000000000..7b2b6277f1 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller/chainsaw-test.yaml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysis-controller +spec: + steps: + - name: step-00 + try: + - script: + content: | + kubectl apply -f mock-server.yaml -n $NAMESPACE + - script: + content: | + envsubst < install.yaml | kubectl apply -f - -n $NAMESPACE + - name: step-01 + try: + - assert: + file: assert-1.yaml + catch: + - script: + content: kubectl logs -l control-plane=metrics-operator -n keptn-system + - script: + content: kubectl describe Analysis -n $NAMESPACE diff --git a/test/chainsaw/testanalysis/analysis-controller/install.yaml b/test/chainsaw/testanalysis/analysis-controller/install.yaml new file mode 100644 index 0000000000..6d82e7abac --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller/install.yaml @@ -0,0 +1,49 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: ready +spec: + provider: + name: my-mocked-provider + query: 'sum(kube_pod_container_status_ready{namespace="{{.ns}}"})' +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisDefinition +metadata: + name: ed-my-proj-dev-svc1 +spec: + objectives: + - analysisValueTemplateRef: + name: ready + target: + failure: + lessThan: + fixedValue: 2 + warning: + lessThan: + fixedValue: 3 + weight: 1 + keyObjective: false + totalScore: + passPercentage: 90 + warningPercentage: 75 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: analysis-sample +spec: + timeframe: + recent: 5m + args: + "ns": "keptn-system" + analysisDefinition: + name: ed-my-proj-dev-svc1 +--- +apiVersion: metrics.keptn.sh/v1beta1 +kind: KeptnMetricsProvider +metadata: + name: my-mocked-provider +spec: + type: prometheus + targetServer: "http://mockserver.$NAMESPACE.svc.cluster.local:1080" diff --git a/test/chainsaw/testanalysis/analysis-controller/mock-server.yaml b/test/chainsaw/testanalysis/analysis-controller/mock-server.yaml new file mode 100644 index 0000000000..b01cda4fb3 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-controller/mock-server.yaml @@ -0,0 +1,140 @@ +apiVersion: v1 +kind: Service +metadata: + name: mockserver +spec: + ports: + - name: serviceport + port: 1080 + protocol: TCP + targetPort: serviceport + selector: + app: mockserver + sessionAffinity: None + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: mockserver + name: mockserver +spec: + replicas: 1 + selector: + matchLabels: + app: mockserver + template: + metadata: + labels: + app: mockserver + name: mockserver + spec: + containers: + - env: + - name: MOCKSERVER_LOG_LEVEL + value: INFO + - name: SERVER_PORT + value: "1080" + image: mockserver/mockserver:mockserver-5.13.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 10 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + name: mockserver + ports: + - containerPort: 1080 + name: serviceport + protocol: TCP + readinessProbe: + failureThreshold: 10 + initialDelaySeconds: 2 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: serviceport + timeoutSeconds: 1 + volumeMounts: + - mountPath: /config + name: config-volume + - mountPath: /libs + name: libs-volume + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: config-volume + - configMap: + defaultMode: 420 + name: mockserver-config + optional: true + name: libs-volume +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: mockserver-config +data: + initializerJson.json: |- + [ + { + "httpRequest": { + "path": "/api/v1/query_range", + "method": "POST" + }, + "httpResponse": { + "body": { + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": { + "__name__": "metric-name", + "job": "", + "instance": "" + }, + "values": [[1669714193.275, "4"]] + } + ] + } + }, + "statusCode": 200 + } + } + ] + mockserver.properties: |- + ############################### + # MockServer & Proxy Settings # + ############################### + # Socket & Port Settings + # socket timeout in milliseconds (default 120000) + mockserver.maxSocketTimeout=120000 + # Certificate Generation + # dynamically generated CA key pair (if they don't already exist in + specified directory) + mockserver.dynamicallyCreateCertificateAuthorityCertificate=true + # save dynamically generated CA key pair in working directory + mockserver.directoryToSaveDynamicSSLCertificate=. + # certificate domain name (default "localhost") + mockserver.sslCertificateDomainName=localhost + # comma separated list of ip addresses for Subject Alternative Name domain + names (default empty list) + mockserver.sslSubjectAlternativeNameDomains=www.example.com,www.another.com + # comma separated list of ip addresses for Subject Alternative Name ips + (default empty list) + mockserver.sslSubjectAlternativeNameIps=127.0.0.1 + # CORS + # enable CORS for MockServer REST API + mockserver.enableCORSForAPI=true + # enable CORS for all responses + mockserver.enableCORSForAllResponses=true + # Json Initialization + mockserver.initializationJsonPath=/config/initializerJson.json diff --git a/test/chainsaw/testanalysis/analysis-resources/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysis-resources/chainsaw-test.yaml new file mode 100755 index 0000000000..5113aa0669 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-resources/chainsaw-test.yaml @@ -0,0 +1,40 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysis-resources +spec: + steps: + - name: step-00 + try: + - apply: + file: invalid-analysis-1.yaml + expect: + - check: + ($error != null): true + - apply: + file: invalid-analysis-2.yaml + expect: + - check: + ($error != null): true + - apply: + file: invalid-analysis-3.yaml + expect: + - check: + ($error != null): true + - apply: + file: valid-analysis-1.yaml + - apply: + file: valid-analysis-2.yaml + - name: step-01 + try: + - assert: + file: valid-analysis-1.yaml + - assert: + file: valid-analysis-2.yaml + - error: + file: invalid-analysis-1.yaml + - error: + file: invalid-analysis-2.yaml + - error: + file: invalid-analysis-3.yaml diff --git a/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-1.yaml b/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-1.yaml new file mode 100644 index 0000000000..cfcff9e781 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-1.yaml @@ -0,0 +1,17 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: invalid-analysis-1 +spec: + timeframe: + # using 'recent' and 'from'/'to' at the same time + recent: 5m + from: 2023-05-05T05:05:05Z + to: 2023-05-05T10:10:10Z + args: + project: my-project + stage: dev + service: svc1 + foo: bar # can be any key/value pair; NOT only project/stage/service + analysisDefinition: + name: ed-my-proj-dev-svc1 diff --git a/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-2.yaml b/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-2.yaml new file mode 100644 index 0000000000..9606cda5f9 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-2.yaml @@ -0,0 +1,15 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: invalid-analysis-2 +spec: + timeframe: + # using invalid 'recent' value + recent: "five minutes" + args: + project: my-project + stage: dev + service: svc1 + foo: bar # can be any key/value pair; NOT only project/stage/service + analysisDefinition: + name: ed-my-proj-dev-svc1 diff --git a/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-3.yaml b/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-3.yaml new file mode 100644 index 0000000000..84941353dd --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-resources/invalid-analysis-3.yaml @@ -0,0 +1,16 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: invalid-analysis-3 +spec: + timeframe: + # 'from' is before 'to' + to: 2023-05-05T05:05:05Z + from: 2023-05-05T10:10:10Z + args: + project: my-project + stage: dev + service: svc1 + foo: bar # can be any key/value pair; NOT only project/stage/service + analysisDefinition: + name: ed-my-proj-dev-svc1 diff --git a/test/chainsaw/testanalysis/analysis-resources/valid-analysis-1.yaml b/test/chainsaw/testanalysis/analysis-resources/valid-analysis-1.yaml new file mode 100644 index 0000000000..b506e2b6b1 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-resources/valid-analysis-1.yaml @@ -0,0 +1,16 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: valid-analysis-1 +spec: + timeframe: + # using from/to timestamps + from: 2023-05-05T05:05:05Z + to: 2023-05-05T10:10:10Z + args: + project: my-project + stage: dev + service: svc1 + foo: bar # can be any key/value pair; NOT only project/stage/service + analysisDefinition: + name: ed-my-proj-dev-svc1 diff --git a/test/chainsaw/testanalysis/analysis-resources/valid-analysis-2.yaml b/test/chainsaw/testanalysis/analysis-resources/valid-analysis-2.yaml new file mode 100644 index 0000000000..1156d93ccc --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-resources/valid-analysis-2.yaml @@ -0,0 +1,15 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: Analysis +metadata: + name: valid-analysis-2 +spec: + timeframe: + # using 'recent' + recent: 5m + args: + project: my-project + stage: dev + service: svc1 + foo: bar # can be any key/value pair; NOT only project/stage/service + analysisDefinition: + name: ed-my-proj-dev-svc1 diff --git a/test/chainsaw/testanalysis/analysis-value-template/00-assert.yaml b/test/chainsaw/testanalysis/analysis-value-template/00-assert.yaml new file mode 100644 index 0000000000..2dd1051df4 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-value-template/00-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: analysis-value-template-sample +spec: + provider: + name: some-provider + query: "sum(kube_pod_container_resource_limits{resource='{{.Resource}}'}) - sum(kube_node_status_capacity{resource='{{.Resource}}'})" diff --git a/test/chainsaw/testanalysis/analysis-value-template/00-install.yaml b/test/chainsaw/testanalysis/analysis-value-template/00-install.yaml new file mode 100644 index 0000000000..2dd1051df4 --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-value-template/00-install.yaml @@ -0,0 +1,8 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisValueTemplate +metadata: + name: analysis-value-template-sample +spec: + provider: + name: some-provider + query: "sum(kube_pod_container_resource_limits{resource='{{.Resource}}'}) - sum(kube_node_status_capacity{resource='{{.Resource}}'})" diff --git a/test/chainsaw/testanalysis/analysis-value-template/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysis-value-template/chainsaw-test.yaml new file mode 100755 index 0000000000..b2cec7fabf --- /dev/null +++ b/test/chainsaw/testanalysis/analysis-value-template/chainsaw-test.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysis-value-template +spec: + steps: + - name: step-00 + try: + - apply: + file: 00-install.yaml + - assert: + file: 00-assert.yaml diff --git a/test/chainsaw/testanalysis/analysisdefinition-validate/badanalysis.yaml b/test/chainsaw/testanalysis/analysisdefinition-validate/badanalysis.yaml new file mode 100644 index 0000000000..f894a30c65 --- /dev/null +++ b/test/chainsaw/testanalysis/analysisdefinition-validate/badanalysis.yaml @@ -0,0 +1,20 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisDefinition +metadata: + name: ed-my-proj-dev-svc1 +spec: + objectives: + - analysisValueTemplateRef: + name: ready + target: + failure: + lessThan: + fixedValue: 2 + warning: + lessThan: + fixedValue: 3 + weight: 1 + keyObjective: false + totalScore: + passPercentage: 50 + warningPercentage: 75 diff --git a/test/chainsaw/testanalysis/analysisdefinition-validate/chainsaw-test.yaml b/test/chainsaw/testanalysis/analysisdefinition-validate/chainsaw-test.yaml new file mode 100755 index 0000000000..8bd4e729db --- /dev/null +++ b/test/chainsaw/testanalysis/analysisdefinition-validate/chainsaw-test.yaml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: analysisdefinition-validate +spec: + steps: + - name: step-00 + try: + - apply: + file: badanalysis.yaml + expect: + - check: + ($error != null): true + - apply: + file: goodanalysis.yaml + - name: step-01 + try: + - assert: + file: goodanalysis.yaml + - error: + file: badanalysis.yaml diff --git a/test/chainsaw/testanalysis/analysisdefinition-validate/goodanalysis.yaml b/test/chainsaw/testanalysis/analysisdefinition-validate/goodanalysis.yaml new file mode 100644 index 0000000000..18c84bffaa --- /dev/null +++ b/test/chainsaw/testanalysis/analysisdefinition-validate/goodanalysis.yaml @@ -0,0 +1,20 @@ +apiVersion: metrics.keptn.sh/v1beta1 +kind: AnalysisDefinition +metadata: + name: ed-my-proj-dev-svc2 +spec: + objectives: + - analysisValueTemplateRef: + name: ready + target: + failure: + lessThan: + fixedValue: 2 + warning: + lessThan: + fixedValue: 3 + weight: 1 + keyObjective: false + totalScore: + passPercentage: 90 + warningPercentage: 75