From 462f22cab190cb80e7732b54e88c0cd3c1bdf453 Mon Sep 17 00:00:00 2001 From: Janos Laszlo Vasik Date: Tue, 25 Jun 2024 14:43:22 +0200 Subject: [PATCH] feat: RHINENG-9589 add autopromotion pipeline tests --- .github/workflows/checkimages.yaml | 2 +- Dockerfile | 7 +++ Makefile | 12 +++- README.md | 52 ++++++++++++--- ...openshift_stage_test_template_generator.rb | 32 ++++++++++ config/stage_test/kustomization.yaml | 22 +++++++ config/stage_test/stage_test.yaml | 23 +++++++ config/stage_test/stage_test_role.yaml | 14 +++++ .../stage_test/stage_test_role_binding.yaml | 11 ++++ .../stage_test_service_account.yaml | 4 ++ deploy_template.yaml | 2 +- stage_test.sh | 24 +++++++ stage_test_template.yaml | 63 +++++++++++++++++++ 13 files changed, 257 insertions(+), 11 deletions(-) create mode 100644 config/plugins/openshift_stage_test_template_generator.rb create mode 100644 config/stage_test/kustomization.yaml create mode 100644 config/stage_test/stage_test.yaml create mode 100644 config/stage_test/stage_test_role.yaml create mode 100644 config/stage_test/stage_test_role_binding.yaml create mode 100644 config/stage_test/stage_test_service_account.yaml create mode 100644 stage_test.sh create mode 100644 stage_test_template.yaml diff --git a/.github/workflows/checkimages.yaml b/.github/workflows/checkimages.yaml index a796db9..f300f0d 100644 --- a/.github/workflows/checkimages.yaml +++ b/.github/workflows/checkimages.yaml @@ -49,7 +49,7 @@ jobs: skopeo inspect "docker://quay.io/cloudservices/floorist:$tag" > /dev/null sed -i "/name: FLOORIST_IMAGE_TAG/{n;s/value: .*/value: '$tag'/}" \ config/templated/template_params.yaml - make openshift-template + make openshift-templates - name: Commit changes (if any) run: | git config user.name 'Update-a-Bot' diff --git a/Dockerfile b/Dockerfile index 9eb643e..d93e0b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,12 @@ USER root RUN dnf -y upgrade && \ dnf -y clean all +RUN curl --output oc.tar.gz \ + https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz && \ + tar -xvf oc.tar.gz oc && \ + mv oc /usr/local/bin/ && \ + rm oc.tar.gz + USER ${USER_UID} COPY requirements.yml ${HOME}/requirements.yml @@ -16,3 +22,4 @@ RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ COPY watches.yaml ${HOME}/watches.yaml COPY roles/ ${HOME}/roles/ COPY playbooks/ ${HOME}/playbooks/ +COPY stage_test.sh ${HOME}/stage_test.sh diff --git a/Makefile b/Makefile index 881014a..f06bb23 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,9 @@ IMG ?= $(IMAGE_TAG_BASE):$(VERSION) # Build parameters IMG_BUILD_PARAMS ?= -# OpenShift Template file +# OpenShift Template files OPENSHIFT_TEMPLATE ?= deploy_template.yaml +OPENSHIFT_STAGE_TEST_TEMPLATE ?= stage_test_template.yaml .PHONY: all all: podman-build @@ -146,6 +147,15 @@ openshift-template: kustomize config/plugins/openshift_template_generator.rb config/templated/template_params.yaml \ > "${OPENSHIFT_TEMPLATE}" +.PHONY: openshift-stage-test-template +openshift-stage-test-template: kustomize + $(KUSTOMIZE) build config/stage_test | \ + config/plugins/openshift_stage_test_template_generator.rb \ + > "${OPENSHIFT_STAGE_TEST_TEMPLATE}" + +.PHONY: openshift-templates +openshift-templates: openshift-template openshift-stage-test-template + .PHONY: ansible-operator ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. diff --git a/README.md b/README.md index 5b8bf70..8ccefe5 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ Kubernetes Operator to manage scheduling of metrics export with [Floorist](https 1. [Prerequisites](#prerequisites) 2. [Test environment](#test-environment) 3. [Build and deploy](#build-and-deploy) - 4. [OpenShift template](#openshift-template) + 4. [OpenShift templates](#openshift-templates) 5. [Trying it out](#trying-it-out) + 6. [Stage test](#stage-test) ## Description @@ -166,20 +167,33 @@ Replace `SETOPERATORIMAGETAG` with desired image tag for the operator. Optionally, `IMAGE_TAG_BASE` can be set to use a custom container registry. For example `IMAGE_TAG_BASE=quay.io/yourusername/floorist-operator`. -### OpenShift template +### OpenShift templates OpenShift utilizes [`Template`](https://docs.openshift.com/container-platform/4.7/openshift_images/using-templates.html) resources. -Due to current limitation in some environments there was a `openshift-teplate` Makefile target created -along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool. -The `openshift-teplate` target generates an OpenShift `Template` out of kustomized resources -configured within [`config/templated/`](config/templated/kustomization.yaml). +Due to current limitation in some environments there was a `openshift-teplates` Makefile target created +along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) and +[`openshift_stage_test_template_generator.rb`](config/plugins/openshift_stage_test_template_generator.rb) tools. +The `openshift-teplates` target generates an OpenShift `Template` out of kustomized resources +configured within [`config/templated/`](config/templated/kustomization.yaml) and +[`config/stage_test/`](config/stage_test/kustomization.yaml) . -To (re)generate OpenShift template for this operator use: +To (re)generate OpenShift templates for this operator and it's test job use: +``` +make openshift-templates +``` +The results are written in the `deploy_template.yaml` and `stage_test_template.yaml` files. + +It is also possible to (re)generate only the operator's or only the test's template: + +To (re)generate the operator's template: ``` make openshift-template ``` -The result is writte in the `deploy_template.yaml` file. +To (re)generate the test's template: +``` +make openshift-stage-test-template +``` ### Trying it out @@ -205,3 +219,25 @@ Observe status of the worker pod: ``` minikube kubectl -- get pod -l 'job-name=floorist-floorplan-sample-exporter-manual' ``` + +### Stage test + +For the purpose of testing before auto-promotion from staging to production environment we've developed a job with a +24h delay. This test checks for the successful creation of cronjobs and jobs by the operator and it also asserts +the successful completion of the jobs. + +It is possible to use the test locally, altough with limited usability in the form of non-unique job names. This means +that each time the test is re-ran it needs to be deleted first. + +To run the test locally: +``` +minikube kubectl -- apply -k /config/stage_test +``` + +If you'd like to see the test pass: + 1. set up the [test environment](#test-environment) + 2. create a [sample cronjob](#trying-it-out) and be sure to manually trigger a job + 3. deploy the test using the command above + +To understand the differences in the test's configuration in a local and in the staging environment see the comments in +[`openshift_stage_test_template_generator.rb`](config/plugins/openshift_stage_test_template_generator.rb). diff --git a/config/plugins/openshift_stage_test_template_generator.rb b/config/plugins/openshift_stage_test_template_generator.rb new file mode 100644 index 0000000..8f4d2d6 --- /dev/null +++ b/config/plugins/openshift_stage_test_template_generator.rb @@ -0,0 +1,32 @@ +#!/bin/ruby + +require 'yaml' + +objects = YAML.load_stream(STDIN) + +# Replacing namespace 'default' with 'floorist-operator-system' due to the difference in namespaces +# between the local (minikube) and staging environments +stage_namespace = 'floorist-operator-system' + +objects.each do |object| + # Replacing 'name' key with 'generateName' key ensuring Job name uniqueness in the staging environment + if object['kind'] == 'Job' + object['metadata']['name'] += '-' + object['metadata']['generateName'] = object['metadata'].delete('name') + end + + object['subjects'][0]['namespace'] = stage_namespace if object['kind'] == 'RoleBinding' + + object['metadata']['namespace'] = stage_namespace +end + +template = { + 'apiVersion' => 'v1', + 'kind' => 'Template', + 'metadata' => { + 'name' => 'floorist-operator-stage-test' + }, + 'objects' => objects +} + +YAML.dump(template, STDOUT) diff --git a/config/stage_test/kustomization.yaml b/config/stage_test/kustomization.yaml new file mode 100644 index 0000000..79ba7c3 --- /dev/null +++ b/config/stage_test/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Adds namespace to all resources. +namespace: default + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: floorist-operator- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +resources: +- stage_test_service_account.yaml +- stage_test_role.yaml +- stage_test_role_binding.yaml +- stage_test.yaml diff --git a/config/stage_test/stage_test.yaml b/config/stage_test/stage_test.yaml new file mode 100644 index 0000000..08e7ec3 --- /dev/null +++ b/config/stage_test/stage_test.yaml @@ -0,0 +1,23 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: stage-test +spec: + template: + spec: + restartPolicy: Never + serviceAccountName: stage-test-service-account + containers: + - image: quay.io/cloudservices/floorist-operator:latest + name: floorist-operator-stage-test + imagePullPolicy: Always + command: + - sh + args: + - stage_test.sh + resources: + requests: + memory: 256Mi + cpu: 300m + limits: + memory: 512Mi diff --git a/config/stage_test/stage_test_role.yaml b/config/stage_test/stage_test_role.yaml new file mode 100644 index 0000000..ed79109 --- /dev/null +++ b/config/stage_test/stage_test_role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stage-test-role +rules: +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch diff --git a/config/stage_test/stage_test_role_binding.yaml b/config/stage_test/stage_test_role_binding.yaml new file mode 100644 index 0000000..11c2eb5 --- /dev/null +++ b/config/stage_test/stage_test_role_binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stage-test-rolebinding +roleRef: + kind: Role + name: stage-test-role + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: stage-test-service-account diff --git a/config/stage_test/stage_test_service_account.yaml b/config/stage_test/stage_test_service_account.yaml new file mode 100644 index 0000000..743ad74 --- /dev/null +++ b/config/stage_test/stage_test_service_account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: stage-test-service-account diff --git a/deploy_template.yaml b/deploy_template.yaml index 97bbb6c..2ca76f6 100644 --- a/deploy_template.yaml +++ b/deploy_template.yaml @@ -68,7 +68,7 @@ objects: description: Floorist Query definition. properties: chunksize: - description: + description: type: integer prefix: description: Valid folder path that will be created under diff --git a/stage_test.sh b/stage_test.sh new file mode 100644 index 0000000..55aba19 --- /dev/null +++ b/stage_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Get names of all cronjobs, if not found any ignore the error message +# to not let k8s raise an error with an exit code indicating success +CRONJOBS=$(oc get cronjobs 2>/dev/null | awk 'NR>1 {print $1}') + +if [ -z "$CRONJOBS" ]; then + echo "ERROR: no cronjobs found" + exit 1 +fi + +for CRONJOB in $CRONJOBS; do + SUCCESS=$(oc get job -l "pod=${CRONJOB}" -o jsonpath='{.items[].status.succeeded}{"\n"}') + + if [ -z "$SUCCESS" ]; then + echo "ERROR: cronjob $CRONJOB has not created any jobs" + exit 1 + fi + + if [ "$SUCCESS" != "1" ]; then + echo "ERROR: cronjob $CRONJOB has not created successful jobs" + exit 1 + fi +done diff --git a/stage_test_template.yaml b/stage_test_template.yaml new file mode 100644 index 0000000..da5c3d2 --- /dev/null +++ b/stage_test_template.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: v1 +kind: Template +metadata: + name: floorist-operator-stage-test +objects: +- apiVersion: v1 + kind: ServiceAccount + metadata: + name: floorist-operator-stage-test-service-account + namespace: floorist-operator-system +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: floorist-operator-stage-test-role + namespace: floorist-operator-system + rules: + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: floorist-operator-stage-test-rolebinding + namespace: floorist-operator-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: floorist-operator-stage-test-role + subjects: + - kind: ServiceAccount + name: floorist-operator-stage-test-service-account + namespace: floorist-operator-system +- apiVersion: batch/v1 + kind: Job + metadata: + namespace: floorist-operator-system + generateName: floorist-operator-stage-test- + spec: + template: + spec: + containers: + - args: + - stage_test.sh + command: + - sh + image: quay.io/cloudservices/floorist-operator:latest + imagePullPolicy: Always + name: floorist-operator-stage-test + resources: + limits: + memory: 512Mi + requests: + cpu: 300m + memory: 256Mi + restartPolicy: Never + serviceAccountName: floorist-operator-stage-test-service-account