feat: RHINENG-9589 add autopromotion pipeline tests

.github/workflows/checkimages.yaml

@@ -49,7 +49,7 @@ jobs:
           skopeo inspect "docker://quay.io/cloudservices/floorist:$tag" > /dev/null
           sed -i "/name: FLOORIST_IMAGE_TAG/{n;s/value: .*/value: '$tag'/}" \
             config/templated/template_params.yaml
-          make openshift-template
+          make openshift-templates
       - name: Commit changes (if any)
         run: |
           git config user.name 'Update-a-Bot'

Dockerfile

@@ -7,6 +7,12 @@ USER root
 RUN dnf -y upgrade && \
     dnf -y clean all
 
+RUN curl --output oc.tar.gz \
+ https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest/openshift-client-linux-amd64-rhel8.tar.gz && \
+ tar -xvf oc.tar.gz oc && \
+ mv oc /usr/local/bin/ && \
+ rm oc.tar.gz
+
 USER ${USER_UID}
 
 COPY requirements.yml ${HOME}/requirements.yml
@@ -16,3 +22,4 @@ RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \
 COPY watches.yaml ${HOME}/watches.yaml
 COPY roles/ ${HOME}/roles/
 COPY playbooks/ ${HOME}/playbooks/
+COPY stage_test.sh ${HOME}/stage_test.sh

Makefile

@@ -52,8 +52,9 @@ IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
 # Build parameters
 IMG_BUILD_PARAMS ?=
 
-# OpenShift Template file
+# OpenShift Template files
 OPENSHIFT_TEMPLATE ?= deploy_template.yaml
+OPENSHIFT_STAGE_TEST_TEMPLATE ?= stage_test_template.yaml
 
 .PHONY: all
 all: podman-build
@@ -112,10 +113,25 @@ deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/c
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
+.PHONY: deploy-test
+deploy-test: kustomize ## Deploy stage test to the K8s cluster specified in ~/.kube/config.
+	cd config/stage_test/local && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/stage_test/local | kubectl apply -f -
+
+.PHONY: deploy-all
+deploy-all: deploy deploy-test
+
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
 
+.PHONY: undeploy-test
+undeploy-test: ## Undeploy stage test from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/stage_test/local | kubectl delete -f -
+
+.PHONY: undeploy-all
+undeploy-all: undeploy undeploy-test
+
 minikube-secrets: config/minikube/.secrets/database.txt config/minikube/.secrets/minio.txt
 
 config/minikube/.secrets/database.txt config/minikube/.secrets/minio.txt:
@@ -143,9 +159,18 @@ endif
 .PHONY: openshift-template
 openshift-template: kustomize
 	$(KUSTOMIZE) build config/templated | \
-	config/plugins/openshift_template_generator.rb config/templated/template_params.yaml \
+	config/plugins/openshift_template_generator.rb config/templated/template_params.yaml floorist-operator \
 	> "${OPENSHIFT_TEMPLATE}"
 
+.PHONY: openshift-stage-test-template
+openshift-stage-test-template: kustomize
+	$(KUSTOMIZE) build config/stage_test | \
+	config/plugins/openshift_template_generator.rb config/stage_test/template_params.yaml floorist-operator-stage-test \
+	> "${OPENSHIFT_STAGE_TEST_TEMPLATE}"
+
+.PHONY: openshift-templates
+openshift-templates: openshift-template openshift-stage-test-template
+
 .PHONY: ansible-operator
 ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator
 ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist.

README.md

@@ -12,8 +12,9 @@ Kubernetes Operator to manage scheduling of metrics export with [Floorist](https
    1. [Prerequisites](#prerequisites)
    2. [Test environment](#test-environment)
    3. [Build and deploy](#build-and-deploy)
-   4. [OpenShift template](#openshift-template)
+   4. [OpenShift templates](#openshift-templates)
    5. [Trying it out](#trying-it-out)
+   6. [Stage test](#stage-test)
 
 
 ## Description
@@ -166,20 +167,33 @@ Replace `SETOPERATORIMAGETAG` with desired image tag for the operator.
 
 Optionally, `IMAGE_TAG_BASE` can be set to use a custom container registry. For example `IMAGE_TAG_BASE=quay.io/yourusername/floorist-operator`.
 
-### OpenShift template
+### OpenShift templates
 
 OpenShift utilizes [`Template`](https://docs.openshift.com/container-platform/4.7/openshift_images/using-templates.html)
 resources.
-Due to current limitation in some environments there was a `openshift-teplate` Makefile target created
-along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool.
-The `openshift-teplate` target generates an OpenShift `Template` out of kustomized resources
-configured within [`config/templated/`](config/templated/kustomization.yaml).
+Due to current limitation in some environments there was a `openshift-teplates` Makefile target created
+along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) and
+[`openshift_stage_test_template_generator.rb`](config/plugins/openshift_stage_test_template_generator.rb) tools.
+The `openshift-teplates` target generates an OpenShift `Template` out of kustomized resources
+configured within [`config/templated/`](config/templated/kustomization.yaml) and
+[`config/stage_test/`](config/stage_test/kustomization.yaml) .
 
-To (re)generate OpenShift template for this operator use:
+To (re)generate OpenShift templates for this operator and it's test job use:
+```
+make openshift-templates
+```
+The results are written in the `deploy_template.yaml` and `stage_test_template.yaml` files.
+
+It is also possible to (re)generate only the operator's or only the test's template:
+
+To (re)generate the operator's template:
 ```
 make openshift-template
 ```
-The result is writte in the `deploy_template.yaml` file.
+To (re)generate the test's template:
+```
+make openshift-stage-test-template
+```
 
 ### Trying it out
 
@@ -205,3 +219,30 @@ Observe status of the worker pod:
 ```
 minikube kubectl -- get pod -l 'job-name=floorist-floorplan-sample-exporter-manual'
 ```
+
+### Stage test
+
+For the purpose of testing before auto-promotion from staging to production environment we've developed a job with a
+24h delay. This test job checks for the successful creation of cronjobs and jobs by the operator, it asserts
+the successful completion of the jobs and compares the floorist images that the operator is configured with and the
+one in use by the jobs.
+
+It is possible to use the test locally, altough with the following limitations:
+- Job names are not unique: each time the test is re-ran it needs to be deleted first
+- The test that compares the operator's image of floorist with the images used in jobs by the operator is not suitable
+for the local environment.
+
+To run the test locally:
+```
+minikube kubectl -- apply -k config/stage_test/local/
+```
+or:
+```
+VERSION=SETOPERATORIMAGETAG make deploy-test
+```
+
+If you'd like to see the test pass:
+  1. [install the operator](#installation)
+  1. set up the [test environment](#test-environment)
+  2. create a [sample cronjob](#trying-it-out) and be sure to manually trigger a job
+  3. deploy the test using one of the commands above

config/plugins/openshift_template_generator.rb

@@ -2,8 +2,8 @@
 
 require 'yaml'
 
-if ARGV[0].nil?
-  puts "Usage: #{$0} parameters.yaml"
+if ARGV[0].nil? || ARGV[1].nil?
+  puts "Usage: #{$PROGRAM_NAME} parameters.yaml name_of_template"
   exit 1
 end
 
@@ -14,7 +14,7 @@
   'apiVersion' => 'v1',
   'kind' => 'Template',
   'metadata' => {
-    'name' => 'floorist-operator'
+    'name' => ARGV[1]
   },
   'objects' => objects,
   'parameters' => parameters

config/stage_test/kustomization.yaml

@@ -0,0 +1,17 @@
+namespace: floorist-operator-system
+
+resources:
+- ./local
+
+images:
+- name: quay.io/cloudservices/floorist-operator
+  newName: ${IMAGE}
+  newTag: ${IMAGE_TAG}
+
+patches:
+- path: test_params.config.yaml
+  target:
+    group: batch
+    kind: Job
+    name: stage-test
+    version: v1

config/stage_test/local/kustomization.yaml

@@ -0,0 +1,27 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# Adds namespace to all resources.
+namespace: default
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: floorist-operator-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+resources:
+- stage_test_service_account.yaml
+- stage_test_role.yaml
+- stage_test_role_binding.yaml
+- stage_test.yaml
+
+images:
+- name: controller
+  newName: quay.io/cloudservices/floorist-operator
+  newTag: latest

config/stage_test/local/stage_test.yaml

@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stage-test
+spec:
+  backoffLimit: 2
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: stage-test-service-account
+      containers:
+        - image: controller:latest
+          name: floorist-operator-stage-test
+          imagePullPolicy: Always
+          command:
+            - sh
+          args:
+            - stage_test.sh
+            - local
+          resources:
+            requests:
+              memory: 256Mi
+              cpu: 300m
+            limits:
+              memory: 512Mi

config/stage_test/local/stage_test_role.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stage-test-role
+rules:
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
+  verbs:
+  - get
+  - list
+  - watch

config/stage_test/local/stage_test_role_binding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stage-test-rolebinding
+roleRef:
+  kind: Role
+  name: stage-test-role
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: stage-test-service-account

config/stage_test/local/stage_test_service_account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stage-test-service-account

config/stage_test/template_params.yaml

@@ -0,0 +1,7 @@
+- description: Operator image
+  name: IMAGE
+  value: quay.io/cloudservices/floorist-operator
+- description: Operator image tag
+  name: IMAGE_TAG
+  value: ''
+  required: true

config/stage_test/test_params.config.yaml

@@ -0,0 +1,5 @@
+- op: replace
+  path: /metadata/name
+  value: floorist-operator-stage-test-${IMAGE_TAG}
+- op: remove
+  path: /spec/template/spec/containers/0/args/1

deploy_template.yaml

@@ -68,7 +68,7 @@ objects:
                     description: Floorist Query definition.
                     properties:
                       chunksize:
-                        description: 
+                        description:
                         type: integer
                       prefix:
                         description: Valid folder path that will be created under

stage_test.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# The first parameter is only set locally, where we can not check the Floorist image used
+if [ -z "$1" ]; then
+    NEW_FLOORIST_IMG="$(
+        oc get deployment.apps -o jsonpath='{.items[].spec.template.spec.containers[].env[1]}{"\n"}'
+    ):$(
+        oc get deployment.apps -o jsonpath='{.items[].spec.template.spec.containers[].env[2]}{"\n"}'
+    )"
+fi
+
+CRONJOBS=$(oc get cronjobs | awk 'NR>1 {print $1}')
+
+if [ -z "$CRONJOBS" ]; then
+    echo "ERROR: no cronjobs found"
+    exit 1
+fi
+
+for CRONJOB in $CRONJOBS; do
+    SUCCESS=$(oc get job -l "pod=${CRONJOB}" -o jsonpath='{.items[].status.succeeded}{"\n"}')
+
+    if [ -z "$SUCCESS" ]; then
+        echo "ERROR: cronjob $CRONJOB has not created any jobs"
+        exit 1
+    fi
+
+    if [ "$SUCCESS" != "1" ]; then
+        echo "ERROR: cronjob $CRONJOB has not created successful jobs"
+        exit 1
+    fi
+
+    if [ -z "$1" ]; then
+        if [ "$NEW_FLOORIST_IMG" != "$(oc get job -l "pod=${CRONJOB}" -o jsonpath='{.items[].spec.template.spec.containers[].image}{"\n"}')" ]; then
+            echo "ERROR: cronjob $CRONJOB is not configured with the new Floorist image"
+            exit 1
+        fi
+    fi
+done