feat: RHINENG-9589 add autopromotion pipeline tests

.github/workflows/checkimages.yaml

@@ -49,7 +49,7 @@ jobs:
           skopeo inspect "docker://quay.io/cloudservices/floorist:$tag" > /dev/null
           sed -i "/name: FLOORIST_IMAGE_TAG/{n;s/value: .*/value: '$tag'/}" \
             config/templated/template_params.yaml
-          make openshift-template
+          make openshift-templates
       - name: Commit changes (if any)
         run: |
           git config user.name 'Update-a-Bot'

Dockerfile

@@ -7,6 +7,12 @@ USER root
 RUN dnf -y upgrade && \
     dnf -y clean all
 
+RUN curl --output oc.tar.gz \
+ https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz && \
+ tar -xvf oc.tar.gz oc && \
+ mv oc /usr/local/bin/ && \
+ rm oc.tar.gz
+
 USER ${USER_UID}
 
 COPY requirements.yml ${HOME}/requirements.yml
@@ -16,3 +22,4 @@ RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \
 COPY watches.yaml ${HOME}/watches.yaml
 COPY roles/ ${HOME}/roles/
 COPY playbooks/ ${HOME}/playbooks/
+COPY stage_test.sh ${HOME}/stage_test.sh

Makefile

@@ -52,8 +52,9 @@ IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
 # Build parameters
 IMG_BUILD_PARAMS ?=
 
-# OpenShift Template file
+# OpenShift Template files
 OPENSHIFT_TEMPLATE ?= deploy_template.yaml
+OPENSHIFT_STAGE_TEST_TEMPLATE ?= stage_test_template.yaml
 
 .PHONY: all
 all: podman-build
@@ -146,6 +147,15 @@ openshift-template: kustomize
 	config/plugins/openshift_template_generator.rb config/templated/template_params.yaml \
 	> "${OPENSHIFT_TEMPLATE}"
 
+.PHONY: openshift-stage-test-template
+openshift-stage-test-template: kustomize
+	$(KUSTOMIZE) build config/stage_test | \
+	config/plugins/openshift_stage_test_template_generator.rb \
+	> "${OPENSHIFT_STAGE_TEST_TEMPLATE}"
+
+.PHONY: openshift-templates
+openshift-templates: openshift-template openshift-stage-test-template
+
 .PHONY: ansible-operator
 ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator
 ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist.

README.md

@@ -12,8 +12,9 @@ Kubernetes Operator to manage scheduling of metrics export with [Floorist](https
    1. [Prerequisites](#prerequisites)
    2. [Test environment](#test-environment)
    3. [Build and deploy](#build-and-deploy)
-   4. [OpenShift template](#openshift-template)
+   4. [OpenShift templates](#openshift-templates)
    5. [Trying it out](#trying-it-out)
+   6. [Stage test](#stage-test)
 
 
 ## Description
@@ -166,20 +167,33 @@ Replace `SETOPERATORIMAGETAG` with desired image tag for the operator.
 
 Optionally, `IMAGE_TAG_BASE` can be set to use a custom container registry. For example `IMAGE_TAG_BASE=quay.io/yourusername/floorist-operator`.
 
-### OpenShift template
+### OpenShift templates
 
 OpenShift utilizes [`Template`](https://docs.openshift.com/container-platform/4.7/openshift_images/using-templates.html)
 resources.
-Due to current limitation in some environments there was a `openshift-teplate` Makefile target created
-along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool.
-The `openshift-teplate` target generates an OpenShift `Template` out of kustomized resources
-configured within [`config/templated/`](config/templated/kustomization.yaml).
+Due to current limitation in some environments there was a `openshift-teplates` Makefile target created
+along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) and
+[`openshift_stage_test_template_generator.rb`](config/plugins/openshift_stage_test_template_generator.rb) tools.
+The `openshift-teplates` target generates an OpenShift `Template` out of kustomized resources
+configured within [`config/templated/`](config/templated/kustomization.yaml) and
+[`config/stage_test/`](config/stage_test/kustomization.yaml) .
 
-To (re)generate OpenShift template for this operator use:
+To (re)generate OpenShift templates for this operator and it's test job use:
+```
+make openshift-templates
+```
+The results are written in the `deploy_template.yaml` and `stage_test_template.yaml` files.
+
+It is also possible to (re)generate only the operator's or only the test's template:
+
+To (re)generate the operator's template:
 ```
 make openshift-template
 ```
-The result is writte in the `deploy_template.yaml` file.
+To (re)generate the test's template:
+```
+make openshift-stage-test-template
+```
 
 ### Trying it out
 
@@ -205,3 +219,25 @@ Observe status of the worker pod:
 ```
 minikube kubectl -- get pod -l 'job-name=floorist-floorplan-sample-exporter-manual'
 ```
+
+### Stage test
+
+For the purpose of testing before auto-promotion from staging to production environment we've developed a job with a
+24h delay. This test checks for the successful creation of cronjobs and jobs by the operator and it also asserts
+the successful completion of the jobs.
+
+It is possible to use the test locally, altough with limited usability in the form of non-unique job names. This means
+that each time the test is re-ran it needs to be deleted first.
+
+To run the test locally:
+```
+minikube kubectl -- apply -k /config/stage_test
+```
+
+If you'd like to see the test pass:
+  1. set up the [test environment](#test-environment)
+  2. create a [sample cronjob](#trying-it-out) and be sure to manually trigger a job
+  3. deploy the test using the command above
+
+To understand the differences in the test's configuration in a local and in the staging environment see the comments in
+[`openshift_stage_test_template_generator.rb`](config/plugins/openshift_stage_test_template_generator.rb).

config/plugins/openshift_stage_test_template_generator.rb

@@ -0,0 +1,32 @@
+#!/bin/ruby
+
+require 'yaml'
+
+objects = YAML.load_stream(STDIN)
+
+# Replacing namespace 'default' with 'floorist-operator-system' due to the difference in namespaces
+# between the local (minikube) and staging environments
+stage_namespace = 'floorist-operator-system'
+
+objects.each do |object|
+  # Replacing 'name' key with 'generateName' key ensuring Job name uniqueness in the staging environment
+  if object['kind'] == 'Job'
+    object['metadata']['name'] += '-'
+    object['metadata']['generateName'] = object['metadata'].delete('name')
+  end
+
+  object['subjects'][0]['namespace'] = stage_namespace if object['kind'] == 'RoleBinding'
+
+  object['metadata']['namespace'] = stage_namespace
+end
+
+template = {
+  'apiVersion' => 'v1',
+  'kind' => 'Template',
+  'metadata' => {
+    'name' => 'floorist-operator-stage-test'
+  },
+  'objects' => objects
+}
+
+YAML.dump(template, STDOUT)

config/stage_test/kustomization.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# Adds namespace to all resources.
+namespace: default
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: floorist-operator-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+resources:
+- stage_test_service_account.yaml
+- stage_test_role.yaml
+- stage_test_role_binding.yaml
+- stage_test.yaml

config/stage_test/stage_test.yaml

@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: stage-test
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: stage-test-service-account
+      containers:
+        - image: quay.io/cloudservices/floorist-operator:latest
+          name: floorist-operator-stage-test
+          imagePullPolicy: Always
+          command:
+            - sh
+          args:
+            - stage_test.sh
+          resources:
+            requests:
+              memory: 256Mi
+              cpu: 300m
+            limits:
+              memory: 512Mi

config/stage_test/stage_test_role.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: stage-test-role
+rules:
+- apiGroups:
+  - batch
+  resources:
+  - cronjobs
+  - jobs
+  verbs:
+  - get
+  - list
+  - watch

config/stage_test/stage_test_role_binding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: stage-test-rolebinding
+roleRef:
+  kind: Role
+  name: stage-test-role
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: stage-test-service-account

config/stage_test/stage_test_service_account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stage-test-service-account

deploy_template.yaml

@@ -68,7 +68,7 @@ objects:
                     description: Floorist Query definition.
                     properties:
                       chunksize:
-                        description: 
+                        description:
                         type: integer
                       prefix:
                         description: Valid folder path that will be created under

stage_test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Get names of all cronjobs, if not found any ignore the error message
+# to not let k8s raise an error with an exit code indicating success
+CRONJOBS=$(oc get cronjobs 2>/dev/null | awk 'NR>1 {print $1}')
+
+if [ -z "$CRONJOBS" ]; then
+    echo "ERROR: no cronjobs found"
+    exit 1
+fi
+
+for CRONJOB in $CRONJOBS; do
+    SUCCESS=$(oc get job -l "pod=${CRONJOB}" -o jsonpath='{.items[].status.succeeded}{"\n"}')
+
+    if [ -z "$SUCCESS" ]; then
+        echo "ERROR: cronjob $CRONJOB has not created any jobs"
+        exit 1
+    fi
+
+    if [ "$SUCCESS" != "1" ]; then
+        echo "ERROR: cronjob $CRONJOB has not created successful jobs"
+        exit 1
+    fi
+done

stage_test_template.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: v1
+kind: Template
+metadata:
+  name: floorist-operator-stage-test
+objects:
+- apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    name: floorist-operator-stage-test-service-account
+    namespace: floorist-operator-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: floorist-operator-stage-test-role
+    namespace: floorist-operator-system
+  rules:
+  - apiGroups:
+    - batch
+    resources:
+    - cronjobs
+    - jobs
+    verbs:
+    - get
+    - list
+    - watch
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: floorist-operator-stage-test-rolebinding
+    namespace: floorist-operator-system
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: floorist-operator-stage-test-role
+  subjects:
+  - kind: ServiceAccount
+    name: floorist-operator-stage-test-service-account
+    namespace: floorist-operator-system
+- apiVersion: batch/v1
+  kind: Job
+  metadata:
+    namespace: floorist-operator-system
+    generateName: floorist-operator-stage-test-
+  spec:
+    template:
+      spec:
+        containers:
+        - args:
+          - stage_test.sh
+          command:
+          - sh
+          image: quay.io/cloudservices/floorist-operator:latest
+          imagePullPolicy: Always
+          name: floorist-operator-stage-test
+          resources:
+            limits:
+              memory: 512Mi
+            requests:
+              cpu: 300m
+              memory: 256Mi
+        restartPolicy: Never
+        serviceAccountName: floorist-operator-stage-test-service-account