diff --git a/.github/workflows/checkimages.yaml b/.github/workflows/checkimages.yaml index a796db9..f300f0d 100644 --- a/.github/workflows/checkimages.yaml +++ b/.github/workflows/checkimages.yaml @@ -49,7 +49,7 @@ jobs: skopeo inspect "docker://quay.io/cloudservices/floorist:$tag" > /dev/null sed -i "/name: FLOORIST_IMAGE_TAG/{n;s/value: .*/value: '$tag'/}" \ config/templated/template_params.yaml - make openshift-template + make openshift-templates - name: Commit changes (if any) run: | git config user.name 'Update-a-Bot' diff --git a/Dockerfile b/Dockerfile index be52114..87ca127 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,12 @@ USER root RUN dnf -y upgrade && \ dnf -y clean all +RUN curl --output oc.tar.gz \ + https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest/openshift-client-linux-amd64-rhel8.tar.gz && \ + tar -xvf oc.tar.gz oc && \ + mv oc /usr/local/bin/ && \ + rm oc.tar.gz + USER ${USER_UID} COPY requirements.yml ${HOME}/requirements.yml @@ -16,3 +22,4 @@ RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ COPY watches.yaml ${HOME}/watches.yaml COPY roles/ ${HOME}/roles/ COPY playbooks/ ${HOME}/playbooks/ +COPY stage_test.sh ${HOME}/stage_test.sh diff --git a/Makefile b/Makefile index 881014a..9dab922 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,9 @@ IMG ?= $(IMAGE_TAG_BASE):$(VERSION) # Build parameters IMG_BUILD_PARAMS ?= -# OpenShift Template file +# OpenShift Template files OPENSHIFT_TEMPLATE ?= deploy_template.yaml +OPENSHIFT_STAGE_TEST_TEMPLATE ?= stage_test_template.yaml .PHONY: all all: podman-build @@ -112,10 +113,25 @@ deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/c cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - +.PHONY: deploy-test +deploy-test: kustomize ## Deploy stage test to the K8s cluster specified in ~/.kube/config. + cd config/stage_test/local && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/stage_test/local | kubectl apply -f - + +.PHONY: deploy-all +deploy-all: deploy deploy-test + .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - +.PHONY: undeploy-test +undeploy-test: ## Undeploy stage test from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/stage_test/local | kubectl delete -f - + +.PHONY: undeploy-all +undeploy-all: undeploy undeploy-test + minikube-secrets: config/minikube/.secrets/database.txt config/minikube/.secrets/minio.txt config/minikube/.secrets/database.txt config/minikube/.secrets/minio.txt: @@ -143,9 +159,18 @@ endif .PHONY: openshift-template openshift-template: kustomize $(KUSTOMIZE) build config/templated | \ - config/plugins/openshift_template_generator.rb config/templated/template_params.yaml \ + config/plugins/openshift_template_generator.rb config/templated/template_params.yaml floorist-operator \ > "${OPENSHIFT_TEMPLATE}" +.PHONY: openshift-stage-test-template +openshift-stage-test-template: kustomize + $(KUSTOMIZE) build config/stage_test | \ + config/plugins/openshift_template_generator.rb config/stage_test/template_params.yaml floorist-operator-stage-test \ + > "${OPENSHIFT_STAGE_TEST_TEMPLATE}" + +.PHONY: openshift-templates +openshift-templates: openshift-template openshift-stage-test-template + .PHONY: ansible-operator ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. diff --git a/README.md b/README.md index 5b8bf70..7c11005 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ Kubernetes Operator to manage scheduling of metrics export with [Floorist](https 1. [Prerequisites](#prerequisites) 2. [Test environment](#test-environment) 3. [Build and deploy](#build-and-deploy) - 4. [OpenShift template](#openshift-template) + 4. [OpenShift templates](#openshift-templates) 5. [Trying it out](#trying-it-out) + 6. [Stage test](#stage-test) ## Description @@ -166,20 +167,32 @@ Replace `SETOPERATORIMAGETAG` with desired image tag for the operator. Optionally, `IMAGE_TAG_BASE` can be set to use a custom container registry. For example `IMAGE_TAG_BASE=quay.io/yourusername/floorist-operator`. -### OpenShift template +### OpenShift templates OpenShift utilizes [`Template`](https://docs.openshift.com/container-platform/4.7/openshift_images/using-templates.html) resources. -Due to current limitation in some environments there was a `openshift-teplate` Makefile target created -along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool. -The `openshift-teplate` target generates an OpenShift `Template` out of kustomized resources -configured within [`config/templated/`](config/templated/kustomization.yaml). +Due to current limitation in some environments there was a `openshift-teplates` Makefile target created +along with the [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool. +The `openshift-teplates` target generates an OpenShift `Template` out of kustomized resources +configured within [`config/templated/`](config/templated/kustomization.yaml) and +[`config/stage_test/`](config/stage_test/kustomization.yaml) . -To (re)generate OpenShift template for this operator use: +To (re)generate OpenShift templates for this operator and it's test job use: +``` +make openshift-templates +``` +The results are written in the `deploy_template.yaml` and `stage_test_template.yaml` files. + +It is also possible to (re)generate only the operator's or only the test's template: + +To (re)generate the operator's template: ``` make openshift-template ``` -The result is writte in the `deploy_template.yaml` file. +To (re)generate the test's template: +``` +make openshift-stage-test-template +``` ### Trying it out @@ -205,3 +218,27 @@ Observe status of the worker pod: ``` minikube kubectl -- get pod -l 'job-name=floorist-floorplan-sample-exporter-manual' ``` + +### Stage test + +For the purpose of testing before auto-promotion from staging to production environment we've developed a job with a +24h delay. This test job checks for the successful creation of cronjobs and jobs by the operator and it asserts +the successful completion of the jobs. + +It is possible to use the test locally, altough with the following limitations: +- Job names are not unique: each time the test is re-ran it needs to be deleted first + +To run the test locally: +``` +minikube kubectl -- apply -k config/stage_test/local/ +``` +or: +``` +VERSION=SETOPERATORIMAGETAG make deploy-test +``` + +If you'd like to see the test pass: + 1. [install the operator](#installation) + 2. set up the [test environment](#test-environment) + 3. create a [sample cronjob](#trying-it-out) and be sure to manually trigger a job + 4. deploy the test using one of the commands above diff --git a/config/plugins/openshift_template_generator.rb b/config/plugins/openshift_template_generator.rb index 2e56a92..ffc6c9c 100755 --- a/config/plugins/openshift_template_generator.rb +++ b/config/plugins/openshift_template_generator.rb @@ -2,8 +2,8 @@ require 'yaml' -if ARGV[0].nil? - puts "Usage: #{$0} parameters.yaml" +if ARGV[0].nil? || ARGV[1].nil? + puts "Usage: #{$PROGRAM_NAME} parameters.yaml name_of_template" exit 1 end @@ -14,7 +14,7 @@ 'apiVersion' => 'v1', 'kind' => 'Template', 'metadata' => { - 'name' => 'floorist-operator' + 'name' => ARGV[1] }, 'objects' => objects, 'parameters' => parameters diff --git a/config/stage_test/kustomization.yaml b/config/stage_test/kustomization.yaml new file mode 100644 index 0000000..08219b9 --- /dev/null +++ b/config/stage_test/kustomization.yaml @@ -0,0 +1,17 @@ +namespace: floorist-operator-system + +resources: +- ./local + +images: +- name: quay.io/cloudservices/floorist-operator + newName: ${IMAGE} + newTag: ${IMAGE_TAG} + +patches: +- path: test_params.config.yaml + target: + group: batch + kind: Job + name: stage-test + version: v1 diff --git a/config/stage_test/local/kustomization.yaml b/config/stage_test/local/kustomization.yaml new file mode 100644 index 0000000..df3eccd --- /dev/null +++ b/config/stage_test/local/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Adds namespace to all resources. +namespace: default + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: floorist-operator- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +resources: +- stage_test_service_account.yaml +- stage_test_role.yaml +- stage_test_role_binding.yaml +- stage_test.yaml + +images: +- name: controller + newName: quay.io/cloudservices/floorist-operator + newTag: latest diff --git a/config/stage_test/local/stage_test.yaml b/config/stage_test/local/stage_test.yaml new file mode 100644 index 0000000..961424e --- /dev/null +++ b/config/stage_test/local/stage_test.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: stage-test +spec: + backoffLimit: 2 + template: + spec: + restartPolicy: Never + serviceAccountName: stage-test-service-account + containers: + - image: controller:latest + name: floorist-operator-stage-test + imagePullPolicy: Always + command: + - sh + args: + - stage_test.sh + resources: + requests: + memory: 256Mi + cpu: 200m + limits: + memory: 512Mi diff --git a/config/stage_test/local/stage_test_role.yaml b/config/stage_test/local/stage_test_role.yaml new file mode 100644 index 0000000..ed79109 --- /dev/null +++ b/config/stage_test/local/stage_test_role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stage-test-role +rules: +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch diff --git a/config/stage_test/local/stage_test_role_binding.yaml b/config/stage_test/local/stage_test_role_binding.yaml new file mode 100644 index 0000000..11c2eb5 --- /dev/null +++ b/config/stage_test/local/stage_test_role_binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stage-test-rolebinding +roleRef: + kind: Role + name: stage-test-role + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: stage-test-service-account diff --git a/config/stage_test/local/stage_test_service_account.yaml b/config/stage_test/local/stage_test_service_account.yaml new file mode 100644 index 0000000..743ad74 --- /dev/null +++ b/config/stage_test/local/stage_test_service_account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: stage-test-service-account diff --git a/config/stage_test/template_params.yaml b/config/stage_test/template_params.yaml new file mode 100644 index 0000000..44f3a96 --- /dev/null +++ b/config/stage_test/template_params.yaml @@ -0,0 +1,7 @@ +- description: Operator image + name: IMAGE + value: quay.io/cloudservices/floorist-operator +- description: Operator image tag + name: IMAGE_TAG + value: '' + required: true diff --git a/config/stage_test/test_params.config.yaml b/config/stage_test/test_params.config.yaml new file mode 100644 index 0000000..9ac08cf --- /dev/null +++ b/config/stage_test/test_params.config.yaml @@ -0,0 +1,3 @@ +- op: replace + path: /metadata/name + value: floorist-operator-stage-test-${IMAGE_TAG} diff --git a/deploy_template.yaml b/deploy_template.yaml index 8555ee2..b520973 100644 --- a/deploy_template.yaml +++ b/deploy_template.yaml @@ -68,7 +68,7 @@ objects: description: Floorist Query definition. properties: chunksize: - description: + description: type: integer prefix: description: Valid folder path that will be created under diff --git a/stage_test.sh b/stage_test.sh new file mode 100644 index 0000000..a281a84 --- /dev/null +++ b/stage_test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +CRONJOBS=$(oc get cronjobs | awk 'NR>1 {print $1}') + +if [[ -z "$CRONJOBS" ]]; then + echo "ERROR: no cronjobs found" + exit 1 +fi + +for CRONJOB in $CRONJOBS; do + SUCCESS=$(oc get job -l "pod=${CRONJOB}" -o jsonpath='{.items[].status.succeeded}{"\n"}') + + if [[ -z "$SUCCESS" ]]; then + echo "ERROR: cronjob $CRONJOB has not created any jobs" + exit 1 + fi + + if [[ "$SUCCESS" != "1" ]]; then + echo "ERROR: cronjob $CRONJOB has not created successful jobs" + exit 1 + fi +done diff --git a/stage_test_template.yaml b/stage_test_template.yaml new file mode 100644 index 0000000..f08759b --- /dev/null +++ b/stage_test_template.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: v1 +kind: Template +metadata: + name: floorist-operator-stage-test +objects: +- apiVersion: v1 + kind: ServiceAccount + metadata: + name: floorist-operator-stage-test-service-account + namespace: floorist-operator-system +- apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: floorist-operator-stage-test-role + namespace: floorist-operator-system + rules: + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: floorist-operator-stage-test-rolebinding + namespace: floorist-operator-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: floorist-operator-stage-test-role + subjects: + - kind: ServiceAccount + name: floorist-operator-stage-test-service-account + namespace: floorist-operator-system +- apiVersion: batch/v1 + kind: Job + metadata: + name: floorist-operator-stage-test-${IMAGE_TAG} + namespace: floorist-operator-system + spec: + backoffLimit: 2 + template: + spec: + containers: + - args: + - stage_test.sh + command: + - sh + image: "${IMAGE}:${IMAGE_TAG}" + imagePullPolicy: Always + name: floorist-operator-stage-test + resources: + limits: + memory: 512Mi + requests: + cpu: 200m + memory: 256Mi + restartPolicy: Never + serviceAccountName: floorist-operator-stage-test-service-account +parameters: +- description: Operator image + name: IMAGE + value: quay.io/cloudservices/floorist-operator +- description: Operator image tag + name: IMAGE_TAG + value: '' + required: true