Skip to content

Commit

Permalink
feat: RHINENG-9589 add autopromotion pipeline tests
Browse files Browse the repository at this point in the history
  • Loading branch information
RoamingNoMaD authored and vkrizan committed Jul 16, 2024
1 parent 8fad0a3 commit 72aaacf
Show file tree
Hide file tree
Showing 16 changed files with 285 additions and 15 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/checkimages.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
skopeo inspect "docker://quay.io/cloudservices/floorist:$tag" > /dev/null
sed -i "/name: FLOORIST_IMAGE_TAG/{n;s/value: .*/value: '$tag'/}" \
config/templated/template_params.yaml
make openshift-template
make openshift-templates
- name: Commit changes (if any)
run: |
git config user.name 'Update-a-Bot'
Expand Down
7 changes: 7 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ USER root
RUN dnf -y upgrade && \
dnf -y clean all

RUN curl --output oc.tar.gz \
https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest/openshift-client-linux-amd64-rhel8.tar.gz && \
tar -xvf oc.tar.gz oc && \
mv oc /usr/local/bin/ && \
rm oc.tar.gz

USER ${USER_UID}

COPY requirements.yml ${HOME}/requirements.yml
Expand All @@ -16,3 +22,4 @@ RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \
COPY watches.yaml ${HOME}/watches.yaml
COPY roles/ ${HOME}/roles/
COPY playbooks/ ${HOME}/playbooks/
COPY stage_test.sh ${HOME}/stage_test.sh
29 changes: 27 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
# Build parameters
IMG_BUILD_PARAMS ?=

# OpenShift Template file
# OpenShift Template files
OPENSHIFT_TEMPLATE ?= deploy_template.yaml
OPENSHIFT_STAGE_TEST_TEMPLATE ?= stage_test_template.yaml

.PHONY: all
all: podman-build
Expand Down Expand Up @@ -112,10 +113,25 @@ deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/c
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/default | kubectl apply -f -

.PHONY: deploy-test
deploy-test: kustomize ## Deploy stage test to the K8s cluster specified in ~/.kube/config.
cd config/stage_test/local && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/stage_test/local | kubectl apply -f -

.PHONY: deploy-all
deploy-all: deploy deploy-test

.PHONY: undeploy
undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build config/default | kubectl delete -f -

.PHONY: undeploy-test
undeploy-test: ## Undeploy stage test from the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build config/stage_test/local | kubectl delete -f -

.PHONY: undeploy-all
undeploy-all: undeploy undeploy-test

minikube-secrets: config/minikube/.secrets/database.txt config/minikube/.secrets/minio.txt

config/minikube/.secrets/database.txt config/minikube/.secrets/minio.txt:
Expand Down Expand Up @@ -143,9 +159,18 @@ endif
.PHONY: openshift-template
openshift-template: kustomize
$(KUSTOMIZE) build config/templated | \
config/plugins/openshift_template_generator.rb config/templated/template_params.yaml \
config/plugins/openshift_template_generator.rb config/templated/template_params.yaml floorist-operator \
> "${OPENSHIFT_TEMPLATE}"

.PHONY: openshift-stage-test-template
openshift-stage-test-template: kustomize
$(KUSTOMIZE) build config/stage_test | \
config/plugins/openshift_template_generator.rb config/stage_test/template_params.yaml floorist-operator-stage-test \
> "${OPENSHIFT_STAGE_TEST_TEMPLATE}"

.PHONY: openshift-templates
openshift-templates: openshift-template openshift-stage-test-template

.PHONY: ansible-operator
ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator
ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist.
Expand Down
53 changes: 45 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ Kubernetes Operator to manage scheduling of metrics export with [Floorist](https
1. [Prerequisites](#prerequisites)
2. [Test environment](#test-environment)
3. [Build and deploy](#build-and-deploy)
4. [OpenShift template](#openshift-template)
4. [OpenShift templates](#openshift-templates)
5. [Trying it out](#trying-it-out)
6. [Stage test](#stage-test)


## Description
Expand Down Expand Up @@ -166,20 +167,32 @@ Replace `SETOPERATORIMAGETAG` with desired image tag for the operator.
Optionally, `IMAGE_TAG_BASE` can be set to use a custom container registry. For example `IMAGE_TAG_BASE=quay.io/yourusername/floorist-operator`.
### OpenShift template
### OpenShift templates
OpenShift utilizes [`Template`](https://docs.openshift.com/container-platform/4.7/openshift_images/using-templates.html)
resources.
Due to current limitation in some environments there was a `openshift-teplate` Makefile target created
along with [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool.
The `openshift-teplate` target generates an OpenShift `Template` out of kustomized resources
configured within [`config/templated/`](config/templated/kustomization.yaml).
Due to current limitation in some environments there was a `openshift-teplates` Makefile target created
along with the [`openshift_template_generator.rb`](config/plugins/openshift_template_generator.rb) tool.
The `openshift-teplates` target generates an OpenShift `Template` out of kustomized resources
configured within [`config/templated/`](config/templated/kustomization.yaml) and
[`config/stage_test/`](config/stage_test/kustomization.yaml) .
To (re)generate OpenShift template for this operator use:
To (re)generate OpenShift templates for this operator and it's test job use:
```
make openshift-templates
```
The results are written in the `deploy_template.yaml` and `stage_test_template.yaml` files.
It is also possible to (re)generate only the operator's or only the test's template:
To (re)generate the operator's template:
```
make openshift-template
```
The result is writte in the `deploy_template.yaml` file.
To (re)generate the test's template:
```
make openshift-stage-test-template
```
### Trying it out
Expand All @@ -205,3 +218,27 @@ Observe status of the worker pod:
```
minikube kubectl -- get pod -l 'job-name=floorist-floorplan-sample-exporter-manual'
```
### Stage test
For the purpose of testing before auto-promotion from staging to production environment we've developed a job with a
24h delay. This test job checks for the successful creation of cronjobs and jobs by the operator and it asserts
the successful completion of the jobs.
It is possible to use the test locally, altough with the following limitations:
- Job names are not unique: each time the test is re-ran it needs to be deleted first
To run the test locally:
```
minikube kubectl -- apply -k config/stage_test/local/
```
or:
```
VERSION=SETOPERATORIMAGETAG make deploy-test
```
If you'd like to see the test pass:
1. [install the operator](#installation)
2. set up the [test environment](#test-environment)
3. create a [sample cronjob](#trying-it-out) and be sure to manually trigger a job
4. deploy the test using one of the commands above
6 changes: 3 additions & 3 deletions config/plugins/openshift_template_generator.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

require 'yaml'

if ARGV[0].nil?
puts "Usage: #{$0} parameters.yaml"
if ARGV[0].nil? || ARGV[1].nil?
puts "Usage: #{$PROGRAM_NAME} parameters.yaml name_of_template"
exit 1
end

Expand All @@ -14,7 +14,7 @@
'apiVersion' => 'v1',
'kind' => 'Template',
'metadata' => {
'name' => 'floorist-operator'
'name' => ARGV[1]
},
'objects' => objects,
'parameters' => parameters
Expand Down
17 changes: 17 additions & 0 deletions config/stage_test/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
namespace: floorist-operator-system

resources:
- ./local

images:
- name: quay.io/cloudservices/floorist-operator
newName: ${IMAGE}
newTag: ${IMAGE_TAG}

patches:
- path: test_params.config.yaml
target:
group: batch
kind: Job
name: stage-test
version: v1
27 changes: 27 additions & 0 deletions config/stage_test/local/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

# Adds namespace to all resources.
namespace: default

# Value of this field is prepended to the
# names of all resources, e.g. a deployment named
# "wordpress" becomes "alices-wordpress".
# Note that it should also match with the prefix (text before '-') of the namespace
# field above.
namePrefix: floorist-operator-

# Labels to add to all resources and selectors.
#commonLabels:
# someName: someValue

resources:
- stage_test_service_account.yaml
- stage_test_role.yaml
- stage_test_role_binding.yaml
- stage_test.yaml

images:
- name: controller
newName: quay.io/cloudservices/floorist-operator
newTag: latest
24 changes: 24 additions & 0 deletions config/stage_test/local/stage_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: batch/v1
kind: Job
metadata:
name: stage-test
spec:
backoffLimit: 2
template:
spec:
restartPolicy: Never
serviceAccountName: stage-test-service-account
containers:
- image: controller:latest
name: floorist-operator-stage-test
imagePullPolicy: Always
command:
- sh
args:
- stage_test.sh
resources:
requests:
memory: 256Mi
cpu: 200m
limits:
memory: 512Mi
14 changes: 14 additions & 0 deletions config/stage_test/local/stage_test_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: stage-test-role
rules:
- apiGroups:
- batch
resources:
- cronjobs
- jobs
verbs:
- get
- list
- watch
11 changes: 11 additions & 0 deletions config/stage_test/local/stage_test_role_binding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: stage-test-rolebinding
roleRef:
kind: Role
name: stage-test-role
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: stage-test-service-account
4 changes: 4 additions & 0 deletions config/stage_test/local/stage_test_service_account.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: stage-test-service-account
7 changes: 7 additions & 0 deletions config/stage_test/template_params.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
- description: Operator image
name: IMAGE
value: quay.io/cloudservices/floorist-operator
- description: Operator image tag
name: IMAGE_TAG
value: ''
required: true
3 changes: 3 additions & 0 deletions config/stage_test/test_params.config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
- op: replace
path: /metadata/name
value: floorist-operator-stage-test-${IMAGE_TAG}
2 changes: 1 addition & 1 deletion deploy_template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ objects:
description: Floorist Query definition.
properties:
chunksize:
description:
description:
type: integer
prefix:
description: Valid folder path that will be created under
Expand Down
22 changes: 22 additions & 0 deletions stage_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash

CRONJOBS=$(oc get cronjobs | awk 'NR>1 {print $1}')

if [[ -z "$CRONJOBS" ]]; then
echo "ERROR: no cronjobs found"
exit 1
fi

for CRONJOB in $CRONJOBS; do
SUCCESS=$(oc get job -l "pod=${CRONJOB}" -o jsonpath='{.items[].status.succeeded}{"\n"}')

if [[ -z "$SUCCESS" ]]; then
echo "ERROR: cronjob $CRONJOB has not created any jobs"
exit 1
fi

if [[ "$SUCCESS" != "1" ]]; then
echo "ERROR: cronjob $CRONJOB has not created successful jobs"
exit 1
fi
done
Loading

0 comments on commit 72aaacf

Please sign in to comment.