1001 kubernetes cronjobs

Makefile

@@ -516,7 +516,7 @@ run-rest-tests = $(foreach image,$(rest-tests),tests/$(image))
 # List of Lagoon Services needed for REST endpoint testing
 deployment-test-services-rest = $(deployment-test-services-main) rest2tasks
 .PHONY: $(run-rest-tests)
-$(run-rest-tests): minishift build/node__6-builder build/node__8-builder build/oc-build-deploy-dind $(foreach image,$(deployment-test-services-rest),build/$(image)) push-minishift
+$(run-rest-tests): minishift build/node__6-builder build/node__8-builder build/oc-build-deploy-dind build/broker-single $(foreach image,$(deployment-test-services-rest),build/$(image)) push-minishift
 		$(eval testname = $(subst tests/,,$@))
 		IMAGE_REPO=$(CI_BUILD_TAG) docker-compose -p $(CI_BUILD_TAG) up -d $(deployment-test-services-rest)
 		IMAGE_REPO=$(CI_BUILD_TAG) docker exec -i $$(docker-compose -p $(CI_BUILD_TAG) ps -q tests) ansible-playbook /ansible/tests/$(testname).yaml $(testparameter)

images/commons/lagoon/cronjob.sh

@@ -1,12 +1,12 @@
 #!/bin/sh
 
 exit_trap() {
-  # We want to exit with 0 so OpenShift does not try to restart us
-  # see https://docs.openshift.com/container-platform/3.6/dev_guide/jobs.html#creating-a-job-known-issues
-  exit 0
+  rv=$?
+  # TODO: Send Lagoon API information about our CronJob Success or Failure
+  exit $rv
 }
 
-# on exit, always call exit_trap to always exit with 0
+# on exit, always call exit_trap
 trap exit_trap EXIT
 
 echo "$(date --utc +%FT%TZ) CRONJOB: $@"

images/oc-build-deploy-dind/build-deploy-docker-compose.sh

@@ -11,6 +11,27 @@ function outputToYaml() {
   set -x
 }
 
+function cronScheduleMoreOftenThan15Minutes() {
+  #takes a unexpanded cron schedule, returns 0 if it's more often that 15 minutes
+  MINUTE=$(echo $1 | (read -a ARRAY; echo ${ARRAY[0]}) )
+  if [[ $MINUTE =~ ^(M|H|\*)\/([0-5]?[0-9])$ ]]; then
+    # Match found for M/xx, H/xx or */xx
+    # Check if xx is smaller than 15, which means this cronjob runs more often than every 15 minutes.
+    STEP=${BASH_REMATCH[2]}
+    if [ $STEP -le 15 ]; then
+      return 0
+    else
+      return 1
+    fi
+  elif [[ $MINUTE =~ ^\*$ ]]; then
+    # We are running every minute
+    return 0
+  else
+    # all other cases are more often than 15 minutes
+    return 1
+  fi
+}
+
 ##############################################
 ### PREPARATION
 ##############################################
@@ -726,7 +747,7 @@ do
   fi
 
   CRONJOB_COUNTER=0
-  CRONJOBS_ARRAY=()
+  CRONJOBS_ARRAY_INSIDE_POD=()   #crons run inside an existing pod more frequently than every 15 minutes
   while [ -n "$(cat .lagoon.yml | shyaml keys environments.${BRANCH//./\\.}.cronjobs.$CRONJOB_COUNTER 2> /dev/null)" ]
   do
 
@@ -739,41 +760,46 @@ do
       # Add this cronjob to the native cleanup array, this will remove native cronjobs at the end of this script
       NATIVE_CRONJOB_CLEANUP_ARRAY+=("cronjob-${SERVICE_NAME}-${CRONJOB_NAME}")
 
-      CRONJOB_SCHEDULE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.cronjobs.$CRONJOB_COUNTER.schedule)
+      CRONJOB_SCHEDULE_RAW=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.cronjobs.$CRONJOB_COUNTER.schedule)
+
       # Convert the Cronjob Schedule for additional features and better spread
-      CRONJOB_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "$CRONJOB_SCHEDULE")
+      CRONJOB_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "$CRONJOB_SCHEDULE_RAW")
       CRONJOB_COMMAND=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.cronjobs.$CRONJOB_COUNTER.command)
 
-      CRONJOBS_ARRAY+=("${CRONJOB_SCHEDULE} ${CRONJOB_COMMAND}")
+      if cronScheduleMoreOftenThan15Minutes "$CRONJOB_SCHEDULE_RAW" ; then
+        # If this cronjob is more often than 15 minutes, we run the cronjob inside the pod itself
+        CRONJOBS_ARRAY_INSIDE_POD+=("${CRONJOB_SCHEDULE} ${CRONJOB_COMMAND}")
+      else
+        # This cronjob runs less ofen than every 15 minutes, we create a kubernetes native cronjob for it.
+        OPENSHIFT_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/custom-cronjob.yml"
 
-    fi
+        if [ ! -f $OPENSHIFT_TEMPLATE ]; then
+          echo "No cronjob support for service '${SERVICE_NAME}' with type '${SERVICE_TYPE}', please contact the Lagoon maintainers to implement cronjob support"; exit 1;
+        else
 
-    let CRONJOB_COUNTER=CRONJOB_COUNTER+1
-  done
+          # Create a copy of TEMPLATE_PARAMETERS so we can restore it
+          NO_CRON_PARAMETERS=(${TEMPLATE_PARAMETERS[@]})
 
-  # Generate cronjobs if service type defines them
-  SERVICE_CRONJOB_FILE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/cronjobs.yml"
-  if [ -f $SERVICE_CRONJOB_FILE ]; then
-    CRONJOB_COUNTER=0
-    while [ -n "$(cat ${SERVICE_CRONJOB_FILE} | shyaml keys $CRONJOB_COUNTER 2> /dev/null)" ]
-    do
+          TEMPLATE_PARAMETERS+=(-p CRONJOB_NAME="${CRONJOB_NAME,,}")
+          TEMPLATE_PARAMETERS+=(-p CRONJOB_SCHEDULE="${CRONJOB_SCHEDULE}")
+          TEMPLATE_PARAMETERS+=(-p CRONJOB_COMMAND="${CRONJOB_COMMAND}")
 
-      CRONJOB_NAME=$(cat ${SERVICE_CRONJOB_FILE} | shyaml get-value $CRONJOB_COUNTER.name | sed "s/[^[:alnum:]-]/-/g" | sed "s/^-//g")
-      # Add this cronjob to the native cleanup array, this will remove native cronjobs at the end of this script
-      NATIVE_CRONJOB_CLEANUP_ARRAY+=("cronjob-${SERVICE_NAME}-${CRONJOB_NAME}")
+          . /oc-build-deploy/scripts/exec-openshift-resources-with-images.sh
 
-      CRONJOB_SCHEDULE=$(cat ${SERVICE_CRONJOB_FILE} | shyaml get-value $CRONJOB_COUNTER.schedule)
-      # Convert the Cronjob Schedule for additional features and better spread
-      CRONJOB_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "$CRONJOB_SCHEDULE")
-      CRONJOB_COMMAND=$(cat ${SERVICE_CRONJOB_FILE} | shyaml get-value $CRONJOB_COUNTER.command)
+          # restore template parameters without any cronjobs in them (allows to create a secondary cronjob, plus also any other templates)
+          TEMPLATE_PARAMETERS=(${NO_CRON_PARAMETERS[@]})
+
+        fi
+      fi
+    fi
+
+    let CRONJOB_COUNTER=CRONJOB_COUNTER+1
+  done
 
-      CRONJOBS_ARRAY+=("${CRONJOB_SCHEDULE} ${CRONJOB_COMMAND}")
-      let CRONJOB_COUNTER=CRONJOB_COUNTER+1
-    done
-  fi
 
-  if [[ ${#CRONJOBS_ARRAY[@]} -ge 1 ]]; then
-    CRONJOBS_ONELINE=$(printf "%s\\n" "${CRONJOBS_ARRAY[@]}")
+  # if there are cronjobs running inside pods, add them to the deploymentconfig.
+  if [[ ${#CRONJOBS_ARRAY_INSIDE_POD[@]} -ge 1 ]]; then
+    CRONJOBS_ONELINE=$(printf "%s\\n" "${CRONJOBS_ARRAY_INSIDE_POD[@]}")
     TEMPLATE_PARAMETERS+=(-p CRONJOBS="${CRONJOBS_ONELINE}")
   fi
 
@@ -885,17 +911,6 @@ do
   fi
 done
 
-##############################################
-### CLEANUP NATIVE CRONJOBS NOW RUNNING WITHIN CONTAINERS DIRECTLY
-##############################################
-
-for CRONJOB in "${NATIVE_CRONJOB_CLEANUP_ARRAY[@]}"
-do
-  if oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} get cronjob ${CRONJOB}; then
-    oc --insecure-skip-tls-verify -n ${OPENSHIFT_PROJECT} delete cronjob ${CRONJOB}
-  fi
-done
-
 ##############################################
 ### RUN POST-ROLLOUT tasks defined in .lagoon.yml
 ##############################################

images/oc-build-deploy-dind/build-deploy.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -x
 set -eo pipefail
+set -o noglob
 
 OPENSHIFT_REGISTRY=docker-registry.default.svc:5000
 OPENSHIFT_PROJECT=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)

images/oc-build-deploy-dind/openshift-templates/cli-persistent/custom-cronjob.yml

@@ -37,6 +37,18 @@ parameters:
   - name: PERSISTENT_STORAGE_NAME
     description: Name of the Persistent Volume Claim to use
     required: true
+  - name: PERSISTENT_STORAGE_CLASS
+    description: Name of the Storage Class to use
+    value: ''
+  - name: PERSISTENT_STORAGE_SIZE
+    description: Size of the Storage to request
+    value: '5Gi'
+  - name: DEPLOYMENT_STRATEGY
+    description: Strategy of Deploymentconfig
+    value: "Rolling"
+  - name: SERVICE_IMAGE
+    description: Pullable image of service
+    required: true
   - name: CRONJOB_NAME
     description: Name of this cronjob
     required: true
@@ -46,32 +58,42 @@ parameters:
   - name: CRONJOB_COMMAND
     description: Command of this cronjob
     required: true
-  - name: DEPLOYMENT_STRATEGY
-    description: Strategy of Deploymentconfig
-    value: "Rolling"
-  - name: SERVICE_IMAGE
-    description: Pullable image of service
-    required: true
 objects:
-- apiVersion: batch/v2alpha1
+- apiVersion: batch/v1beta1
   kind: CronJob
   metadata:
     name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME}
   spec:
     schedule: "${CRONJOB_SCHEDULE}"
     concurrencyPolicy: Forbid
-    successfulJobsHistoryLimit: 1
+    successfulJobsHistoryLimit: 0
     failedJobsHistoryLimit: 1
     jobTemplate:
+      metadata:
+        labels:
+          cronjob: ${CRONJOB_NAME}
+          branch: ${SAFE_BRANCH}
+          project: ${SAFE_PROJECT}
+          parent: ${SERVICE_NAME}
       spec:
+        backoffLimit: 1
         template:
           metadata:
             labels:
               cronjob: ${CRONJOB_NAME}
               branch: ${SAFE_BRANCH}
               project: ${SAFE_PROJECT}
-              parent: cronjob-${SERVICE_NAME}-${CRONJOB_NAME}
+              parent: ${SERVICE_NAME}
           spec:
+            tolerations:
+              - effect: NoSchedule
+                key: autoscaled
+                operator: Equal
+                value: 'true'
+              - effect: NoSchedule
+                key: appuio.ch/autoscaled
+                operator: Equal
+                value: 'true'
             volumes:
               - name: ${PERSISTENT_STORAGE_NAME}
                 persistentVolumeClaim:
@@ -81,28 +103,29 @@ objects:
                   defaultMode: 420
                   secretName: lagoon-sshkey
             containers:
-            - name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME}
-              image: ${SERVICE_IMAGE}
-              command:
-                - /lagoon/cronjob.sh
-                - "${CRONJOB_COMMAND}"
-              volumeMounts:
-              - name: ${PERSISTENT_STORAGE_NAME}
-                mountPath: ${PERSISTENT_STORAGE_PATH}
-              envFrom:
-              - configMapRef:
-                  name: lagoon-env
-              env:
-                ## LAGOON_GIT_SHA is injected directly and not loaded via `lagoon-env` config
-                ## This will cause the cli to redeploy on every deployment, even the files have not changed
-                - name: LAGOON_GIT_SHA
-                  value: ${LAGOON_GIT_SHA}
-                - name: SERVICE_NAME
-                  value: ${SERVICE_NAME}
-              volumeMounts:
-                - name: ${PERSISTENT_STORAGE_NAME}
-                  mountPath: ${PERSISTENT_STORAGE_PATH}
-                - mountPath: /var/run/secrets/lagoon/sshkey/
-                  name: lagoon-sshkey
-                  readOnly: true
-            restartPolicy: OnFailure
+              - image: ${SERVICE_IMAGE}
+                command:
+                  - /lagoon/cronjob.sh
+                  - "${CRONJOB_COMMAND}"
+                name: cronjob-${SERVICE_NAME}-${CRONJOB_NAME}
+                envFrom:
+                  - configMapRef:
+                      name: lagoon-env
+                env:
+                  ## LAGOON_GIT_SHA is injected directly and not loaded via `lagoon-env` config
+                  ## This will cause the cli to redeploy on every deployment, even the files have not changed
+                  - name: LAGOON_GIT_SHA
+                    value: ${LAGOON_GIT_SHA}
+                  - name: SERVICE_NAME
+                    value: ${SERVICE_NAME}
+                volumeMounts:
+                  - name: ${PERSISTENT_STORAGE_NAME}
+                    mountPath: ${PERSISTENT_STORAGE_PATH}
+                  - mountPath: /var/run/secrets/lagoon/sshkey/
+                    name: lagoon-sshkey
+                    readOnly: true
+                resources:
+                  requests:
+                    cpu: 10m
+                    memory: 10Mi
+            restartPolicy: Never