Incremental version checks during migration

bin/ck8s

@@ -36,6 +36,7 @@ usage() {
   echo "  upgrade <wc|sc|both> <vX.Y> apply               runs all apply steps upgrading the environment" 1>&2
   echo "  upgrade <wc|sc|both> <vX.Y> prepare             runs all prepare steps upgrading the configuration" 1>&2
   echo "  validate <wc|sc>                                validates config files" 1>&2
+  echo "  version <wc|sc>                                 shows apps version" 1>&2
   exit 1
 }
 
@@ -153,5 +154,17 @@ diagnostics)
   shift
   "${here}/diagnostics.bash" "${@}"
   ;;
+version)
+  shift
+  case "${1:-both}" in
+  both)
+    log_info "sc version:" "$(get_version "sc")"
+    log_info "wc version:" "$(get_version "wc")"
+    ;;
+  sc | wc)
+    log_info "${1} version:" "$(get_version "${1}")"
+    ;;
+  esac
+  ;;
 *) usage ;;
 esac

bin/common.bash

@@ -636,3 +636,8 @@ with_s3cfg() {
   #       bucket needs to be created.
   sops_exec_file_no_fifo "${s3cfg}" 'S3COMMAND_CONFIG_FILE="{}" '"${*}"
 }
+
+# Retrieve apps version from configmap
+get_version() {
+  "${here}/ops.bash" kubectl "${1}" get cm -n kube-system apps-meta -o jsonpath --template='{.data.version}'
+}

bin/upgrade.bash

@@ -83,6 +83,18 @@ apply() {
     config_validate wc
   fi
 
+  # Create a configmap. This fails if done twice.
+  if [[ "${CK8S_CLUSTER:-}" =~ ^(sc|both)$ ]]; then
+    if ! record_migration_begin sc; then
+      log_fatal "prepare already started in sc (kubectl delete -n kube-system configmap apps-upgrade to try again)"
+    fi
+  fi
+  if [[ "${CK8S_CLUSTER:-}" =~ ^(wc|both)$ ]]; then
+    if ! record_migration_begin wc; then
+      log_fatal "prepare already started in wc (kubectl delete -n kube-system configmap apps-upgrade to try again)"
+    fi
+  fi
+
   # TODO: Template validation
   # for prefix in sc wc; do
   #   template_validate "${prefix}"
@@ -101,6 +113,12 @@ apply() {
     log_info "apply snippet \"${snippet##"${ROOT}/migration/"}\":"
     if "${snippet}" execute; then
       log_info "apply snippet success\n---"
+      if [[ "${CK8S_CLUSTER:-}" == "both" ]]; then
+        record_migration_step "sc" "${snippet}"
+        record_migration_step "wc" "${snippet}"
+      else
+        record_migration_step "${CK8S_CLUSTER}" "${snippet}"
+      fi
     else
       local return="${?}"
       log_error "apply snippet execute failure"
@@ -118,6 +136,13 @@ apply() {
       fi
     fi
   done
+
+  if [[ "${CK8S_CLUSTER:-}" =~ ^(sc|both)$ ]]; then
+    record_migration_done sc
+  fi
+  if [[ "${CK8S_CLUSTER:-}" =~ ^(wc|both)$ ]]; then
+    record_migration_done wc
+  fi
 }
 
 usage() {

scripts/migration/lib.sh

@@ -265,6 +265,7 @@ check_version() {
     return
   fi
 
+  # `--exit-status` can be used instead of comparing to "null"
   common_override=$(yq4 '.global.ck8sVersion' "${CK8S_CONFIG_PATH}/common-config.yaml")
   sc_wc_override=$(yq4 '.global.ck8sVersion' "${CK8S_CONFIG_PATH}/${1}-config.yaml")
   if [ "$common_override" != "null" ] || [ "$sc_wc_override" != "null" ]; then
@@ -281,6 +282,14 @@ check_version() {
     fi
   fi
 
+  cluster_version="$(get_apps_version "${1}" >/dev/null 2>&1 || true)"
+  if [ -n "${cluster_version}" ]; then
+    log_info "Currently running ${cluster_version} according to cluster"
+    if [ "${common_override}" != "null" ] && [ "${cluster_version%.*}" != "${common_override}" ]; then
+      log_warn "Version mismatch, cluster ${cluster_version}, config ${common_override}"
+    fi
+  fi
+
   if [[ ! "${VERSION["${1}-config"]}" =~ v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
     log_warn "reducing version validation of ${1}-config for version \"${VERSION["${1}-config"]}\""
   else
@@ -367,6 +376,46 @@ append_trap() {
   trap "$(new_trap)" "${signal}"
 }
 
+# usage: [[ "$(get_apps_version)" == "0.x" ]]
+get_apps_version() {
+  kubectl_do "${1}" get cm -n kube-system apps-meta -o jsonpath --template="{.data.version}"
+}
+
+# Usage: record_migration_begin sc|wc
+record_migration_begin() {
+  # This ConfigMap should only exist while doing an upgrade.
+  # Abort if it already exists
+  kubectl_do "${1}" create configmap --dry-run=client -o yaml \
+    -n kube-system apps-upgrade --from-literal "prepared=${CK8S_TARGET_VERSION}" 2>/dev/null |
+    yq4 '.metadata.labels["app.kubernetes.io/managed-by"] = "welck8s-apps-upgrade"' - |
+    kubectl_do "${1}" create -f - >/dev/null 2>/dev/null
+}
+
+# Get currently prepared version
+get_prepared_version() {
+  kubectl_do "${1}" get cm -n kube-system apps-upgrade -o jsonpath --template="{.data.prepared}" 2>/dev/null
+}
+
+# Usage: record_migration_step sc|wc step-description
+record_migration_step() {
+  kubectl_do "${1}" get -n kube-system cm apps-upgrade -o yaml 2>/dev/null |
+    last_step="${2##*/}" \
+      yq4 --exit-status 'select(.data.prepared == strenv(CK8S_TARGET_VERSION)) |
+      .data.last_step = strenv(last_step)' |
+    kubectl_do "${1}" replace -f - >/dev/null 2>/dev/null
+}
+
+# Usage: record_migration_done sc|wc
+record_migration_done() {
+  # Record the upgraded-to version. Create if it does not already exist.
+  kubectl_do "${1}" patch -n kube-system cm apps-meta --type=merge \
+    -p "$(yq4 --null-input --output-format json '.data.version = strenv(CK8S_TARGET_VERSION)')" 2>/dev/null ||
+    kubectl_do "${1}" create configmap -n kube-system apps-meta \
+      --from-literal "version=${CK8S_TARGET_VERSION}" 2>/dev/null
+  # Complete the migration.
+  kubectl_do "${1}" delete configmap -n kube-system apps-upgrade >/dev/null 2>/dev/null
+}
+
 # shellcheck source=scripts/migration/helm.sh
 source "${ROOT}/scripts/migration/helm.sh"
 # shellcheck source=scripts/migration/helmfile.sh