CASM-4908 Runtime container image signature validation

* Use NCN images which support HTTPS on PIT Nexus and "registry.local/*" > ["pit.nmn/*, "registry.local/*"] mirroring rule * Deploy Kyverno prepend-registry policy * Manually prepend registry.local/ to chart images, missed by prepend-registry Kyverno policy * Move Kyverno charts to separate manifest, deploy before any other chart * Temporarily override record for `registry.local` in CoreDNS configmap during fresh install, restore right after. * Stop supporting images such as "alpine:latest" - we need to know exactly how to mirror them, as docker.io/alpine or docker.io/library/alpine.
Cray-HPE · Oct 15, 2024 · ecf9aa0 · ecf9aa0
1 parent f1ec43c
commit ecf9aa0
Show file tree

Hide file tree

Showing 12 changed files with 363 additions and 105 deletions.
diff --git a/Jenkinsfile.github b/Jenkinsfile.github
@@ -47,8 +47,8 @@ pipeline {
         ARTIFACTORY = credentials('artifactory-algol60-readonly')
         PARALLEL_JOBS = "75%"
         SNYK_TOKEN = credentials('SNYK_TOKEN')
-        SLACK_CHANNEL_NOTIFY = "${isPublishedRelease() ? "#casm_release_management" : ""}"
-        SLACK_CHANNEL_ALERTS = "${isPublishedRelease() ? "#csm-release-alerts" : ""}"
+        SLACK_CHANNEL_NOTIFY = "${env.TAG_NAME ? "@mikhail.tupitsyn" : ""}"
+        SLACK_CHANNEL_ALERTS = ""
     }
 
     stages {
@@ -275,32 +275,37 @@ pipeline {
                 - Deprecated API usage: <${env.BUILD_URL}/artifact/dist/pluto-report.txt|pluto-report.txt>
                 - Snyk results: <${env.SNYK_RESULTS_SHEET_URL}|${env.SNYK_RESULTS_SHEET}> (raw scan results: <${env.SNYK_RESULTS_URL}|${env.SNYK_RESULTS_FILENAME}>)
                 """.stripIndent().trim())
-                if ( isPublishedRelease() ) {
+                if ( env.TAG_NAME ) {
                     // Fresh install current build on yasha
-                    build(job: "Cray-HPE/csm-vshasta-deploy/main", wait: false, parameters: [
-                        // Test fresh installs on yasha
-                        string(name: "ENVIRONMENT", value: "yasha"),
-                        // Install version which was just built
-                        string(name: "CSM_RELEASE", value: env.RELEASE_VERSION),
-                        // Where to report results
-                        string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY)
-                    ])
+                    // build(job: "Cray-HPE/csm-vshasta-deploy/main", wait: false, parameters: [
+                    //     // Test fresh installs on yasha
+                    //     string(name: "ENVIRONMENT", value: "scanlan"),
+                    //     // Install version which was just built
+                    //     string(name: "CSM_RELEASE", value: env.RELEASE_VERSION),
+                    //     // Where to report results
+                    //     string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY),
+                    //     string(name: "POSTINSTALL_DESTROY", value: "never"),
+                    //     string(name: "DOCS_CSM_VERSION", value: csmUtils.findDocsCsmByBranch("feature/prepend-registry"))
+                    // ])
                     // Upgrade last previous release to current build on vex
                     build(job: "Cray-HPE/csm-vshasta-deploy/main", wait: false, parameters: [
                         // Test upgrades on vex
-                        string(name: "ENVIRONMENT", value: "vex"),
+                        string(name: "ENVIRONMENT", value: "taryon"),
                         // Automatically evaluate last previous release e.g. 1.4.3
                         string(name: "CSM_RELEASE", value: ""),
                         // Upgrade to what had been just built
                         string(name: "CSM_RELEASE_UPGRADE", value: env.RELEASE_VERSION),
                         // Upgrade after fresh install
                         booleanParam(name: "UPGRADE", value: true),
                         // Where to report
-                        string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY)
-                    ])
-                    build(job: "Cray-HPE/csm-release-internal-upload/main", wait: false, parameters: [
-                        string(name: "RELEASE_VERSION", value: "${env.RELEASE_VERSION}"),
+                        string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY),
+                        string(name: "LGI_BRANCH", value: "feature/dont-scale-opa"),
+                        // string(name: "POSTINSTALL_DESTROY", value: "never"),
+                        string(name: "DOCS_CSM_VERSION_UPGRADE", value: csmUtils.findDocsCsmByBranch("feature/prepend-registry"))
                     ])
+                    // build(job: "Cray-HPE/csm-release-internal-upload/main", wait: false, parameters: [
+                    //     string(name: "RELEASE_VERSION", value: "${env.RELEASE_VERSION}"),
+                    // ])
                 }
             }
         }

diff --git a/assets.sh b/assets.sh
@@ -41,34 +41,34 @@ KERNEL_VERSION='6.4.0-150600.23.17-default'
 KERNEL_DEFAULT_DEBUGINFO_VERSION="${KERNEL_VERSION//-default/}.1"
 
 # The image ID may not always match the other images and should be defined individually.
-KUBERNETES_IMAGE_ID=6.2.20
+KUBERNETES_IMAGE_ID=cc0b830-1728683115554
 KUBERNETES_ASSETS=(
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/kubernetes/${KUBERNETES_IMAGE_ID}/kubernetes-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.squashfs"
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/kubernetes/${KUBERNETES_IMAGE_ID}/${KERNEL_VERSION}-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.kernel"
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/kubernetes/${KUBERNETES_IMAGE_ID}/initrd.img-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.xz"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/kubernetes/${KUBERNETES_IMAGE_ID}/kubernetes-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.squashfs"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/kubernetes/${KUBERNETES_IMAGE_ID}/${KERNEL_VERSION}-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.kernel"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/kubernetes/${KUBERNETES_IMAGE_ID}/initrd.img-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.xz"
 )
 
 # The image ID may not always match the other images and should be defined individually.
-PIT_IMAGE_ID=6.2.20
+PIT_IMAGE_ID=cc0b830-1728683115554
 PIT_ASSETS=(
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/pre-install-toolkit/${PIT_IMAGE_ID}/pre-install-toolkit-${PIT_IMAGE_ID}-${NCN_ARCH}.iso"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/pre-install-toolkit/${PIT_IMAGE_ID}/pre-install-toolkit-${PIT_IMAGE_ID}-${NCN_ARCH}.iso"
 )
 
 # The image ID may not always match the other images and should be defined individually.
-STORAGE_CEPH_IMAGE_ID=6.2.20
+STORAGE_CEPH_IMAGE_ID=cc0b830-1728683115554
 STORAGE_CEPH_ASSETS=(
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/storage-ceph-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.squashfs"
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/${KERNEL_VERSION}-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.kernel"
-    "https://artifactory.algol60.net/artifactory/csm-images/stable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/initrd.img-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.xz"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/storage-ceph-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.squashfs"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/${KERNEL_VERSION}-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.kernel"
+    "https://artifactory.algol60.net/artifactory/csm-images/unstable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/initrd.img-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.xz"
 )
 
 # The image ID may not always match the other images and should be defined individually.
-COMPUTE_IMAGE_ID=6.2.20
+COMPUTE_IMAGE_ID=cc0b830-1728683115554
 for arch in "${CN_ARCH[@]}"; do
     eval "COMPUTE_${arch}_ASSETS"=\( \
-        "https://artifactory.algol60.net/artifactory/csm-images/stable/compute/${COMPUTE_IMAGE_ID}/compute-${COMPUTE_IMAGE_ID}-${arch}.squashfs" \
-        "https://artifactory.algol60.net/artifactory/csm-images/stable/compute/${COMPUTE_IMAGE_ID}/${KERNEL_VERSION}-${COMPUTE_IMAGE_ID}-${arch}.kernel" \
-        "https://artifactory.algol60.net/artifactory/csm-images/stable/compute/${COMPUTE_IMAGE_ID}/initrd.img-${COMPUTE_IMAGE_ID}-${arch}.xz" \
+        "https://artifactory.algol60.net/artifactory/csm-images/unstable/compute/${COMPUTE_IMAGE_ID}/compute-${COMPUTE_IMAGE_ID}-${arch}.squashfs" \
+        "https://artifactory.algol60.net/artifactory/csm-images/unstable/compute/${COMPUTE_IMAGE_ID}/${KERNEL_VERSION}-${COMPUTE_IMAGE_ID}-${arch}.kernel" \
+        "https://artifactory.algol60.net/artifactory/csm-images/unstable/compute/${COMPUTE_IMAGE_ID}/initrd.img-${COMPUTE_IMAGE_ID}-${arch}.xz" \
     \)
 done
 

diff --git a/build/images/extract.sh b/build/images/extract.sh
@@ -95,7 +95,8 @@ function extract-images() {
     ## Second: attempt to extract images from fully templated manifests (avoiding CRDs)
 
     # cray-service chart refers to postgresql.connectionPooler image as .dockerImage
-    printf "%s\n" "$CHART_TEMPLATE" | yq e -N 'select(.kind? != "CustomResourceDefinition") | .. | (.image?, .dockerImage?) | select(type == "!!str")' | tee "${cacheflags[@]}" >> "$IMAGE_LIST_FILE"
+    # ClusterPolicy in kyverno-policies chart may have "image:" field which has nothing to do with image definitions
+    printf "%s\n" "$CHART_TEMPLATE" | yq e -N 'select(.kind? != "CustomResourceDefinition" and .kind? != "ClusterPolicy") | .. | (.image?, .dockerImage?) | select(type == "!!str")' | tee "${cacheflags[@]}" >> "$IMAGE_LIST_FILE"
 
     ## Third: support "{image: {repository: aaa, tag: bbb}}" construct from cray-sysmgmt-health chart
     printf "%s\n" "$CHART_TEMPLATE" | yq e -N 'select(.kind? != "CustomResourceDefinition") | .. | select(.image?|type == "!!map") | (.image.repository + ":" + .image.tag)' | tee "${cacheflags[@]}" >> "$IMAGE_LIST_FILE"

diff --git a/build/images/inspect.sh b/build/images/inspect.sh
@@ -11,36 +11,25 @@ function usage() {
     exit 255
 }
 
-function resolve_canonical() {
-    local image="${1#docker://}"
-    if [[ "${image}" != *.*:* ]]; then
-        # alpine:latest > docker.io/library/alpine:latest
-        echo "docker.io/library/${image}"
-    else
-        # nothing needs to be changed
-        echo "${image}"
-    fi
-}
-
-# All images must come from artifactory.algol60.net/csm-docker/stable. Otherwise, we
-# can't guarantee reproducibility of builds, when CSM_BASE_VERSION is set.
+# We don't pull external images into CSM, all images must come from artifactory.algol60.net and have
+# valid signature.
 function resolve_mirror() {
-    local image="${1#docker://}"
-    if [[ "$image" == artifactory.algol60.net/csm-docker/stable/* ]]; then
+    local image="${1}"
+    if [[ "$image" == artifactory.algol60.net/* ]]; then
         # nothing needs to be changed
         echo "${image}"
     else
         # docker.io/library/alpine:latest > artifactory.algol60.net/csm-docker/stable/docker.io/library/alpine:latest
         # quay.io/skopeo/stable:v1.4.1 > artifactory.algol60.net/csm-docker/stable/quay.io/skopeo/stable:v1.4.1
+        # quay.io/reactiveops/ci-images:v11-alpine > artifactory.algol60.net/csm-docker/stable/quay.io/reactiveops/ci-images:v11-alpine
         echo "artifactory.algol60.net/csm-docker/stable/${image}"
     fi
 }
 
 [[ $# -gt 0 ]] || usage
 
 while [[ $# -gt 0 ]]; do
-    # Resolve image to canonical form, e.g., alpine -> docker.io/library/alpine
-    image="$(resolve_canonical "${1#docker://}")"
+    image="${1#registry.local/}"
 
     # Resolve image as an artifactory.algol60.net mirror
     image_mirror="$(resolve_mirror "$image")"

diff --git a/docker/index.yaml b/docker/index.yaml
@@ -110,6 +110,16 @@ artifactory.algol60.net/csm-docker/stable:
     docker.io/zeromq/zeromq:
       - v4.0.5
 
+    # During 1.5 > 1.6 upgrade, pre-upgrade Helm hooks do restarts of etcd and kafka clusters.
+    # We need to include older 1.5 images again into 1.6, to re-upload them to Nexus, together
+    # with their signatures. Otherwise restarted services won't start, claiming absence of signatures.
+    docker.io/bitnami/etcd:
+      - 3.5.9-debian-11-r15-patch
+    docker.io/bitnami/bitnami-shell:
+      - 11-debian-11-r128
+    quay.io/strimzi/kafka:
+      - 0.27.1-noJSM-chainsaw-kafka-2.8.1
+
     # XXX Missing from a SPIRE chart?
     gcr.io/spiffe-io/oidc-discovery-provider:
       - 0.12.2

diff --git a/hack/embedded-repo.sh b/hack/embedded-repo.sh
@@ -12,9 +12,9 @@ if [ $# -ne 1 ] || ([ "${1}" != "--validate" ] && [ "${1}" != "--download" ]); t
     echo "Lists of packages and repo configurations, installed onto NCN images, "
     echo "are expected to be published along with NCN image files as:"
     echo ""
-    echo "    csm-images/stable/<ncn_type>/<ncn_version>/installed-<ncn_version>-<arch>.packages"
-    echo "    csm-images/stable/<ncn_type>/<ncn_version>/installed.deps-<ncn_version>-<arch>.packages"
-    echo "    csm-images/stable/<ncn_type>/<ncn_version>/installed-<ncn_version>-<arch>.repos"
+    echo "    csm-images/unstable/<ncn_type>/<ncn_version>/installed-<ncn_version>-<arch>.packages"
+    echo "    csm-images/unstable/<ncn_type>/<ncn_version>/installed.deps-<ncn_version>-<arch>.packages"
+    echo "    csm-images/unstable/<ncn_type>/<ncn_version>/installed-<ncn_version>-<arch>.repos"
     echo ""
     echo "With --validate, validate presence of all RPM packages in repositories."
     echo "With --download, download RPMs into ${BUILDDIR}/rpm/embedded, filtering out those which are alredy in ${BUILDDIR}/rpm,"
@@ -34,7 +34,7 @@ for LIST_TYPE in installed installed.deps; do
         "pre-install-toolkit/${PIT_IMAGE_ID}/${LIST_TYPE}-${PIT_IMAGE_ID}-${NCN_ARCH}.packages" \
         "kubernetes/${KUBERNETES_IMAGE_ID}/${LIST_TYPE}-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.packages" \
         "storage-ceph/${STORAGE_CEPH_IMAGE_ID}/${LIST_TYPE}-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.packages"; do
-            curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/stable/${LIST_URL}"
+            curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/unstable/${LIST_URL}"
     done
 done | tr '=' '-' | sort -u > "${TMPDIR}/ncn.rpm-list"
 
@@ -53,7 +53,7 @@ for REPOS_URL in \
     "pre-install-toolkit/${PIT_IMAGE_ID}/installed-${PIT_IMAGE_ID}-${NCN_ARCH}.repos" \
     "kubernetes/${KUBERNETES_IMAGE_ID}/installed-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.repos" \
     "storage-ceph/${STORAGE_CEPH_IMAGE_ID}/installed-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.repos"; do
-        curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/stable/${REPOS_URL}"
+        curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/unstable/${REPOS_URL}"
 done | grep -E '^baseurl=https://' \
      | sed -e 's/^baseurl=//' \
      | sed -e 's|https://[^@]*@|https://|' \

diff --git a/install.sh b/install.sh
@@ -74,6 +74,50 @@ function deploy() {
     loftsman ship --charts-path "${ROOTDIR}/helm" --manifest-path "$1"
 }
 
+# To support image signature validation etc, we have to address all images in cluster as registry.local/<image_name>.
+# Image names may be specified in that form in Helm chart or k8s manifest, or mutated automatically when pod is created
+# by "prepend-registry" Kyverno policy.
+#
+# We have a mirroring configuration in /etc/containerd/config.toml, which looks for images in PIT Nexus first, and then
+# in k8s Nexus, if PIT Nexus is not available. This covers image pulls during fresh installs, upgrade, node rebuild or runtime.
+# However, Kyverno image signature validation polocy does not honor these mirroring settings. Therefore, specifically for Kyverno
+# admission controller during fresh install, we need to point "registry.local" to "pit.nmn" endpoint. We do this by adding "hosts { ... }"
+# section into CoreDNS configmap. Once fresh install is complete, "hosts { ... }" section is removed from CoreDNS configmap.
+#
+function switch_registry_local_to_pit() {
+    local ip_address
+    local tmpdir
+    ip_address=$(ssh ncn-m002 cloud-init query ds | jq -r '.meta_data.Global.host_records[] | select(.aliases[]? == "pit.nmn") | .ip')
+    tmpdir=$(mktemp -d)
+    trap 'rm -Rf "${tmpdir}"' RETURN
+    kubectl -n kube-system get configmap coredns -o json | jq -r '.data.Corefile' > "${tmpdir}/Corefile.orig"
+    if grep -Eq 'hosts *{' "${tmpdir}/Corefile.orig"; then
+        echo "Coredns configmap is already patched"
+    else
+        echo "Patching coredns configmap ..."
+        awk '{gsub(/^\.:53 *\{/, "&\n    hosts {\n        '${ip_address}' registry.local\n        fallthrough\n    }", $0)}1' \
+            "${tmpdir}/Corefile.orig" > "${tmpdir}/Corefile"
+        kubectl -n kube-system create configmap coredns --from-file="${tmpdir}/Corefile" -o yaml --dry-run=client | kubectl replace -f -
+    fi
+}
+
+function switch_registry_local_to_k8s() {
+    local ip_address
+    local tmpdir
+    ip_address=$(ssh ncn-m002 cloud-init query ds | jq -r '.meta_data.Global.host_records[] | select(.aliases[]? == "registry.local") | .ip')
+    tmpdir=$(mktemp -d)
+    trap 'rm -Rf "${tmpdir}"' RETURN
+    echo "Restoring coredns configmap ..."
+    kubectl -n kube-system get configmap coredns -o json | jq -r '.data.Corefile' > "${tmpdir}/Corefile.orig"
+    awk '/hosts *\{/{stop=1} stop==0{print} /\}/{stop=0}' "${tmpdir}/Corefile.orig" > "${tmpdir}/Corefile"
+    kubectl -n kube-system create configmap coredns --from-file="${tmpdir}/Corefile" -o yaml --dry-run=client | kubectl replace -f -
+}
+
+switch_registry_local_to_pit
+
+# Deploy Kyverno to perform image registry mutation and signature validation for all other charts
+deploy "${BUILDDIR}/manifests/kyverno.yaml"
+
 # Deploy services critical for Nexus to run
 deploy "${BUILDDIR}/manifests/storage.yaml"
 deploy "${BUILDDIR}/manifests/platform.yaml"
@@ -120,6 +164,8 @@ if is_vshasta_node; then
     deploy "${BUILDDIR}/manifests/vshasta.yaml"
 fi
 
+switch_registry_local_to_k8s
+
 set +x
 cat >&2 <<EOF
 + CSM applications and services deployed