diff --git a/Jenkinsfile.github b/Jenkinsfile.github index 3d5548560..57800881c 100644 --- a/Jenkinsfile.github +++ b/Jenkinsfile.github @@ -47,8 +47,8 @@ pipeline { ARTIFACTORY = credentials('artifactory-algol60-readonly') PARALLEL_JOBS = "75%" SNYK_TOKEN = credentials('SNYK_TOKEN') - SLACK_CHANNEL_NOTIFY = "${isPublishedRelease() ? "#casm_release_management" : ""}" - SLACK_CHANNEL_ALERTS = "${isPublishedRelease() ? "#csm-release-alerts" : ""}" + SLACK_CHANNEL_NOTIFY = "${env.TAG_NAME ? "@mikhail.tupitsyn" : ""}" + SLACK_CHANNEL_ALERTS = "" } stages { @@ -275,20 +275,22 @@ pipeline { - Deprecated API usage: <${env.BUILD_URL}/artifact/dist/pluto-report.txt|pluto-report.txt> - Snyk results: <${env.SNYK_RESULTS_SHEET_URL}|${env.SNYK_RESULTS_SHEET}> (raw scan results: <${env.SNYK_RESULTS_URL}|${env.SNYK_RESULTS_FILENAME}>) """.stripIndent().trim()) - if ( isPublishedRelease() ) { + if ( env.TAG_NAME ) { // Fresh install current build on yasha - build(job: "Cray-HPE/csm-vshasta-deploy/main", wait: false, parameters: [ - // Test fresh installs on yasha - string(name: "ENVIRONMENT", value: "yasha"), - // Install version which was just built - string(name: "CSM_RELEASE", value: env.RELEASE_VERSION), - // Where to report results - string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY) - ]) + // build(job: "Cray-HPE/csm-vshasta-deploy/main", wait: false, parameters: [ + // // Test fresh installs on yasha + // string(name: "ENVIRONMENT", value: "scanlan"), + // // Install version which was just built + // string(name: "CSM_RELEASE", value: env.RELEASE_VERSION), + // // Where to report results + // string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY), + // string(name: "POSTINSTALL_DESTROY", value: "never"), + // string(name: "DOCS_CSM_VERSION", value: csmUtils.findDocsCsmByBranch("feature/prepend-registry")) + // ]) // Upgrade last previous release to current build on vex build(job: "Cray-HPE/csm-vshasta-deploy/main", wait: false, parameters: [ // Test upgrades on vex - string(name: "ENVIRONMENT", value: "vex"), + string(name: "ENVIRONMENT", value: "taryon"), // Automatically evaluate last previous release e.g. 1.4.3 string(name: "CSM_RELEASE", value: ""), // Upgrade to what had been just built @@ -296,11 +298,14 @@ pipeline { // Upgrade after fresh install booleanParam(name: "UPGRADE", value: true), // Where to report - string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY) - ]) - build(job: "Cray-HPE/csm-release-internal-upload/main", wait: false, parameters: [ - string(name: "RELEASE_VERSION", value: "${env.RELEASE_VERSION}"), + string(name: "SLACK_REPORT_CHANNEL", value: env.SLACK_CHANNEL_NOTIFY), + string(name: "LGI_BRANCH", value: "feature/dont-scale-opa"), + // string(name: "POSTINSTALL_DESTROY", value: "never"), + string(name: "DOCS_CSM_VERSION_UPGRADE", value: csmUtils.findDocsCsmByBranch("feature/prepend-registry")) ]) + // build(job: "Cray-HPE/csm-release-internal-upload/main", wait: false, parameters: [ + // string(name: "RELEASE_VERSION", value: "${env.RELEASE_VERSION}"), + // ]) } } } diff --git a/assets.sh b/assets.sh index cc06c75f6..f3391a1c2 100755 --- a/assets.sh +++ b/assets.sh @@ -41,34 +41,34 @@ KERNEL_VERSION='6.4.0-150600.23.17-default' KERNEL_DEFAULT_DEBUGINFO_VERSION="${KERNEL_VERSION//-default/}.1" # The image ID may not always match the other images and should be defined individually. -KUBERNETES_IMAGE_ID=6.2.20 +KUBERNETES_IMAGE_ID=cc0b830-1728683115554 KUBERNETES_ASSETS=( - "https://artifactory.algol60.net/artifactory/csm-images/stable/kubernetes/${KUBERNETES_IMAGE_ID}/kubernetes-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.squashfs" - "https://artifactory.algol60.net/artifactory/csm-images/stable/kubernetes/${KUBERNETES_IMAGE_ID}/${KERNEL_VERSION}-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.kernel" - "https://artifactory.algol60.net/artifactory/csm-images/stable/kubernetes/${KUBERNETES_IMAGE_ID}/initrd.img-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.xz" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/kubernetes/${KUBERNETES_IMAGE_ID}/kubernetes-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.squashfs" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/kubernetes/${KUBERNETES_IMAGE_ID}/${KERNEL_VERSION}-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.kernel" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/kubernetes/${KUBERNETES_IMAGE_ID}/initrd.img-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.xz" ) # The image ID may not always match the other images and should be defined individually. -PIT_IMAGE_ID=6.2.20 +PIT_IMAGE_ID=cc0b830-1728683115554 PIT_ASSETS=( - "https://artifactory.algol60.net/artifactory/csm-images/stable/pre-install-toolkit/${PIT_IMAGE_ID}/pre-install-toolkit-${PIT_IMAGE_ID}-${NCN_ARCH}.iso" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/pre-install-toolkit/${PIT_IMAGE_ID}/pre-install-toolkit-${PIT_IMAGE_ID}-${NCN_ARCH}.iso" ) # The image ID may not always match the other images and should be defined individually. -STORAGE_CEPH_IMAGE_ID=6.2.20 +STORAGE_CEPH_IMAGE_ID=cc0b830-1728683115554 STORAGE_CEPH_ASSETS=( - "https://artifactory.algol60.net/artifactory/csm-images/stable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/storage-ceph-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.squashfs" - "https://artifactory.algol60.net/artifactory/csm-images/stable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/${KERNEL_VERSION}-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.kernel" - "https://artifactory.algol60.net/artifactory/csm-images/stable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/initrd.img-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.xz" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/storage-ceph-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.squashfs" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/${KERNEL_VERSION}-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.kernel" + "https://artifactory.algol60.net/artifactory/csm-images/unstable/storage-ceph/${STORAGE_CEPH_IMAGE_ID}/initrd.img-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.xz" ) # The image ID may not always match the other images and should be defined individually. -COMPUTE_IMAGE_ID=6.2.20 +COMPUTE_IMAGE_ID=cc0b830-1728683115554 for arch in "${CN_ARCH[@]}"; do eval "COMPUTE_${arch}_ASSETS"=\( \ - "https://artifactory.algol60.net/artifactory/csm-images/stable/compute/${COMPUTE_IMAGE_ID}/compute-${COMPUTE_IMAGE_ID}-${arch}.squashfs" \ - "https://artifactory.algol60.net/artifactory/csm-images/stable/compute/${COMPUTE_IMAGE_ID}/${KERNEL_VERSION}-${COMPUTE_IMAGE_ID}-${arch}.kernel" \ - "https://artifactory.algol60.net/artifactory/csm-images/stable/compute/${COMPUTE_IMAGE_ID}/initrd.img-${COMPUTE_IMAGE_ID}-${arch}.xz" \ + "https://artifactory.algol60.net/artifactory/csm-images/unstable/compute/${COMPUTE_IMAGE_ID}/compute-${COMPUTE_IMAGE_ID}-${arch}.squashfs" \ + "https://artifactory.algol60.net/artifactory/csm-images/unstable/compute/${COMPUTE_IMAGE_ID}/${KERNEL_VERSION}-${COMPUTE_IMAGE_ID}-${arch}.kernel" \ + "https://artifactory.algol60.net/artifactory/csm-images/unstable/compute/${COMPUTE_IMAGE_ID}/initrd.img-${COMPUTE_IMAGE_ID}-${arch}.xz" \ \) done diff --git a/build/images/extract.sh b/build/images/extract.sh index 64fba9cfa..902208693 100755 --- a/build/images/extract.sh +++ b/build/images/extract.sh @@ -95,7 +95,8 @@ function extract-images() { ## Second: attempt to extract images from fully templated manifests (avoiding CRDs) # cray-service chart refers to postgresql.connectionPooler image as .dockerImage - printf "%s\n" "$CHART_TEMPLATE" | yq e -N 'select(.kind? != "CustomResourceDefinition") | .. | (.image?, .dockerImage?) | select(type == "!!str")' | tee "${cacheflags[@]}" >> "$IMAGE_LIST_FILE" + # ClusterPolicy in kyverno-policies chart may have "image:" field which has nothing to do with image definitions + printf "%s\n" "$CHART_TEMPLATE" | yq e -N 'select(.kind? != "CustomResourceDefinition" and .kind? != "ClusterPolicy") | .. | (.image?, .dockerImage?) | select(type == "!!str")' | tee "${cacheflags[@]}" >> "$IMAGE_LIST_FILE" ## Third: support "{image: {repository: aaa, tag: bbb}}" construct from cray-sysmgmt-health chart printf "%s\n" "$CHART_TEMPLATE" | yq e -N 'select(.kind? != "CustomResourceDefinition") | .. | select(.image?|type == "!!map") | (.image.repository + ":" + .image.tag)' | tee "${cacheflags[@]}" >> "$IMAGE_LIST_FILE" diff --git a/build/images/inspect.sh b/build/images/inspect.sh index d0702597c..82e2eee8f 100755 --- a/build/images/inspect.sh +++ b/build/images/inspect.sh @@ -11,27 +11,17 @@ function usage() { exit 255 } -function resolve_canonical() { - local image="${1#docker://}" - if [[ "${image}" != *.*:* ]]; then - # alpine:latest > docker.io/library/alpine:latest - echo "docker.io/library/${image}" - else - # nothing needs to be changed - echo "${image}" - fi -} - -# All images must come from artifactory.algol60.net/csm-docker/stable. Otherwise, we -# can't guarantee reproducibility of builds, when CSM_BASE_VERSION is set. +# We don't pull external images into CSM, all images must come from artifactory.algol60.net and have +# valid signature. function resolve_mirror() { - local image="${1#docker://}" - if [[ "$image" == artifactory.algol60.net/csm-docker/stable/* ]]; then + local image="${1}" + if [[ "$image" == artifactory.algol60.net/* ]]; then # nothing needs to be changed echo "${image}" else # docker.io/library/alpine:latest > artifactory.algol60.net/csm-docker/stable/docker.io/library/alpine:latest # quay.io/skopeo/stable:v1.4.1 > artifactory.algol60.net/csm-docker/stable/quay.io/skopeo/stable:v1.4.1 + # quay.io/reactiveops/ci-images:v11-alpine > artifactory.algol60.net/csm-docker/stable/quay.io/reactiveops/ci-images:v11-alpine echo "artifactory.algol60.net/csm-docker/stable/${image}" fi } @@ -39,8 +29,7 @@ function resolve_mirror() { [[ $# -gt 0 ]] || usage while [[ $# -gt 0 ]]; do - # Resolve image to canonical form, e.g., alpine -> docker.io/library/alpine - image="$(resolve_canonical "${1#docker://}")" + image="${1#registry.local/}" # Resolve image as an artifactory.algol60.net mirror image_mirror="$(resolve_mirror "$image")" diff --git a/docker/index.yaml b/docker/index.yaml index 584e9b5b4..8dda78678 100644 --- a/docker/index.yaml +++ b/docker/index.yaml @@ -110,6 +110,16 @@ artifactory.algol60.net/csm-docker/stable: docker.io/zeromq/zeromq: - v4.0.5 + # During 1.5 > 1.6 upgrade, pre-upgrade Helm hooks do restarts of etcd and kafka clusters. + # We need to include older 1.5 images again into 1.6, to re-upload them to Nexus, together + # with their signatures. Otherwise restarted services won't start, claiming absence of signatures. + docker.io/bitnami/etcd: + - 3.5.9-debian-11-r15-patch + docker.io/bitnami/bitnami-shell: + - 11-debian-11-r128 + quay.io/strimzi/kafka: + - 0.27.1-noJSM-chainsaw-kafka-2.8.1 + # XXX Missing from a SPIRE chart? gcr.io/spiffe-io/oidc-discovery-provider: - 0.12.2 diff --git a/hack/embedded-repo.sh b/hack/embedded-repo.sh index 1dda0af34..c691933cc 100755 --- a/hack/embedded-repo.sh +++ b/hack/embedded-repo.sh @@ -12,9 +12,9 @@ if [ $# -ne 1 ] || ([ "${1}" != "--validate" ] && [ "${1}" != "--download" ]); t echo "Lists of packages and repo configurations, installed onto NCN images, " echo "are expected to be published along with NCN image files as:" echo "" - echo " csm-images/stable///installed--.packages" - echo " csm-images/stable///installed.deps--.packages" - echo " csm-images/stable///installed--.repos" + echo " csm-images/unstable///installed--.packages" + echo " csm-images/unstable///installed.deps--.packages" + echo " csm-images/unstable///installed--.repos" echo "" echo "With --validate, validate presence of all RPM packages in repositories." echo "With --download, download RPMs into ${BUILDDIR}/rpm/embedded, filtering out those which are alredy in ${BUILDDIR}/rpm," @@ -34,7 +34,7 @@ for LIST_TYPE in installed installed.deps; do "pre-install-toolkit/${PIT_IMAGE_ID}/${LIST_TYPE}-${PIT_IMAGE_ID}-${NCN_ARCH}.packages" \ "kubernetes/${KUBERNETES_IMAGE_ID}/${LIST_TYPE}-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.packages" \ "storage-ceph/${STORAGE_CEPH_IMAGE_ID}/${LIST_TYPE}-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.packages"; do - curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/stable/${LIST_URL}" + curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/unstable/${LIST_URL}" done done | tr '=' '-' | sort -u > "${TMPDIR}/ncn.rpm-list" @@ -53,7 +53,7 @@ for REPOS_URL in \ "pre-install-toolkit/${PIT_IMAGE_ID}/installed-${PIT_IMAGE_ID}-${NCN_ARCH}.repos" \ "kubernetes/${KUBERNETES_IMAGE_ID}/installed-${KUBERNETES_IMAGE_ID}-${NCN_ARCH}.repos" \ "storage-ceph/${STORAGE_CEPH_IMAGE_ID}/installed-${STORAGE_CEPH_IMAGE_ID}-${NCN_ARCH}.repos"; do - curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/stable/${REPOS_URL}" + curl -Ss -f -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" "https://artifactory.algol60.net/artifactory/csm-images/unstable/${REPOS_URL}" done | grep -E '^baseurl=https://' \ | sed -e 's/^baseurl=//' \ | sed -e 's|https://[^@]*@|https://|' \ diff --git a/install.sh b/install.sh index b5b4fd3cb..254dd17ea 100755 --- a/install.sh +++ b/install.sh @@ -74,6 +74,50 @@ function deploy() { loftsman ship --charts-path "${ROOTDIR}/helm" --manifest-path "$1" } +# To support image signature validation etc, we have to address all images in cluster as registry.local/. +# Image names may be specified in that form in Helm chart or k8s manifest, or mutated automatically when pod is created +# by "prepend-registry" Kyverno policy. +# +# We have a mirroring configuration in /etc/containerd/config.toml, which looks for images in PIT Nexus first, and then +# in k8s Nexus, if PIT Nexus is not available. This covers image pulls during fresh installs, upgrade, node rebuild or runtime. +# However, Kyverno image signature validation polocy does not honor these mirroring settings. Therefore, specifically for Kyverno +# admission controller during fresh install, we need to point "registry.local" to "pit.nmn" endpoint. We do this by adding "hosts { ... }" +# section into CoreDNS configmap. Once fresh install is complete, "hosts { ... }" section is removed from CoreDNS configmap. +# +function switch_registry_local_to_pit() { + local ip_address + local tmpdir + ip_address=$(ssh ncn-m002 cloud-init query ds | jq -r '.meta_data.Global.host_records[] | select(.aliases[]? == "pit.nmn") | .ip') + tmpdir=$(mktemp -d) + trap 'rm -Rf "${tmpdir}"' RETURN + kubectl -n kube-system get configmap coredns -o json | jq -r '.data.Corefile' > "${tmpdir}/Corefile.orig" + if grep -Eq 'hosts *{' "${tmpdir}/Corefile.orig"; then + echo "Coredns configmap is already patched" + else + echo "Patching coredns configmap ..." + awk '{gsub(/^\.:53 *\{/, "&\n hosts {\n '${ip_address}' registry.local\n fallthrough\n }", $0)}1' \ + "${tmpdir}/Corefile.orig" > "${tmpdir}/Corefile" + kubectl -n kube-system create configmap coredns --from-file="${tmpdir}/Corefile" -o yaml --dry-run=client | kubectl replace -f - + fi +} + +function switch_registry_local_to_k8s() { + local ip_address + local tmpdir + ip_address=$(ssh ncn-m002 cloud-init query ds | jq -r '.meta_data.Global.host_records[] | select(.aliases[]? == "registry.local") | .ip') + tmpdir=$(mktemp -d) + trap 'rm -Rf "${tmpdir}"' RETURN + echo "Restoring coredns configmap ..." + kubectl -n kube-system get configmap coredns -o json | jq -r '.data.Corefile' > "${tmpdir}/Corefile.orig" + awk '/hosts *\{/{stop=1} stop==0{print} /\}/{stop=0}' "${tmpdir}/Corefile.orig" > "${tmpdir}/Corefile" + kubectl -n kube-system create configmap coredns --from-file="${tmpdir}/Corefile" -o yaml --dry-run=client | kubectl replace -f - +} + +switch_registry_local_to_pit + +# Deploy Kyverno to perform image registry mutation and signature validation for all other charts +deploy "${BUILDDIR}/manifests/kyverno.yaml" + # Deploy services critical for Nexus to run deploy "${BUILDDIR}/manifests/storage.yaml" deploy "${BUILDDIR}/manifests/platform.yaml" @@ -120,6 +164,8 @@ if is_vshasta_node; then deploy "${BUILDDIR}/manifests/vshasta.yaml" fi +switch_registry_local_to_k8s + set +x cat >&2 <