diff --git a/infra/gcp/ensure-conformance-storage.sh b/infra/gcp/ensure-conformance-storage.sh
index 84e99aa2512..3953a69b331 100755
--- a/infra/gcp/ensure-conformance-storage.sh
+++ b/infra/gcp/ensure-conformance-storage.sh
@@ -43,100 +43,113 @@ function usage() {
 
 PROJECT="k8s-conform"
 
+CONFORMANCE_SERVICES=(
+    # secretmanager to host service-account keys
+    secretmanager.googleapis.com
+    # storage-api to store results in GCS via JSON API
+    storage-api.googleapis.com
+    # storage-component to store results in GCS via XML API
+    storage-component.googleapis.com
+)
+
+readonly CONFORMANCE_RETENTION="10y"
+
+# "Offering" comes from https://github.com/cncf/k8s-conformance/blob/master/terms-conditions/Certified_Kubernetes_Terms.md
 # NB: Please keep this sorted.
-CONFORMANCE_BUCKETS=(
+CONFORMANCE_OFFERINGS=(
     cri-o
     huaweicloud
     inspur
     provider-openstack
     s390x-k8s
 )
+
 if [ $# = 0 ]; then
     # default to all conformance buckets
-    set -- "${CONFORMANCE_BUCKETS[@]}"
+    set -- "${CONFORMANCE_OFFERINGS[@]}"
 fi
 
 # Make the project, if needed
-color 6 "Ensuring project exists: ${PROJECT}"
-ensure_project "${PROJECT}"
+function ensure_conformance_project() {
+    color 6 "Ensuring project exists: ${PROJECT}"
+    ensure_project "${PROJECT}"
+
+    # Enable GCS APIs
+    color 6 "Ensuring only necessary services are enabled for conformance project: ${PROJECT}"
+    ensure_only_services "${PROJECT}" "${CONFORMANCE_SERVICES[@]}"
+}
+
+# ensure_conformance_bucket ensure that:
+# - the given bucket exists and is publicly readable
+# - the given bucket has the conformance retention policy setup
+# - the given bucket is writable by the given group
+function ensure_conformance_bucket() {
+    local bucket="${1}"
+    local writers="${2}"
+
+    color 6 "Ensuring ${PROJECT} contains world readadble GCS bucket: ${bucket}"
+    ensure_public_gcs_bucket "${PROJECT}" "${bucket}"
+
+    color 6 "Empowering GCS admins for GCS bucket: ${bucket}"
+    empower_gcs_admins "${PROJECT}" "${bucket}"
+
+    color 6 "Ensuring ${bucket} retention policy is set to: ${CONFORMANCE_RETENTION}"
+    ensure_gcs_bucket_retention "${bucket}" "${CONFORMANCE_RETENTION}"
+
+    color 6 "Empowering ${writers} to write to GCS bucket: ${bucket}"
+    empower_group_to_write_gcs_bucket "${writers}" "${bucket}"
+
+}
+
+# ensure_conformance_serviceaccount ensures that:
+# - a serviceaccount of the given name exists in PROJECT
+# - it can write to the given bucket
+# - it has a private key stored in a secret in PROJECT accessible to the given group
+function ensure_conformance_serviceaccount() {
+    local name="${1}"
+    local bucket="${2}"
+    local secret_accessors="${3}"
+
+    local email="$(svc_acct_email "${PROJECT}" "${name}")"
+    local secret="${name}-key"
+    local private_key_file="${TMPDIR}/key.json"
+
+    color 6 "Ensuring service account exists: ${email}"
+    ensure_service_account "${PROJECT}"  "${name}" "Grants write access to ${bucket}"
+
+    color 6 "Ensuring ${PROJECT} contains secret ${secret} with private key for ${email}"
+    ensure_serviceaccount_key_secret "${PROJECT}" "${secret}" "${email}"
+
+    color 6 "Empowering ${secret_accessors} to access secret: ${secret}"
+    ensure_secret_role_binding \
+        "projects/${PROJECT}/secrets/${secret}" \
+        "group:${secret_accessors}" \
+        "roles/secretmanager.secretAccessor"
+
+    color 6 "Empowering ${email} to write to ${bucket}"
+    empower_svcacct_to_write_gcs_bucket "${email}"  "${bucket}"
+}
 
-# Enable GCS APIs
-color 6 "Ensuring only necessary services are enabled for conformance project: ${PROJECT}"
-ensure_only_services "${PROJECT}" \
-    storage-component.googleapis.com \
-    secretmanager.googleapis.com \
+ensure_conformance_project
 
 color 6 "Ensuring all conformance buckets"
-for REPO; do
-    color 3 "Configuring conformance bucket for ${REPO}"
-
-    # The group that can write to this conformance repo.
-    BUCKET_WRITERS="k8s-infra-conform-${REPO}@kubernetes.io"
-
-    # The names of the buckets
-    BUCKET="gs://${PROJECT}-${REPO}" # used by humans
-
-    # Every project gets some GCS buckets
-    color 3 "Configuring bucket: ${BUCKET}"
-
-    # Create the bucket
-    color 6 "Ensuring the bucket exists and is world readable"
-    ensure_public_gcs_bucket "${PROJECT}" "${BUCKET}"
-
-    color 6 "Ensuring the GCS bucket retention policy is set: ${PROJECT}"
-    RETENTION="10y"
-    ensure_gcs_bucket_retention "${BUCKET}" "${RETENTION}"
-
-    # Enable admins on the bucket
-    color 6 "Empowering GCS admins"
-    empower_gcs_admins "${PROJECT}" "${BUCKET}"
-
-    # Enable writers on the bucket
-    color 6 "Empowering ${BUCKET_WRITERS} to GCS"
-    empower_group_to_write_gcs_bucket "${BUCKET_WRITERS}" "${BUCKET}"
-
-    (
-        readonly SERVICE_ACCOUNT_NAME="service-${REPO}"
-        readonly SERVICE_ACCOUNT_EMAIL="$(svc_acct_email "${PROJECT}" \
-            "${SERVICE_ACCOUNT_NAME}")"
-        readonly SECRET_ID="${SERVICE_ACCOUNT_NAME}-key"
-        readonly KEY_FILE="${TMPDIR}/key.json"
-
-        if ! gcloud iam service-accounts describe "${SERVICE_ACCOUNT_EMAIL}" \
-            --project "${PROJECT}" >/dev/null 2>&1
-        then
-            color 6 "Creating service account: ${SERVICE_ACCOUNT_NAME}"
-            ensure_service_account \
-                "${PROJECT}" \
-                "${SERVICE_ACCOUNT_NAME}" \
-                "${SERVICE_ACCOUNT_NAME}"
-
-            color 6 "Empowering service account: ${SERVICE_ACCOUNT_NAME} to GCS"
-            empower_svcacct_to_write_gcs_bucket \
-                "${SERVICE_ACCOUNT_EMAIL}" \
-                "${BUCKET}"
-
-            color 6 "Creating private key for service account: ${SERVICE_ACCOUNT_NAME}"
-            gcloud iam service-accounts keys create "${KEY_FILE}" \
-                --project "${PROJECT}" \
-                --iam-account "${SERVICE_ACCOUNT_EMAIL}"
-
-            color 6 "Creating secret to store private key"
-            gcloud secrets create "${SECRET_ID}" \
-                --project "${PROJECT}" \
-                --replication-policy "automatic"
-
-            color 6 "Adding private key to secret"
-            gcloud secrets versions add "${SECRET_ID}" \
-                --project "${PROJECT}" \
-                --data-file "${KEY_FILE}"
-
-            color 6 "Empowering ${BUCKET_WRITERS} for read secret"
-            ensure_secret_role_binding \
-                "projects/${PROJECT}/secrets/${SECRET_ID}" \
-                "group:${BUCKET_WRITERS}" \
-                "roles/secretmanager.secretAccessor"
-        fi
-    )
+for OFFERING; do
+    # The GCS bucket to hold conformance results for this offering
+    BUCKET="gs://${PROJECT}-${OFFERING}" # used by humans
+    # The group that can write to GCS bucket
+    BUCKET_WRITERS="k8s-infra-conform-${OFFERING}@kubernetes.io"
+    # The service account that can write to the GCS bucket
+    SERVICE_ACCOUNT_NAME="service-${OFFERING}"
+
+    if ! (printf '%s\n' "${CONFORMANCE_OFFERINGS[@]}" | grep -q "^${OFFERING}$"); then
+      color 2 "Skipping unrecognized conformance offering: ${OFFERING}"
+      continue
+    fi
+
+    color 3 "Ensuring conformance bucket for ${OFFERING}"
+    ensure_conformance_bucket "${BUCKET}" "${BUCKET_WRITERS}" 2>&1 | indent
+
+    color 3 "Ensuring conformance service account for ${OFFERING}" 2>&1 | indent
+    ensure_conformance_serviceaccount "${SERVICE_ACCOUNT_NAME}" "${BUCKET}" "${BUCKET_WRITERS}"
 done 2>&1 | indent
 color 6 "Done"
diff --git a/infra/gcp/lib.sh b/infra/gcp/lib.sh
old mode 100755
new mode 100644
index 367eccdd226..06b5e92466b
--- a/infra/gcp/lib.sh
+++ b/infra/gcp/lib.sh
@@ -15,14 +15,23 @@
 # limitations under the License.
 
 readonly TMPDIR=$(mktemp -d "/tmp/k8sio-infra-gcp-lib.XXXXX")
-trap 'rm -rf "${TMPDIR}"' EXIT
+function cleanup_tmpdir() {
+  if [ "${K8S_INFRA_DEBUG:-"false"}" == "true" ]; then
+    echo "K8S_INFRA_DEBUG mode, not removing tmpdir: ${TMPDIR}"
+    ls -l "${TMPDIR}"
+  else
+    rm -rf "${TMPDIR}"
+  fi
+}
+trap 'cleanup_tmpdir' EXIT
 
 # This is a library of functions used to create GCP stuff.
 
 . "$(dirname "${BASH_SOURCE[0]}")/lib_util.sh"
 . "$(dirname "${BASH_SOURCE[0]}")/lib_iam.sh"
-. "$(dirname "${BASH_SOURCE[0]}")/lib_gcr.sh"
 . "$(dirname "${BASH_SOURCE[0]}")/lib_gcs.sh"
+. "$(dirname "${BASH_SOURCE[0]}")/lib_gcr.sh"
+. "$(dirname "${BASH_SOURCE[0]}")/lib_gsm.sh"
 
 # The group that admins all GCR repos.
 GCR_ADMINS="k8s-infra-artifact-admins@kubernetes.io"
@@ -434,29 +443,6 @@ function empower_artifact_auditor_invoker() {
         --region=us-central1
 }
 
-# Create a service account
-# $1: The GCP project
-# $2: The account name (e.g. "foo-manager")
-# $3: The account display-name (e.g. "Manages all foo")
-function ensure_service_account() {
-    if [ $# != 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
-        echo "ensure_service_account(project, name, display_name) requires 3 arguments" >&2
-        return 1
-    fi
-    local project="$1"
-    local name="$2"
-    local display_name="$3"
-
-    local acct=$(svc_acct_email "${project}" "${name}")
-
-    if ! gcloud --project "${project}" iam service-accounts describe "${acct}" >/dev/null 2>&1; then
-        gcloud --project "${project}" \
-            iam service-accounts create \
-            "${name}" \
-            --display-name="${display_name}"
-    fi
-}
-
 # Ensure that DNS managed zone exists, creating one if need.
 # $1 The GCP project
 # $2 The managed zone name (e.g. kubernetes-io)
diff --git a/infra/gcp/lib_gcr.sh b/infra/gcp/lib_gcr.sh
old mode 100755
new mode 100644
diff --git a/infra/gcp/lib_gcs.sh b/infra/gcp/lib_gcs.sh
old mode 100755
new mode 100644
diff --git a/infra/gcp/lib_gsm.sh b/infra/gcp/lib_gsm.sh
new file mode 100644
index 00000000000..7247f5d2c34
--- /dev/null
+++ b/infra/gcp/lib_gsm.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Google Secret Manager (GSM) utility functions
+#
+# This MUST NOT be used directly. Source it via lib.sh instead.
+
+# Returns full name of a secret in the given project with the given secret id
+# Arguments:
+#   $1: The project id hosting the secret (e.g. "k8s-infra-foo")
+#   $2: The secret name (e.g. "my-secret")
+function secret_full_name() {
+    if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then
+        echo "secret_full_name(project, secret) requires 2 arguments" >&2
+        return 1
+    fi
+
+    local project="${1}"
+    local secret="${2}"
+
+    # this command would take longer, require privileges, and fail if not found
+    # gcloud secrets describe --projects ${project} ${name} --format='value(name)'
+    echo "projects/${project}/secrets/${secret}"
+}
+
+# Ensures a secret exists in the given project with the given name
+# Arguments:
+#   $1: The project id hosting the secret (e.g. "k8s-infra-foo")
+#   $2: The secret name (e.g. "my-secret")
+function ensure_secret() {
+    if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then
+        echo "ensure_secret(project, secret) requires 2 arguments" >&2
+        return 1
+    fi
+
+    local project="${1}"
+    local secret="${2}"
+
+    if ! gcloud secrets describe --project "${project}" "${secret}" > /dev/null; then
+      gcloud secrets create --project "${project}" "${secret}"
+    fi
+}
+
+# Ensures a secret exists in the given project with the given name. If the
+# secret does not exist, it is pre-populated with a newly created private key
+# for the given service-account
+# Arguments:
+#   $1: The project id hosting the secret (e.g. "k8s-infra-foo")
+#   $2: The secret name (e.g. "my-secret")
+#   $3: The service-account (e.g. "foo@k8s-infra.iam.gserviceaccount.com")
+function ensure_serviceaccount_key_secret() {
+    if [ ! $# -eq 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
+        echo "ensure_serviceaccount_key_secret(project, secret, serviceaccountt) requires 3 arguments" >&2
+        return 1
+    fi
+
+    local project="${1}"
+    local secret="${2}"
+    local serviceaccount="${3}"
+
+    local private_key_file="${TMPDIR}/key.json"
+
+    if ! gcloud secrets describe --project "${project}" "${secret}" > /dev/null; then
+        ensure_secret "${project}" "${secret}"
+
+        gcloud iam service-accounts keys create "${private_key_file}" \
+            --project "${project}" \
+            --iam-account "${email}"
+
+        gcloud secrets versions add "${secret}" \
+            --project "${project}" \
+            --data-file "${private_key_file}"
+    fi
+}
diff --git a/infra/gcp/lib_iam.sh b/infra/gcp/lib_iam.sh
old mode 100755
new mode 100644
index 9d04edaa139..1c547114841
--- a/infra/gcp/lib_iam.sh
+++ b/infra/gcp/lib_iam.sh
@@ -21,6 +21,42 @@
 #
 # This MUST NOT be used directly. Source it via lib.sh instead.
 
+# Ensure that a the given GCP project contains a service account with
+# the given name and display_name
+# $1: The GCP project
+# $2: The account name (e.g. "foo-manager")
+# $3: The account display-name (e.g. "Manages all foo")
+function ensure_service_account() {
+    if [ $# != 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
+        echo "ensure_service_account(project, name, display_name) requires 3 arguments" >&2
+        return 1
+    fi
+    local project="$1"
+    local name="$2"
+    local display_name="$3"
+
+    local email=$(svc_acct_email "${project}" "${name}")
+
+    local before="${TMPDIR}/service-account.before.yaml"
+    local after="${TMPDIR}/service-account.after.yaml"
+    local verb=""
+
+    if ! gcloud iam service-accounts --project "${project}" describe "${email}" >"${before}" 2>/dev/null; then
+        verb="create"
+    elif [ "$(<"${before}" yq -r .displayName)" != "${display_name}" ]; then
+        verb="update"
+    fi
+
+    if [ -n "${verb}" ]; then
+        gcloud iam service-accounts "${verb}" \
+          --project "${project}" \
+          "${email}" \
+          --display-name="${display_name}"
+        gcloud iam service-accounts --project "${project}" describe "${email}" > "${after}"
+        diff_colorized "${before}" "${after}"
+    fi
+}
+
 # Ensure that custom IAM role exists in organization and in sync with definition in file
 # Arguments:
 #   $1:  The role name (e.g. "foo.barrer")
@@ -364,7 +400,7 @@ function _ensure_removed_custom_iam_role() {
 function _format_iam_policy() {
   # shellcheck disable=SC2016
   # $r is a jq variable, not a bash expression
-  yq -y '.bindings
+  yq -y '(.bindings // [])
     | map(.role as $r | .members | map({member: ., role: $r}))
     | flatten | sort_by(.member)'
 }
diff --git a/infra/gcp/lib_util.sh b/infra/gcp/lib_util.sh
old mode 100755
new mode 100644