Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ensure k8s-conform service accounts have objectAdmin rights #1890

Merged
merged 6 commits into from
Apr 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 93 additions & 80 deletions infra/gcp/ensure-conformance-storage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,100 +43,113 @@ function usage() {

PROJECT="k8s-conform"

CONFORMANCE_SERVICES=(
# secretmanager to host service-account keys
secretmanager.googleapis.com
# storage-api to store results in GCS via JSON API
storage-api.googleapis.com
# storage-component to store results in GCS via XML API
storage-component.googleapis.com
)

readonly CONFORMANCE_RETENTION="10y"

# "Offering" comes from https://github.com/cncf/k8s-conformance/blob/master/terms-conditions/Certified_Kubernetes_Terms.md
# NB: Please keep this sorted.
CONFORMANCE_BUCKETS=(
CONFORMANCE_OFFERINGS=(
cri-o
huaweicloud
inspur
provider-openstack
s390x-k8s
)

if [ $# = 0 ]; then
# default to all conformance buckets
set -- "${CONFORMANCE_BUCKETS[@]}"
set -- "${CONFORMANCE_OFFERINGS[@]}"
fi

# Make the project, if needed
color 6 "Ensuring project exists: ${PROJECT}"
ensure_project "${PROJECT}"
function ensure_conformance_project() {
color 6 "Ensuring project exists: ${PROJECT}"
ensure_project "${PROJECT}"

# Enable GCS APIs
color 6 "Ensuring only necessary services are enabled for conformance project: ${PROJECT}"
ensure_only_services "${PROJECT}" "${CONFORMANCE_SERVICES[@]}"
}

# ensure_conformance_bucket ensure that:
# - the given bucket exists and is publicly readable
# - the given bucket has the conformance retention policy setup
# - the given bucket is writable by the given group
function ensure_conformance_bucket() {
local bucket="${1}"
local writers="${2}"

color 6 "Ensuring ${PROJECT} contains world readadble GCS bucket: ${bucket}"
ensure_public_gcs_bucket "${PROJECT}" "${bucket}"

color 6 "Empowering GCS admins for GCS bucket: ${bucket}"
empower_gcs_admins "${PROJECT}" "${bucket}"

color 6 "Ensuring ${bucket} retention policy is set to: ${CONFORMANCE_RETENTION}"
ensure_gcs_bucket_retention "${bucket}" "${CONFORMANCE_RETENTION}"

color 6 "Empowering ${writers} to write to GCS bucket: ${bucket}"
empower_group_to_write_gcs_bucket "${writers}" "${bucket}"

}

# ensure_conformance_serviceaccount ensures that:
# - a serviceaccount of the given name exists in PROJECT
# - it can write to the given bucket
# - it has a private key stored in a secret in PROJECT accessible to the given group
function ensure_conformance_serviceaccount() {
local name="${1}"
local bucket="${2}"
local secret_accessors="${3}"

local email="$(svc_acct_email "${PROJECT}" "${name}")"
local secret="${name}-key"
local private_key_file="${TMPDIR}/key.json"

color 6 "Ensuring service account exists: ${email}"
ensure_service_account "${PROJECT}" "${name}" "Grants write access to ${bucket}"

color 6 "Ensuring ${PROJECT} contains secret ${secret} with private key for ${email}"
ensure_serviceaccount_key_secret "${PROJECT}" "${secret}" "${email}"

color 6 "Empowering ${secret_accessors} to access secret: ${secret}"
ensure_secret_role_binding \
"projects/${PROJECT}/secrets/${secret}" \
"group:${secret_accessors}" \
"roles/secretmanager.secretAccessor"

color 6 "Empowering ${email} to write to ${bucket}"
empower_svcacct_to_write_gcs_bucket "${email}" "${bucket}"
}

# Enable GCS APIs
color 6 "Ensuring only necessary services are enabled for conformance project: ${PROJECT}"
ensure_only_services "${PROJECT}" \
storage-component.googleapis.com \
secretmanager.googleapis.com \
ensure_conformance_project

color 6 "Ensuring all conformance buckets"
for REPO; do
color 3 "Configuring conformance bucket for ${REPO}"

# The group that can write to this conformance repo.
BUCKET_WRITERS="k8s-infra-conform-${REPO}@kubernetes.io"

# The names of the buckets
BUCKET="gs://${PROJECT}-${REPO}" # used by humans

# Every project gets some GCS buckets
color 3 "Configuring bucket: ${BUCKET}"

# Create the bucket
color 6 "Ensuring the bucket exists and is world readable"
ensure_public_gcs_bucket "${PROJECT}" "${BUCKET}"

color 6 "Ensuring the GCS bucket retention policy is set: ${PROJECT}"
RETENTION="10y"
ensure_gcs_bucket_retention "${BUCKET}" "${RETENTION}"

# Enable admins on the bucket
color 6 "Empowering GCS admins"
empower_gcs_admins "${PROJECT}" "${BUCKET}"

# Enable writers on the bucket
color 6 "Empowering ${BUCKET_WRITERS} to GCS"
empower_group_to_write_gcs_bucket "${BUCKET_WRITERS}" "${BUCKET}"

(
readonly SERVICE_ACCOUNT_NAME="service-${REPO}"
readonly SERVICE_ACCOUNT_EMAIL="$(svc_acct_email "${PROJECT}" \
"${SERVICE_ACCOUNT_NAME}")"
readonly SECRET_ID="${SERVICE_ACCOUNT_NAME}-key"
readonly KEY_FILE="${TMPDIR}/key.json"

if ! gcloud iam service-accounts describe "${SERVICE_ACCOUNT_EMAIL}" \
--project "${PROJECT}" >/dev/null 2>&1
then
color 6 "Creating service account: ${SERVICE_ACCOUNT_NAME}"
ensure_service_account \
"${PROJECT}" \
"${SERVICE_ACCOUNT_NAME}" \
"${SERVICE_ACCOUNT_NAME}"

color 6 "Empowering service account: ${SERVICE_ACCOUNT_NAME} to GCS"
empower_svcacct_to_write_gcs_bucket \
"${SERVICE_ACCOUNT_EMAIL}" \
"${BUCKET}"

color 6 "Creating private key for service account: ${SERVICE_ACCOUNT_NAME}"
gcloud iam service-accounts keys create "${KEY_FILE}" \
--project "${PROJECT}" \
--iam-account "${SERVICE_ACCOUNT_EMAIL}"

color 6 "Creating secret to store private key"
gcloud secrets create "${SECRET_ID}" \
--project "${PROJECT}" \
--replication-policy "automatic"

color 6 "Adding private key to secret"
gcloud secrets versions add "${SECRET_ID}" \
--project "${PROJECT}" \
--data-file "${KEY_FILE}"

color 6 "Empowering ${BUCKET_WRITERS} for read secret"
ensure_secret_role_binding \
"projects/${PROJECT}/secrets/${SECRET_ID}" \
"group:${BUCKET_WRITERS}" \
"roles/secretmanager.secretAccessor"
fi
)
for OFFERING; do
# The GCS bucket to hold conformance results for this offering
BUCKET="gs://${PROJECT}-${OFFERING}" # used by humans
# The group that can write to GCS bucket
BUCKET_WRITERS="k8s-infra-conform-${OFFERING}@kubernetes.io"
# The service account that can write to the GCS bucket
SERVICE_ACCOUNT_NAME="service-${OFFERING}"

if ! (printf '%s\n' "${CONFORMANCE_OFFERINGS[@]}" | grep -q "^${OFFERING}$"); then
color 2 "Skipping unrecognized conformance offering: ${OFFERING}"
continue
fi

color 3 "Ensuring conformance bucket for ${OFFERING}"
ensure_conformance_bucket "${BUCKET}" "${BUCKET_WRITERS}" 2>&1 | indent

color 3 "Ensuring conformance service account for ${OFFERING}" 2>&1 | indent
ensure_conformance_serviceaccount "${SERVICE_ACCOUNT_NAME}" "${BUCKET}" "${BUCKET_WRITERS}"
done 2>&1 | indent
color 6 "Done"
36 changes: 11 additions & 25 deletions infra/gcp/lib.sh
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,23 @@
# limitations under the License.

readonly TMPDIR=$(mktemp -d "/tmp/k8sio-infra-gcp-lib.XXXXX")
trap 'rm -rf "${TMPDIR}"' EXIT
function cleanup_tmpdir() {
if [ "${K8S_INFRA_DEBUG:-"false"}" == "true" ]; then
echo "K8S_INFRA_DEBUG mode, not removing tmpdir: ${TMPDIR}"
ls -l "${TMPDIR}"
else
rm -rf "${TMPDIR}"
fi
}
trap 'cleanup_tmpdir' EXIT

# This is a library of functions used to create GCP stuff.

. "$(dirname "${BASH_SOURCE[0]}")/lib_util.sh"
. "$(dirname "${BASH_SOURCE[0]}")/lib_iam.sh"
. "$(dirname "${BASH_SOURCE[0]}")/lib_gcr.sh"
. "$(dirname "${BASH_SOURCE[0]}")/lib_gcs.sh"
. "$(dirname "${BASH_SOURCE[0]}")/lib_gcr.sh"
. "$(dirname "${BASH_SOURCE[0]}")/lib_gsm.sh"

# The group that admins all GCR repos.
GCR_ADMINS="[email protected]"
Expand Down Expand Up @@ -434,29 +443,6 @@ function empower_artifact_auditor_invoker() {
--region=us-central1
}

# Create a service account
# $1: The GCP project
# $2: The account name (e.g. "foo-manager")
# $3: The account display-name (e.g. "Manages all foo")
function ensure_service_account() {
if [ $# != 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
echo "ensure_service_account(project, name, display_name) requires 3 arguments" >&2
return 1
fi
local project="$1"
local name="$2"
local display_name="$3"

local acct=$(svc_acct_email "${project}" "${name}")

if ! gcloud --project "${project}" iam service-accounts describe "${acct}" >/dev/null 2>&1; then
gcloud --project "${project}" \
iam service-accounts create \
"${name}" \
--display-name="${display_name}"
fi
}

# Ensure that DNS managed zone exists, creating one if need.
# $1 The GCP project
# $2 The managed zone name (e.g. kubernetes-io)
Expand Down
Empty file modified infra/gcp/lib_gcr.sh
100755 → 100644
Empty file.
Empty file modified infra/gcp/lib_gcs.sh
100755 → 100644
Empty file.
87 changes: 87 additions & 0 deletions infra/gcp/lib_gsm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Google Secret Manager (GSM) utility functions
#
# This MUST NOT be used directly. Source it via lib.sh instead.

# Returns full name of a secret in the given project with the given secret id
# Arguments:
# $1: The project id hosting the secret (e.g. "k8s-infra-foo")
# $2: The secret name (e.g. "my-secret")
function secret_full_name() {
if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then
echo "secret_full_name(project, secret) requires 2 arguments" >&2
return 1
fi

local project="${1}"
local secret="${2}"

# this command would take longer, require privileges, and fail if not found
# gcloud secrets describe --projects ${project} ${name} --format='value(name)'
echo "projects/${project}/secrets/${secret}"
}

# Ensures a secret exists in the given project with the given name
# Arguments:
# $1: The project id hosting the secret (e.g. "k8s-infra-foo")
# $2: The secret name (e.g. "my-secret")
function ensure_secret() {
if [ ! $# -eq 2 -o -z "$1" -o -z "$2" ]; then
echo "ensure_secret(project, secret) requires 2 arguments" >&2
return 1
fi

local project="${1}"
local secret="${2}"

if ! gcloud secrets describe --project "${project}" "${secret}" > /dev/null; then
gcloud secrets create --project "${project}" "${secret}"
fi
}

# Ensures a secret exists in the given project with the given name. If the
# secret does not exist, it is pre-populated with a newly created private key
# for the given service-account
# Arguments:
# $1: The project id hosting the secret (e.g. "k8s-infra-foo")
# $2: The secret name (e.g. "my-secret")
# $3: The service-account (e.g. "[email protected]")
function ensure_serviceaccount_key_secret() {
if [ ! $# -eq 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
echo "ensure_serviceaccount_key_secret(project, secret, serviceaccountt) requires 3 arguments" >&2
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

email instead of serviceaccountt

return 1
fi

local project="${1}"
local secret="${2}"
local serviceaccount="${3}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@spiffxp you mean email ?


local private_key_file="${TMPDIR}/key.json"

if ! gcloud secrets describe --project "${project}" "${secret}" > /dev/null; then
ensure_secret "${project}" "${secret}"

gcloud iam service-accounts keys create "${private_key_file}" \
--project "${project}" \
--iam-account "${email}"

gcloud secrets versions add "${secret}" \
--project "${project}" \
--data-file "${private_key_file}"
fi
}
38 changes: 37 additions & 1 deletion infra/gcp/lib_iam.sh
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,42 @@
#
# This MUST NOT be used directly. Source it via lib.sh instead.

# Ensure that a the given GCP project contains a service account with
# the given name and display_name
# $1: The GCP project
# $2: The account name (e.g. "foo-manager")
# $3: The account display-name (e.g. "Manages all foo")
function ensure_service_account() {
if [ $# != 3 -o -z "$1" -o -z "$2" -o -z "$3" ]; then
echo "ensure_service_account(project, name, display_name) requires 3 arguments" >&2
return 1
fi
local project="$1"
local name="$2"
local display_name="$3"

local email=$(svc_acct_email "${project}" "${name}")

local before="${TMPDIR}/service-account.before.yaml"
local after="${TMPDIR}/service-account.after.yaml"
local verb=""

if ! gcloud iam service-accounts --project "${project}" describe "${email}" >"${before}" 2>/dev/null; then
verb="create"
elif [ "$(<"${before}" yq -r .displayName)" != "${display_name}" ]; then
verb="update"
fi

if [ -n "${verb}" ]; then
gcloud iam service-accounts "${verb}" \
--project "${project}" \
"${email}" \
--display-name="${display_name}"
gcloud iam service-accounts --project "${project}" describe "${email}" > "${after}"
diff_colorized "${before}" "${after}"
fi
}

# Ensure that custom IAM role exists in organization and in sync with definition in file
# Arguments:
# $1: The role name (e.g. "foo.barrer")
Expand Down Expand Up @@ -364,7 +400,7 @@ function _ensure_removed_custom_iam_role() {
function _format_iam_policy() {
# shellcheck disable=SC2016
# $r is a jq variable, not a bash expression
yq -y '.bindings
yq -y '(.bindings // [])
| map(.role as $r | .members | map({member: ., role: $r}))
| flatten | sort_by(.member)'
}
Expand Down
Empty file modified infra/gcp/lib_util.sh
100755 → 100644
Empty file.