Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI: workload-identity native #4765

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,11 @@ release-*/manifests/calico-*.yaml
# mentioned in the capz book
/sp.json
/cluster.yaml

# CI workload-identity
jwks.json
*.pub
*.key
azure_identity_id
azure_wi_back_compat
openid-configuration.json
38 changes: 35 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ KUSTOMIZE_VER := v5.4.1
KUSTOMIZE_BIN := kustomize
KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)

AZWI_VER := v1.2.2
AZWI_BIN := azwi
AZWI := $(TOOLS_BIN_DIR)/$(AZWI_BIN)-$(AZWI_VER)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we use the default kubeadm-generated keys, then I don't think we need azwi at all.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's correct (planning to do that in a follow-up PR pending priority/cycles)


MOCKGEN_VER := v0.4.0
MOCKGEN_BIN := mockgen
MOCKGEN := $(TOOLS_BIN_DIR)/$(MOCKGEN_BIN)-$(MOCKGEN_VER)
Expand Down Expand Up @@ -177,6 +181,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts
E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
SKIP_CLEANUP ?= false
AZWI_SKIP_CLEANUP ?= false
SKIP_LOG_COLLECTION ?= false
# @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
# to be created with extra mounts for key pairs which is not yet supported
Expand All @@ -191,6 +196,11 @@ LDFLAGS := $(shell hack/version.sh)
CLUSTER_TEMPLATE ?= cluster-template.yaml

export KIND_CLUSTER_NAME ?= capz
export RANDOM_SUFFIX := $(shell /bin/bash -c "echo $$RANDOM")
export AZWI_RESOURCE_GROUP ?= capz-wi-$(RANDOM_SUFFIX)
export CI_RG ?= $(AZWI_RESOURCE_GROUP)
export USER_IDENTITY ?= $(addsuffix $(RANDOM_SUFFIX),$(CI_RG))
export AZURE_IDENTITY_ID_FILEPATH ?= $(ROOT_DIR)/azure_identity_id

## --------------------------------------
## Binaries
Expand Down Expand Up @@ -287,7 +297,7 @@ verify-codespell: codespell ## Verify codespell.
##@ Development:

.PHONY: install-tools # populate hack/tools/bin
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND)
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI)

.PHONY: create-management-cluster
create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
Expand Down Expand Up @@ -341,7 +351,10 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
.PHONY: create-workload-cluster
create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster.
# Create workload Cluster.
@if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \
@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
fi; \
if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \
timeout --foreground 300 bash -c "until $(ENVSUBST) < $(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \
elif [ -f "$(CLUSTER_TEMPLATE)" ]; then \
timeout --foreground 300 bash -c "until $(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; do sleep 5; done"; \
Expand Down Expand Up @@ -686,7 +699,13 @@ test-cover: test ## Run tests with code coverage and generate reports.

.PHONY: kind-create-bootstrap
kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
export AZWI=$${AZWI:-true} KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh
KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh

.PHONY: cleanup-workload-identity
cleanup-workload-identity: ## Cleanup CI workload-identity infra
@if ! [ "$(AZWI_SKIP_CLEANUP)" == "true" ]; then \
./scripts/cleanup-workload-identity.sh \
fi

## --------------------------------------
## Security Scanning
Expand All @@ -708,6 +727,9 @@ kind-create: $(KUBECTL) ## Create capz kind cluster if needed.

.PHONY: tilt-up
tilt-up: install-tools kind-create ## Start tilt and build kind cluster if needed.
@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
fi; \
CLUSTER_TOPOLOGY=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true tilt up

.PHONY: delete-cluster
Expand Down Expand Up @@ -792,6 +814,16 @@ $(HELM): ## Put helm into tools folder.
ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
rm -f $(TOOLS_BIN_DIR)/get_helm.sh

$(AZWI): ## Put azwi into tools folder.
mkdir -p $(TOOLS_BIN_DIR)
rm -f "$(TOOLS_BIN_DIR)/$(AZWI_BIN)*"
curl --retry $(CURL_RETRIES) -fsSL -o $(TOOLS_BIN_DIR)/azwi.tar.gz https://github.com/Azure/azure-workload-identity/releases/download/$(AZWI_VER)/azwi-$(AZWI_VER)-$(GOOS)-$(GOARCH).tar.gz
tar -xf "$(TOOLS_BIN_DIR)/azwi.tar.gz" -C $(TOOLS_BIN_DIR) $(AZWI_BIN)
mv "$(TOOLS_BIN_DIR)/$(AZWI_BIN)" $(AZWI)
ln -sf $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
chmod +x $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
rm -f $(TOOLS_BIN_DIR)/azwi.tar.gz

$(KIND): ## Build kind into tools folder.
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kind $(KIND_BIN) $(KIND_VER)

Expand Down
7 changes: 2 additions & 5 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ settings = {
}

# Auth keys that need to be loaded from the environment
keys = ["AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_SECRET", "AZURE_CLIENT_ID"]
keys = ["AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_ID"]

# Get global settings from tilt-settings.yaml or tilt-settings.json
tilt_file = "./tilt-settings.yaml" if os.path.exists("./tilt-settings.yaml") else "./tilt-settings.json"
Expand Down Expand Up @@ -266,13 +266,10 @@ def create_identity_secret():

os.putenv("AZURE_CLUSTER_IDENTITY_SECRET_NAME", "cluster-identity-secret")
os.putenv("AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE", "default")
os.putenv("CLUSTER_IDENTITY_NAME", "cluster-identity")
os.putenv("CLUSTER_IDENTITY_NAME", "cluster-identity-ci")
os.putenv("ASO_CREDENTIAL_SECRET_NAME", "aso-credentials")

os.putenv("AZURE_CLIENT_SECRET_B64", base64_encode(os.environ.get("AZURE_CLIENT_SECRET")))
local("cat templates/azure-cluster-identity/secret.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True)
local("cat templates/flavors/aks-aso/credentials.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True)
os.unsetenv("AZURE_CLIENT_SECRET_B64")

def create_crs():
# create config maps
Expand Down
2 changes: 1 addition & 1 deletion docs/book/src/topics/workload-identity.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ to give the identity Contributor access to the Azure subscription where the work
- At this stage, you can apply this yaml to create a workload cluster.

Notes:
- Please follow this [link](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/test/ci/cluster-template-prow-workload-identity.yaml)
- Please follow this [link](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/test/ci/cluster-template-prow.yaml)
to see a workload cluster yaml configuration that uses workload identity.
- Creating a workload cluster via workload identity will be
simplified after [this](https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/3589) issue is resolved.
Expand Down
17 changes: 10 additions & 7 deletions e2e.mk
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@
# long-running E2E jobs every time that file changes

##@ E2E Testing:

.PHONY: test-e2e-run
test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
$(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
@$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
fi; \
$(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) \
$(MAKE) cleanup-workload-identity
$(MAKE) clean-release-git

.PHONY: test-e2e
Expand Down
5 changes: 5 additions & 0 deletions hack/create-dev-cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet
export AZURE_LOCATION="${AZURE_LOCATION:-southcentralus}"
export AZURE_RESOURCE_GROUP=${CLUSTER_NAME}

AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:=}"
AZURE_TENANT_ID="${AZURE_TENANT_ID:=}"
AZURE_CLIENT_ID="${AZURE_CLIENT_ID:=}"
AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET:=}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are these now needed for workload identity?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They aren't, these initialization statements are required in order to say "you don't have to declare these variables ahead of time". Without initializing default values the script will complain about unbound variables if they aren't set in the runtime environmental context.


AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')"
AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')"
Expand Down
2 changes: 2 additions & 0 deletions hack/log/redact.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,12 @@ redact_vars=(
"${AZURE_SUBSCRIPTION_ID:-}"
"${AZURE_TENANT_ID:-}"
"${AZURE_JSON_B64:-}"
"${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}"
"$(echo -n "${AZURE_SUBSCRIPTION_ID:-}" | base64 | tr -d '\n')"
"$(echo -n "${AZURE_TENANT_ID:-}" | base64 | tr -d '\n')"
"$(echo -n "${AZURE_CLIENT_ID:-}" | base64 | tr -d '\n')"
"$(echo -n "${AZURE_CLIENT_SECRET:-}" | base64 | tr -d '\n')"
"$(echo -n "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" | base64 | tr -d '\n')"
)

for log_file in "${log_files[@]}"; do
Expand Down
2 changes: 0 additions & 2 deletions hack/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,4 @@ capz::util::generate_ssh_key() {
capz::util::ensure_azure_envs() {
: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
: "${AZURE_TENANT_ID:?Environment variable empty or not defined.}"
: "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not required, no

: "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

both hack/ensure-azcli.sh and hack/ensure-acr-login.sh currently require this field:

az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" > /dev/null

and

docker login -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" capzci.azurecr.io

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here is a possible change set that could address this: https://github.com/kubernetes-sigs/cluster-api-provider-azure/compare/main...jsturtevant:cluster-api-provider-azure:remove-az-cli-cred-checks?expand=1

Let me know if you want to integrate the change into this PR or have me open a separate one

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The acr change lets local users build the image with out having to manually log in. It fixes an issue we saw when running kubernetes-sigs/windows-testing#430 locally with a test ACR instance

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for this! To remove variables in case there are still edge cases from this change, let's do this as a follow-up PR.

}
33 changes: 33 additions & 0 deletions scripts/cleanup-workload-identity.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Copyright 2024 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# Install kubectl and kind
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
# shellcheck source=hack/ensure-azcli.sh
source "${REPO_ROOT}/hack/ensure-azcli.sh"

AZWI_RESOURCE_GROUP="${AZWI_RESOURCE_GROUP:-}"

if [[ -z "${AZWI_RESOURCE_GROUP}" ]]; then
echo AZWI_RESOURCE_GROUP environment variable must be set
exit 1
jackfrancis marked this conversation as resolved.
Show resolved Hide resolved
fi

echo "Cleaning up CI workload-identity infra..."
az group delete --no-wait -y -n "${AZWI_RESOURCE_GROUP}"
Loading