From 5e8c72c4f0f17756b9f540e8df198eb3f3469261 Mon Sep 17 00:00:00 2001 From: Ashutosh Kumar Date: Sat, 10 Dec 2022 18:00:49 +0530 Subject: [PATCH] azwi integration to capz Signed-off-by: Ashutosh Kumar --- Makefile | 8 +- Tiltfile | 10 + api/v1beta1/azureclusteridentity_webhook.go | 16 +- api/v1beta1/types.go | 5 +- azure/scope/identity.go | 10 + azure/scope/workload_identity.go | 155 ++++++++++ ...uster.x-k8s.io_azureclusteridentities.yaml | 1 + config/default/azwi.yaml | 253 ++++++++++++++++ config/manager/manager.yaml | 1 + config/rbac/service_account.yaml | 4 +- scripts/kind-with-registry.sh | 33 +++ ...uster-template-prow-workload-identity.yaml | 220 ++++++++++++++ .../ci/patches/azureclusteridentity-azwi.yaml | 11 + .../prow-workload-identity/kustomization.yaml | 13 + test/e2e/azure_test.go | 51 ++++ test/e2e/config/azure-dev.yaml | 2 + test/e2e/config/azwi.yaml | 272 ++++++++++++++++++ test/e2e/e2e_suite_test.go | 26 +- 18 files changed, 1084 insertions(+), 7 deletions(-) create mode 100644 azure/scope/workload_identity.go create mode 100644 config/default/azwi.yaml create mode 100644 templates/test/ci/cluster-template-prow-workload-identity.yaml create mode 100644 templates/test/ci/patches/azureclusteridentity-azwi.yaml create mode 100644 templates/test/ci/prow-workload-identity/kustomization.yaml create mode 100644 test/e2e/config/azwi.yaml diff --git a/Makefile b/Makefile index b3e3805b4a8..b6ac730c535 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml SKIP_CLEANUP ?= false SKIP_LOG_COLLECTION ?= false -SKIP_CREATE_MGMT_CLUSTER ?= false +SKIP_CREATE_MGMT_CLUSTER ?= true WIN_REPO_URL ?= # Build time versioning details. @@ -644,7 +644,7 @@ test-cover: test ## Run tests with code coverage and generate reports. go tool cover -html=coverage.out -o coverage.html .PHONY: test-e2e-run -test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests. +test-e2e-run: generate-e2e-templates install-tools create-bootstrap-cluster ## Run e2e tests. $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \ -e2e.artifacts-folder="$(ARTIFACTS)" \ @@ -658,6 +658,10 @@ test-e2e: ## Run "docker-build" and "docker-push" rules then run e2e tests. $(MAKE) docker-build docker-push \ test-e2e-run +create-bootstrap-cluster: + KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh + + .PHONY: test-e2e-skip-push test-e2e-skip-push: ## Run "docker-build" rule then run e2e tests. PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ diff --git a/Tiltfile b/Tiltfile index 67f438ce1c8..046c1cb48df 100644 --- a/Tiltfile +++ b/Tiltfile @@ -21,6 +21,7 @@ settings = { "kind_cluster_name": "capz", "capi_version": "v1.4.2", "cert_manager_version": "v1.11.1", + "azwi_version": "v0.14.0", "kubernetes_version": "v1.24.6", "aks_kubernetes_version": "v1.24.6", "flatcar_version": "3374.2.1", @@ -46,6 +47,13 @@ if "allowed_contexts" in settings: if "default_registry" in settings: default_registry(settings.get("default_registry")) +# deploy AZWI webhook +def deploy_azwi(): + version = settings.get("azwi_version") + azwi_uri = "https://github.com/Azure/azure-workload-identity/releases/download/{}/azure-wi-webhook.yaml".format(version) + cmd = "curl -sSL {} | {} | {} apply -f -".format(azwi_uri, envsubst_cmd, kubectl_cmd) + local(cmd, quiet = True) + # deploy CAPI def deploy_capi(): version = settings.get("capi_version") @@ -423,6 +431,8 @@ load("ext://cert_manager", "deploy_cert_manager") if settings.get("deploy_cert_manager"): deploy_cert_manager(version = settings.get("cert_manager_version")) +deploy_azwi() + deploy_capi() create_identity_secret() diff --git a/api/v1beta1/azureclusteridentity_webhook.go b/api/v1beta1/azureclusteridentity_webhook.go index 3301587b346..622b4f20d84 100644 --- a/api/v1beta1/azureclusteridentity_webhook.go +++ b/api/v1beta1/azureclusteridentity_webhook.go @@ -17,7 +17,10 @@ limitations under the License. package v1beta1 import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + webhookutils "sigs.k8s.io/cluster-api-provider-azure/util/webhook" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -40,7 +43,18 @@ func (c *AzureClusterIdentity) ValidateCreate() error { // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. func (c *AzureClusterIdentity) ValidateUpdate(oldRaw runtime.Object) error { - return c.validateClusterIdentity() + var allErrs field.ErrorList + old := oldRaw.(*AzureClusterIdentity) + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "Type"), + old.Spec.Type, + c.Spec.Type); err != nil { + allErrs = append(allErrs, err) + } + if len(allErrs) == 0 { + return c.validateClusterIdentity() + } + return apierrors.NewInvalid(GroupVersion.WithKind("AzureClusterIdentity").GroupKind(), c.Name, allErrs) } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index b1a96bb0915..6b9c5402175 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -535,7 +535,7 @@ const ( ) // IdentityType represents different types of identities. -// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI;ManualServicePrincipal;ServicePrincipalCertificate +// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI;ManualServicePrincipal;ServicePrincipalCertificate;WorkloadIdentity type IdentityType string const ( @@ -550,6 +550,9 @@ const ( // ServicePrincipalCertificate represents a service principal using a certificate as secret. ServicePrincipalCertificate IdentityType = "ServicePrincipalCertificate" + + // WorkloadIdentity represents a WorkloadIdentity. + WorkloadIdentity IdentityType = "WorkloadIdentity" ) // OSDisk defines the operating system disk for a VM. diff --git a/azure/scope/identity.go b/azure/scope/identity.go index e9ee0eb31b3..1ee6e2e8327 100644 --- a/azure/scope/identity.go +++ b/azure/scope/identity.go @@ -142,6 +142,16 @@ func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceMa var authErr error var cred azcore.TokenCredential switch p.Identity.Spec.Type { + case infrav1.WorkloadIdentity: + azwiCredOptions, err := NewWorkloadIdentityCredentialOptions(). + WithTenantID(p.Identity.Spec.TenantID). + WithClientID(p.Identity.Spec.ClientID). + WithDefaults() + if err != nil { + return nil, errors.Wrapf(err, "failed to setup azwi options for identity %s", p.Identity.Name) + } + cred, authErr = NewWorkloadIdentityCredential(azwiCredOptions) + case infrav1.ServicePrincipal, infrav1.ServicePrincipalCertificate, infrav1.UserAssignedMSI: if err := createAzureIdentityWithBindings(ctx, p.Identity, resourceManagerEndpoint, activeDirectoryEndpoint, clusterMeta, p.Client); err != nil { return nil, err diff --git a/azure/scope/workload_identity.go b/azure/scope/workload_identity.go new file mode 100644 index 00000000000..4c44d801c5b --- /dev/null +++ b/azure/scope/workload_identity.go @@ -0,0 +1,155 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "context" + "os" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/pkg/errors" +) + +/* + +AZWI (Azure Workload Identity) required deploying AZWI mutating admission webhook +for self managed clusters e.g. Kind. + +The webhook injects the following environment variables to the pod that +uses a service account with a annotation `azure.workload.identity/use=true` +|-----------------------------------------------------------------------------------| +|AZURE_AUTHORITY_HOST | The Azure Active Directory (AAD) endpoint. | +|AZURE_CLIENT_ID | The application/client ID of the Azure AD | +| | application or user-assigned managed identity. | +|AZURE_TENANT_ID | The tenant ID of the Azure subscription. | +|AZURE_FEDERATED_TOKEN_FILE | The path of the projected service account token file. | +|-----------------------------------------------------------------------------------| + +In addition to the service account, it also projects a signed service account token to the +workload's volume(in this case the capz pod). The volume name is `azure-identity-token` +which is mounted at path `/var/run/secrets/azure/tokens/azure-identity-token` to the pod. + +*/ + +const ( + // AzureFedratedTokenFileENVKey is the env key for AZURE_FEDERATED_TOKEN_FILE. + AzureFedratedTokenFileENVKey = "AZURE_FEDERATED_TOKEN_FILE" + // AzureClientIDENVKey is the env key for AZURE_CLIENT_ID. + AzureClientIDENVKey = "AZURE_CLIENT_ID" + // AzureTenantIDENVKey is the env key for AZURE_TENANT_ID. + AzureTenantIDENVKey = "AZURE_TENANT_ID" +) + +type workloadIdentityCredential struct { + assertion string + file string + cred *azidentity.ClientAssertionCredential + lastRead time.Time +} + +// WorkloadIdentityCredentialOptions contains the configurable options for azwi. +type WorkloadIdentityCredentialOptions struct { + ClientID string + TenantID string + TokenFilePath string + azcore.ClientOptions +} + +// NewWorkloadIdentityCredentialOptions returns an empty instance of WorkloadIdentityCredentialOptions. +func NewWorkloadIdentityCredentialOptions() *WorkloadIdentityCredentialOptions { + return &WorkloadIdentityCredentialOptions{} +} + +// WithClientID sets client ID to WorkloadIdentityCredentialOptions. +func (w *WorkloadIdentityCredentialOptions) WithClientID(clientID string) *WorkloadIdentityCredentialOptions { + w.ClientID = clientID + return w +} + +// WithTenantID sets tenant ID to WorkloadIdentityCredentialOptions. +func (w *WorkloadIdentityCredentialOptions) WithTenantID(tenantID string) *WorkloadIdentityCredentialOptions { + w.TenantID = tenantID + return w +} + +// GetProjectedTokenPath return projected token file path from the env variable. +func GetProjectedTokenPath() (string, error) { + tokenPath := os.Getenv(AzureFedratedTokenFileENVKey) + if strings.TrimSpace(tokenPath) == "" { + return "", errors.New("projected token path not injected") + } + return tokenPath, nil +} + +// WithDefaults sets token file path. It also sets the client tenant ID from injected env in +// case empty values are passed. +func (w *WorkloadIdentityCredentialOptions) WithDefaults() (*WorkloadIdentityCredentialOptions, error) { + tokenFilePath, err := GetProjectedTokenPath() + if err != nil { + return nil, errors.Wrap(err, "failed to get token file path for identity") + } + w.TokenFilePath = tokenFilePath + + // Fallback to using client ID from env variable if not set. + if strings.TrimSpace(w.ClientID) == "" { + w.ClientID = os.Getenv(AzureClientIDENVKey) + if strings.TrimSpace(w.ClientID) == "" { + return nil, errors.New("empty client ID") + } + } + + // // Fallback to using tenant ID from env variable. + if strings.TrimSpace(w.TenantID) == "" { + w.TenantID = os.Getenv(AzureTenantIDENVKey) + if strings.TrimSpace(w.TenantID) == "" { + return nil, errors.New("empty tenant ID") + } + } + return w, nil +} + +// NewWorkloadIdentityCredential returns a workload identity credential. +func NewWorkloadIdentityCredential(options *WorkloadIdentityCredentialOptions) (*workloadIdentityCredential, error) { + w := &workloadIdentityCredential{file: options.TokenFilePath} + cred, err := azidentity.NewClientAssertionCredential(options.TenantID, options.ClientID, w.getAssertion, &azidentity.ClientAssertionCredentialOptions{ClientOptions: options.ClientOptions}) + if err != nil { + return nil, err + } + w.cred = cred + return w, nil +} + +// GetToken returns the token for azwi. +func (w *workloadIdentityCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) { + return w.cred.GetToken(ctx, opts) +} + +func (w *workloadIdentityCredential) getAssertion(context.Context) (string, error) { + if now := time.Now(); w.lastRead.Add(5 * time.Minute).Before(now) { + content, err := os.ReadFile(w.file) + if err != nil { + return "", err + } + w.assertion = string(content) + w.lastRead = now + } + return w.assertion, nil +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml index 6d0333e5f03..ae3cb57b839 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml @@ -150,6 +150,7 @@ spec: - UserAssignedMSI - ManualServicePrincipal - ServicePrincipalCertificate + - WorkloadIdentity type: string required: - clientID diff --git a/config/default/azwi.yaml b/config/default/azwi.yaml new file mode 100644 index 00000000000..a74174bbc9b --- /dev/null +++ b/config/default/azwi.yaml @@ -0,0 +1,253 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-admin + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-manager-role + namespace: system +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-manager-role +rules: + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - patch + - update + - watch + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-manager-rolebinding + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: azure-wi-webhook-manager-role +subjects: + - kind: ServiceAccount + name: azure-wi-webhook-admin + namespace: system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: azure-wi-webhook-manager-role +subjects: + - kind: ServiceAccount + name: azure-wi-webhook-admin + namespace: system +--- +apiVersion: v1 +data: + AZURE_ENVIRONMENT: AzurePublicCloud + AZURE_TENANT_ID: b39138ca-3cee-4b4a-a4d6-cd83d9dd62f0 +kind: ConfigMap +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-config + namespace: system +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-server-cert + namespace: system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + azure-workload-identity.io/system: "true" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-controller-manager + namespace: capz-system +spec: + replicas: 2 + selector: + matchLabels: + azure-workload-identity.io/system: "true" + template: + metadata: + labels: + azure-workload-identity.io/system: "true" + spec: + containers: + - args: + - --arc-cluster=false + command: + - /manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + envFrom: + - configMapRef: + name: azure-wi-webhook-config + image: mcr.microsoft.com/oss/azure/workload-identity/webhook:v0.15.0 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: healthz + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 8095 + name: metrics + protocol: TCP + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /certs + name: cert + readOnly: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: azure-wi-webhook-admin + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: azure-wi-webhook-server-cert +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-controller-manager + namespace: system +spec: + minAvailable: 1 + selector: + matchLabels: + azure-workload-identity.io/system: "true" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + labels: + azure-workload-identity.io/system: "true" + name: azure-wi-webhook-mutating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: azure-wi-webhook-webhook-service + namespace: system + path: /mutate-v1-pod + failurePolicy: Ignore + matchPolicy: Equivalent + name: mutation.azure-workload-identity.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + sideEffects: None diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 0375abd93a7..58c9deddaff 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -16,6 +16,7 @@ spec: labels: control-plane: capz-controller-manager aadpodidbinding: capz-controller-aadpodidentity-selector + azure.workload.identity/use: "true" annotations: kubectl.kubernetes.io/default-container: manager spec: diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index c4180052449..55b973518af 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,5 +1,7 @@ apiVersion: v1 kind: ServiceAccount metadata: + labels: + azure.workload.identity/use: "true" name: manager - namespace: system \ No newline at end of file + namespace: system diff --git a/scripts/kind-with-registry.sh b/scripts/kind-with-registry.sh index 7367f794468..2f0a0ffe485 100755 --- a/scripts/kind-with-registry.sh +++ b/scripts/kind-with-registry.sh @@ -41,10 +41,43 @@ if [ "${running}" != 'true' ]; then docker run -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" registry:2 fi +SERVICE_ACCOUNT_ISSUER="${SERVICE_ACCOUNT_ISSUER:-https://oidcissuercapzci.blob.core.windows.net/oidc-capzci/}" + +if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB}" ]]; then + echo "'SERVICE_ACCOUNT_SIGNING_PUB' is not set." + exit 1 +fi + +if [[ -z "${SERVICE_ACCOUNT_SIGNING_KEY}" ]]; then + echo "'SERVICE_ACCOUNT_SIGNING_KEY' is not set." + exit 1 +fi + +mkdir -p "$HOME"/azwi/creds +echo "${SERVICE_ACCOUNT_SIGNING_PUB}" > "$HOME"/azwi/creds/sa.pub +echo "${SERVICE_ACCOUNT_SIGNING_KEY}" > "$HOME"/azwi/creds/sa.key # create a cluster with the local registry enabled in containerd cat <