Skip to content

Commit

Permalink
azwi integration to capz
Browse files Browse the repository at this point in the history
Signed-off-by: Ashutosh Kumar <[email protected]>
  • Loading branch information
sonasingh46 committed May 5, 2023
1 parent 2d10050 commit 5e8c72c
Show file tree
Hide file tree
Showing 18 changed files with 1,084 additions and 7 deletions.
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
SKIP_CLEANUP ?= false
SKIP_LOG_COLLECTION ?= false
SKIP_CREATE_MGMT_CLUSTER ?= false
SKIP_CREATE_MGMT_CLUSTER ?= true
WIN_REPO_URL ?=

# Build time versioning details.
Expand Down Expand Up @@ -644,7 +644,7 @@ test-cover: test ## Run tests with code coverage and generate reports.
go tool cover -html=coverage.out -o coverage.html

.PHONY: test-e2e-run
test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests.
test-e2e-run: generate-e2e-templates install-tools create-bootstrap-cluster ## Run e2e tests.
$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
$(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
Expand All @@ -658,6 +658,10 @@ test-e2e: ## Run "docker-build" and "docker-push" rules then run e2e tests.
$(MAKE) docker-build docker-push \
test-e2e-run

create-bootstrap-cluster:
KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh


.PHONY: test-e2e-skip-push
test-e2e-skip-push: ## Run "docker-build" rule then run e2e tests.
PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
Expand Down
10 changes: 10 additions & 0 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ settings = {
"kind_cluster_name": "capz",
"capi_version": "v1.4.2",
"cert_manager_version": "v1.11.1",
"azwi_version": "v0.14.0",
"kubernetes_version": "v1.24.6",
"aks_kubernetes_version": "v1.24.6",
"flatcar_version": "3374.2.1",
Expand All @@ -46,6 +47,13 @@ if "allowed_contexts" in settings:
if "default_registry" in settings:
default_registry(settings.get("default_registry"))

# deploy AZWI webhook
def deploy_azwi():
version = settings.get("azwi_version")
azwi_uri = "https://github.com/Azure/azure-workload-identity/releases/download/{}/azure-wi-webhook.yaml".format(version)
cmd = "curl -sSL {} | {} | {} apply -f -".format(azwi_uri, envsubst_cmd, kubectl_cmd)
local(cmd, quiet = True)

# deploy CAPI
def deploy_capi():
version = settings.get("capi_version")
Expand Down Expand Up @@ -423,6 +431,8 @@ load("ext://cert_manager", "deploy_cert_manager")
if settings.get("deploy_cert_manager"):
deploy_cert_manager(version = settings.get("cert_manager_version"))

deploy_azwi()

deploy_capi()

create_identity_secret()
Expand Down
16 changes: 15 additions & 1 deletion api/v1beta1/azureclusteridentity_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ limitations under the License.
package v1beta1

import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field"
webhookutils "sigs.k8s.io/cluster-api-provider-azure/util/webhook"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
Expand All @@ -40,7 +43,18 @@ func (c *AzureClusterIdentity) ValidateCreate() error {

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
func (c *AzureClusterIdentity) ValidateUpdate(oldRaw runtime.Object) error {
return c.validateClusterIdentity()
var allErrs field.ErrorList
old := oldRaw.(*AzureClusterIdentity)
if err := webhookutils.ValidateImmutable(
field.NewPath("Spec", "Type"),
old.Spec.Type,
c.Spec.Type); err != nil {
allErrs = append(allErrs, err)
}
if len(allErrs) == 0 {
return c.validateClusterIdentity()
}
return apierrors.NewInvalid(GroupVersion.WithKind("AzureClusterIdentity").GroupKind(), c.Name, allErrs)
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
Expand Down
5 changes: 4 additions & 1 deletion api/v1beta1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,7 @@ const (
)

// IdentityType represents different types of identities.
// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI;ManualServicePrincipal;ServicePrincipalCertificate
// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI;ManualServicePrincipal;ServicePrincipalCertificate;WorkloadIdentity
type IdentityType string

const (
Expand All @@ -550,6 +550,9 @@ const (

// ServicePrincipalCertificate represents a service principal using a certificate as secret.
ServicePrincipalCertificate IdentityType = "ServicePrincipalCertificate"

// WorkloadIdentity represents a WorkloadIdentity.
WorkloadIdentity IdentityType = "WorkloadIdentity"
)

// OSDisk defines the operating system disk for a VM.
Expand Down
10 changes: 10 additions & 0 deletions azure/scope/identity.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,16 @@ func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceMa
var authErr error
var cred azcore.TokenCredential
switch p.Identity.Spec.Type {
case infrav1.WorkloadIdentity:
azwiCredOptions, err := NewWorkloadIdentityCredentialOptions().
WithTenantID(p.Identity.Spec.TenantID).
WithClientID(p.Identity.Spec.ClientID).
WithDefaults()
if err != nil {
return nil, errors.Wrapf(err, "failed to setup azwi options for identity %s", p.Identity.Name)
}
cred, authErr = NewWorkloadIdentityCredential(azwiCredOptions)

case infrav1.ServicePrincipal, infrav1.ServicePrincipalCertificate, infrav1.UserAssignedMSI:
if err := createAzureIdentityWithBindings(ctx, p.Identity, resourceManagerEndpoint, activeDirectoryEndpoint, clusterMeta, p.Client); err != nil {
return nil, err
Expand Down
155 changes: 155 additions & 0 deletions azure/scope/workload_identity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scope

import (
"context"
"os"
"strings"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/pkg/errors"
)

/*
AZWI (Azure Workload Identity) required deploying AZWI mutating admission webhook
for self managed clusters e.g. Kind.
The webhook injects the following environment variables to the pod that
uses a service account with a annotation `azure.workload.identity/use=true`
|-----------------------------------------------------------------------------------|
|AZURE_AUTHORITY_HOST | The Azure Active Directory (AAD) endpoint. |
|AZURE_CLIENT_ID | The application/client ID of the Azure AD |
| | application or user-assigned managed identity. |
|AZURE_TENANT_ID | The tenant ID of the Azure subscription. |
|AZURE_FEDERATED_TOKEN_FILE | The path of the projected service account token file. |
|-----------------------------------------------------------------------------------|
In addition to the service account, it also projects a signed service account token to the
workload's volume(in this case the capz pod). The volume name is `azure-identity-token`
which is mounted at path `/var/run/secrets/azure/tokens/azure-identity-token` to the pod.
*/

const (
// AzureFedratedTokenFileENVKey is the env key for AZURE_FEDERATED_TOKEN_FILE.
AzureFedratedTokenFileENVKey = "AZURE_FEDERATED_TOKEN_FILE"
// AzureClientIDENVKey is the env key for AZURE_CLIENT_ID.
AzureClientIDENVKey = "AZURE_CLIENT_ID"
// AzureTenantIDENVKey is the env key for AZURE_TENANT_ID.
AzureTenantIDENVKey = "AZURE_TENANT_ID"
)

type workloadIdentityCredential struct {
assertion string
file string
cred *azidentity.ClientAssertionCredential
lastRead time.Time
}

// WorkloadIdentityCredentialOptions contains the configurable options for azwi.
type WorkloadIdentityCredentialOptions struct {
ClientID string
TenantID string
TokenFilePath string
azcore.ClientOptions
}

// NewWorkloadIdentityCredentialOptions returns an empty instance of WorkloadIdentityCredentialOptions.
func NewWorkloadIdentityCredentialOptions() *WorkloadIdentityCredentialOptions {
return &WorkloadIdentityCredentialOptions{}
}

// WithClientID sets client ID to WorkloadIdentityCredentialOptions.
func (w *WorkloadIdentityCredentialOptions) WithClientID(clientID string) *WorkloadIdentityCredentialOptions {
w.ClientID = clientID
return w
}

// WithTenantID sets tenant ID to WorkloadIdentityCredentialOptions.
func (w *WorkloadIdentityCredentialOptions) WithTenantID(tenantID string) *WorkloadIdentityCredentialOptions {
w.TenantID = tenantID
return w
}

// GetProjectedTokenPath return projected token file path from the env variable.
func GetProjectedTokenPath() (string, error) {
tokenPath := os.Getenv(AzureFedratedTokenFileENVKey)
if strings.TrimSpace(tokenPath) == "" {
return "", errors.New("projected token path not injected")
}
return tokenPath, nil
}

// WithDefaults sets token file path. It also sets the client tenant ID from injected env in
// case empty values are passed.
func (w *WorkloadIdentityCredentialOptions) WithDefaults() (*WorkloadIdentityCredentialOptions, error) {
tokenFilePath, err := GetProjectedTokenPath()
if err != nil {
return nil, errors.Wrap(err, "failed to get token file path for identity")
}
w.TokenFilePath = tokenFilePath

// Fallback to using client ID from env variable if not set.
if strings.TrimSpace(w.ClientID) == "" {
w.ClientID = os.Getenv(AzureClientIDENVKey)
if strings.TrimSpace(w.ClientID) == "" {
return nil, errors.New("empty client ID")
}
}

// // Fallback to using tenant ID from env variable.
if strings.TrimSpace(w.TenantID) == "" {
w.TenantID = os.Getenv(AzureTenantIDENVKey)
if strings.TrimSpace(w.TenantID) == "" {
return nil, errors.New("empty tenant ID")
}
}
return w, nil
}

// NewWorkloadIdentityCredential returns a workload identity credential.
func NewWorkloadIdentityCredential(options *WorkloadIdentityCredentialOptions) (*workloadIdentityCredential, error) {
w := &workloadIdentityCredential{file: options.TokenFilePath}
cred, err := azidentity.NewClientAssertionCredential(options.TenantID, options.ClientID, w.getAssertion, &azidentity.ClientAssertionCredentialOptions{ClientOptions: options.ClientOptions})
if err != nil {
return nil, err
}
w.cred = cred
return w, nil
}

// GetToken returns the token for azwi.
func (w *workloadIdentityCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) {
return w.cred.GetToken(ctx, opts)
}

func (w *workloadIdentityCredential) getAssertion(context.Context) (string, error) {
if now := time.Now(); w.lastRead.Add(5 * time.Minute).Before(now) {
content, err := os.ReadFile(w.file)
if err != nil {
return "", err
}
w.assertion = string(content)
w.lastRead = now
}
return w.assertion, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ spec:
- UserAssignedMSI
- ManualServicePrincipal
- ServicePrincipalCertificate
- WorkloadIdentity
type: string
required:
- clientID
Expand Down
Loading

0 comments on commit 5e8c72c

Please sign in to comment.