diff --git a/CHANGELOG.md b/CHANGELOG.md index 95d9588bc34..e7fcdef0b6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General:** Add explicit seccompProfile type to securityContext config ([#3561](https://github.com/kedacore/keda/issues/3561)) - **General:** Add `Min` column to ScaledJob visualization ([#3689](https://github.com/kedacore/keda/issues/3689)) +- **General:** Add support to use pod identities for authentication in Azure Key Vault ([#3813](https://github.com/kedacore/keda/issues/3813) - **Apache Kafka Scaler:** SASL/OAuthbearer Implementation ([#3681](https://github.com/kedacore/keda/issues/3681)) - **Azure AD Pod Identity Authentication:** Improve error messages to emphasize problems around the integration with aad-pod-identity itself ([#3610](https://github.com/kedacore/keda/issues/3610)) - **Azure Pipelines Scaler:** Improved speed of profiling large set of Job Requests from Azure Pipelines ([#3702](https://github.com/kedacore/keda/issues/3702)) diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index e8978096d2f..d97b827a5a7 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -188,13 +188,15 @@ type AzureKeyVault struct { // +optional Credentials *AzureKeyVaultCredentials `json:"credentials"` // +optional + PodIdentity *AuthPodIdentity `json:"podIdentity"` + // +optional Cloud *AzureKeyVaultCloudInfo `json:"cloud"` } type AzureKeyVaultCredentials struct { ClientID string `json:"clientId"` - ClientSecret *AzureKeyVaultClientSecret `json:"clientSecret"` TenantID string `json:"tenantId"` + ClientSecret *AzureKeyVaultClientSecret `json:"clientSecret"` } type AzureKeyVaultClientSecret struct { diff --git a/apis/keda/v1alpha1/zz_generated.deepcopy.go b/apis/keda/v1alpha1/zz_generated.deepcopy.go index b985d9b327f..c2a712e90b2 100644 --- a/apis/keda/v1alpha1/zz_generated.deepcopy.go +++ b/apis/keda/v1alpha1/zz_generated.deepcopy.go @@ -105,6 +105,11 @@ func (in *AzureKeyVault) DeepCopyInto(out *AzureKeyVault) { *out = new(AzureKeyVaultCredentials) (*in).DeepCopyInto(*out) } + if in.PodIdentity != nil { + in, out := &in.PodIdentity, &out.PodIdentity + *out = new(AuthPodIdentity) + **out = **in + } if in.Cloud != nil { in, out := &in.Cloud, &out.Cloud *out = new(AzureKeyVaultCloudInfo) diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index 7048ec3f85c..4d4bacb399a 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -98,6 +98,18 @@ spec: - clientSecret - tenantId type: object + podIdentity: + description: AuthPodIdentity allows users to select the platform + native identity mechanism + properties: + identityId: + type: string + provider: + description: PodIdentityProvider contains the list of providers + type: string + required: + - provider + type: object secrets: items: properties: diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index 1430a6138ec..a1897d688e9 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -97,6 +97,18 @@ spec: - clientSecret - tenantId type: object + podIdentity: + description: AuthPodIdentity allows users to select the platform + native identity mechanism + properties: + identityId: + type: string + provider: + description: PodIdentityProvider contains the list of providers + type: string + required: + - provider + type: object secrets: items: properties: diff --git a/pkg/scaling/resolver/azure_keyvault_handler.go b/pkg/scaling/resolver/azure_keyvault_handler.go index e323bab809c..a76d387b572 100644 --- a/pkg/scaling/resolver/azure_keyvault_handler.go +++ b/pkg/scaling/resolver/azure_keyvault_handler.go @@ -34,13 +34,11 @@ import ( type AzureKeyVaultHandler struct { vault *kedav1alpha1.AzureKeyVault keyvaultClient *keyvault.BaseClient - podIdentity kedav1alpha1.AuthPodIdentity } -func NewAzureKeyVaultHandler(v *kedav1alpha1.AzureKeyVault, podIdentity kedav1alpha1.AuthPodIdentity) *AzureKeyVaultHandler { +func NewAzureKeyVaultHandler(v *kedav1alpha1.AzureKeyVault) *AzureKeyVaultHandler { return &AzureKeyVaultHandler{ - vault: v, - podIdentity: podIdentity, + vault: v, } } @@ -104,7 +102,12 @@ func (vh *AzureKeyVaultHandler) getPropertiesForCloud() (string, string, error) func (vh *AzureKeyVaultHandler) getAuthConfig(ctx context.Context, client client.Client, logger logr.Logger, triggerNamespace, keyVaultResourceURL, activeDirectoryEndpoint string) (auth.AuthorizerConfig, error) { - switch vh.podIdentity.Provider { + podIdentity := vh.vault.PodIdentity + if podIdentity == nil { + podIdentity = &kedav1alpha1.AuthPodIdentity{} + } + + switch podIdentity.Provider { case "", kedav1alpha1.PodIdentityProviderNone: clientID := vh.vault.Credentials.ClientID tenantID := vh.vault.Credentials.TenantID @@ -125,12 +128,12 @@ func (vh *AzureKeyVaultHandler) getAuthConfig(ctx context.Context, client client case kedav1alpha1.PodIdentityProviderAzure: config := auth.NewMSIConfig() config.Resource = keyVaultResourceURL - config.ClientID = vh.podIdentity.IdentityID + config.ClientID = podIdentity.IdentityID return config, nil case kedav1alpha1.PodIdentityProviderAzureWorkload: - return azure.NewAzureADWorkloadIdentityConfig(ctx, vh.podIdentity.IdentityID, keyVaultResourceURL), nil + return azure.NewAzureADWorkloadIdentityConfig(ctx, podIdentity.IdentityID, keyVaultResourceURL), nil default: - return nil, fmt.Errorf("key vault does not support pod identity provider - %s", vh.podIdentity) + return nil, fmt.Errorf("key vault does not support pod identity provider - %s", podIdentity) } } diff --git a/pkg/scaling/resolver/azure_keyvault_handler_test.go b/pkg/scaling/resolver/azure_keyvault_handler_test.go index cff7139e53c..4a6740f567d 100644 --- a/pkg/scaling/resolver/azure_keyvault_handler_test.go +++ b/pkg/scaling/resolver/azure_keyvault_handler_test.go @@ -110,7 +110,7 @@ var testDataset = []testData{ func TestGetPropertiesForCloud(t *testing.T) { for _, testData := range testDataset { - vh := NewAzureKeyVaultHandler(&testData.vault, kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}) + vh := NewAzureKeyVaultHandler(&testData.vault) kvResourceURL, adEndpoint, err := vh.getPropertiesForCloud() diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index 1b47227cad1..41fe4bef4f5 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -215,7 +215,7 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge } } if triggerAuthSpec.AzureKeyVault != nil && len(triggerAuthSpec.AzureKeyVault.Secrets) > 0 { - vaultHandler := NewAzureKeyVaultHandler(triggerAuthSpec.AzureKeyVault, podIdentity) + vaultHandler := NewAzureKeyVaultHandler(triggerAuthSpec.AzureKeyVault) err := vaultHandler.Initialize(ctx, client, logger, triggerNamespace) if err != nil { logger.Error(err, "Error authenticating to Azure Key Vault", "triggerAuthRef.Name", triggerAuthRef.Name) diff --git a/tests/secret-providers/azure_keyvault/azure_keyvault_test.go b/tests/secret-providers/azure_keyvault/azure_keyvault_test.go index f0e6543c026..0b051c5cc7e 100644 --- a/tests/secret-providers/azure_keyvault/azure_keyvault_test.go +++ b/tests/secret-providers/azure_keyvault/azure_keyvault_test.go @@ -41,7 +41,7 @@ var ( deploymentName = fmt.Sprintf("%s-deployment", testName) triggerAuthName = fmt.Sprintf("%s-ta", testName) scaledObjectName = fmt.Sprintf("%s-so", testName) - queueName = fmt.Sprintf("%s-queue", testName) + queueName = fmt.Sprintf("%s-queue-%d", testName, GetRandomNumber()) ) type templateData struct { diff --git a/tests/secret-providers/azure_keyvault_workload_identity/azure_keyvault_workload_identity_test.go b/tests/secret-providers/azure_keyvault_workload_identity/azure_keyvault_workload_identity_test.go new file mode 100644 index 00000000000..38e824db89b --- /dev/null +++ b/tests/secret-providers/azure_keyvault_workload_identity/azure_keyvault_workload_identity_test.go @@ -0,0 +1,227 @@ +//go:build e2e +// +build e2e + +package azure_keyvault_workload_identity_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + "time" + + "github.com/Azure/azure-storage-queue-go/azqueue" + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/scalers/azure" + kedautil "github.com/kedacore/keda/v2/pkg/util" + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-keyvault-workload-identity-queue-test" +) + +var ( + connectionString = os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + keyvaultURI = os.Getenv("AZURE_KEYVAULT_URI") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + queueName = fmt.Sprintf("%s-queue-%d", testName, GetRandomNumber()) +) + +type templateData struct { + TestNamespace string + SecretName string + Connection string + DeploymentName string + TriggerAuthName string + ScaledObjectName string + QueueName string + KeyVaultURI string +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + AzureWebJobsStorage: {{.Connection}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}} + image: ghcr.io/kedacore/tests-azure-queue + resources: + env: + - name: FUNCTIONS_WORKER_RUNTIME + value: node + - name: AzureWebJobsStorage + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: AzureWebJobsStorage +` + + triggerAuthTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + azureKeyVault: + vaultUri: {{.KeyVaultURI}} + podIdentity: + provider: azure-workload + secrets: + - parameter: connection + name: E2E-Storage-ConnectionString +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: 1 + cooldownPeriod: 10 + triggers: + - type: azure-queue + metadata: + queueName: {{.QueueName}} + authenticationRef: + name: {{.TriggerAuthName}} +` +) + +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, connectionString, "AZURE_STORAGE_CONNECTION_STRING env variable is required for key vault tests") + require.NotEmpty(t, keyvaultURI, "AZURE_KEYVAULT_URI env variable is required for key vault tests") + + queueURL, messageURL := createQueue(t) + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after 1 minute") + + // test scaling + testScaleUp(t, kc, messageURL) + testScaleDown(t, kc, messageURL) + + // cleanup + DeleteKubernetesResources(t, kc, testNamespace, data, templates) + cleanupQueue(t, queueURL) +} + +func createQueue(t *testing.T) (azqueue.QueueURL, azqueue.MessagesURL) { + // Create Queue + httpClient := kedautil.CreateHTTPClient(DefaultHTTPTimeOut, false) + credential, endpoint, err := azure.ParseAzureStorageQueueConnection( + context.Background(), httpClient, kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderNone}, + connectionString, "", "") + assert.NoErrorf(t, err, "cannot parse storage connection string - %s", err) + + p := azqueue.NewPipeline(credential, azqueue.PipelineOptions{}) + serviceURL := azqueue.NewServiceURL(*endpoint, p) + queueURL := serviceURL.NewQueueURL(queueName) + + _, err = queueURL.Create(context.Background(), azqueue.Metadata{}) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) + + messageURL := queueURL.NewMessagesURL() + + return queueURL, messageURL +} + +func getTemplateData() (templateData, []Template) { + base64ConnectionString := base64.StdEncoding.EncodeToString([]byte(connectionString)) + + return templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + Connection: base64ConnectionString, + DeploymentName: deploymentName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + QueueName: queueName, + KeyVaultURI: keyvaultURI, + }, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "triggerAuthTemplate", Config: triggerAuthTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + } +} + +func testScaleUp(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale up ---") + for i := 0; i < 5; i++ { + msg := fmt.Sprintf("Message - %d", i) + _, err := messageURL.Enqueue(context.Background(), msg, 0*time.Second, time.Hour) + assert.NoErrorf(t, err, "cannot enqueue message - %s", err) + } + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 1, 60, 1), + "replica count should be 0 after 1 minute") +} + +func testScaleDown(t *testing.T, kc *kubernetes.Clientset, messageURL azqueue.MessagesURL) { + t.Log("--- testing scale down ---") + _, err := messageURL.Clear(context.Background()) + assert.NoErrorf(t, err, "cannot clear queue - %s", err) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after 1 minute") +} + +func cleanupQueue(t *testing.T, queueURL azqueue.QueueURL) { + t.Log("--- cleaning up ---") + _, err := queueURL.Delete(context.Background()) + assert.NoErrorf(t, err, "cannot create storage queue - %s", err) +}