ci: migrate azure jobs to eks prow cluster

Makefile

@@ -61,12 +61,12 @@ export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL
 
 # Generate all combination of all OS, ARCH, and OSVERSIONS for iteration
 ALL_OS = linux windows
-ALL_ARCH.linux = amd64 arm64
-ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
-ALL_ARCH.windows = amd64
-ALL_OSVERSIONS.windows := 1809 ltsc2022
-ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-${osversion}-${arch}))
-ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}})
+ALL_ARCH_linux ?= amd64 arm64
+ALL_OS_ARCH_linux = $(foreach arch, ${ALL_ARCH_linux}, linux-$(arch))
+ALL_ARCH_windows = amd64
+ALL_OSVERSIONS_windows := 1809 ltsc2022
+ALL_OS_ARCH_windows = $(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSIONS_windows}, windows-${osversion}-${arch}))
+ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH_${os}})
 
 # The current context of image building
 # The architecture of the image
@@ -328,24 +328,24 @@ docker-buildx-builder:
 
 .PHONY: container-all
 container-all: docker-buildx-builder
-	for arch in $(ALL_ARCH.linux); do \
+	for arch in $(ALL_ARCH_linux); do \
 		ARCH=$${arch} $(MAKE) container-linux; \
 		ARCH=$${arch} $(MAKE) crd-container-linux; \
 	done
-	for osversion in $(ALL_OSVERSIONS.windows); do \
+	for osversion in $(ALL_OSVERSIONS_windows); do \
   		OSVERSION=$${osversion} $(MAKE) container-windows; \
   	done
 
 .PHONY: push-manifest
 push-manifest:
 	docker manifest create --amend $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
-	docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH.linux), $(CRD_IMAGE_TAG)-${osarch})
+	docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH_linux), $(CRD_IMAGE_TAG)-${osarch})
 	# add "os.version" field to windows images (based on https://github.com/kubernetes/kubernetes/blob/master/build/pause/Makefile)
 	set -x; \
 	registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq ".*[\/\.].*") && echo "" || echo "docker.io/"); \
 	manifest_image_folder=`echo "$${registry_prefix}${IMAGE_TAG}" | sed "s|/|_|g" | sed "s/:/-/"`; \
-	for arch in $(ALL_ARCH.windows); do \
-		for osversion in $(ALL_OSVERSIONS.windows); do \
+	for arch in $(ALL_ARCH_windows); do \
+		for osversion in $(ALL_OSVERSIONS_windows); do \
 			BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \
 			full_version=`docker manifest inspect $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \
 			sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":\"$${full_version}\"/" "${HOME}/.docker/manifests/$${manifest_image_folder}/$${manifest_image_folder}-windows-$${osversion}-$${arch}"; \
@@ -359,11 +359,12 @@ push-manifest:
 ## --------------------------------------
 ## E2E Testing
 ## --------------------------------------
+.PHONY: e2e-install-prerequisites
+e2e-install-prerequisites: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ)
+
 .PHONY: e2e-bootstrap
-e2e-bootstrap: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ) #setup all required binaries and kind cluster for testing
-ifndef TEST_WINDOWS
+e2e-bootstrap: e2e-install-prerequisites #setup all required binaries and kind cluster for testing
 	$(MAKE) setup-kind
-endif
 	docker pull $(IMAGE_TAG) || $(MAKE) e2e-container
 
 .PHONY: setup-kind
@@ -378,12 +379,8 @@ setup-eks-cluster: $(HELM) $(EKSCTL) $(BATS) $(ENVSUBST) $(YQ)
 
 .PHONY: e2e-container
 e2e-container:
-ifdef TEST_WINDOWS
-	$(MAKE) container-all push-manifest
-else
 	$(MAKE) container
 	kind load docker-image --name kind $(IMAGE_TAG) $(CRD_IMAGE_TAG)
-endif
 
 .PHONY: e2e-mock-provider-container
 e2e-mock-provider-container:
@@ -438,7 +435,8 @@ e2e-helm-deploy:
 		--set rotationPollInterval=30s \
 		--set tokenRequests[0].audience="aud1" \
 		--set tokenRequests[1].audience="aud2" \
-		--set tokenRequests[2].audience="conjur"
+		--set tokenRequests[2].audience="conjur" \
+		--set tokenRequests[3].audience="api://AzureADTokenExchange"
 
 .PHONY: e2e-helm-upgrade
 e2e-helm-upgrade:

test/bats/azure.bats

@@ -16,17 +16,10 @@ if [ $TEST_WINDOWS ]; then
   NODE_SELECTOR_OS=windows
 fi
 
-if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then
-    export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6)
-fi
-
-export KEYVAULT_NAME=${KEYVAULT_NAME:-csi-secrets-store-e2e}
+export KEYVAULT_NAME=${KEYVAULT_NAME:-secrets-store-csi-e2e}
 export SECRET_NAME=${KEYVAULT_SECRET_NAME:-secret1}
 export SECRET_VERSION=${KEYVAULT_SECRET_VERSION:-""}
 export SECRET_VALUE=${KEYVAULT_SECRET_VALUE:-"test"}
-export KEY_NAME=${KEYVAULT_KEY_NAME:-key1}
-export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a}
-export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"}
 export LABEL_VALUE=${LABEL_VALUE:-"test"}
 export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
 
@@ -35,8 +28,8 @@ export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
 export API_VERSION=$(get_secrets_store_api_version)
 
 setup() {
-  if [[ -z "${AZURE_CLIENT_ID}" ]] || [[ -z "${AZURE_CLIENT_SECRET}" ]]; then
-    echo "Error: Azure service principal is not provided" >&2
+  if [[ -z "${IDENTITY_CLIENT_ID}" ]]; then
+    echo "Error: Azure managed identity id is not provided" >&2
     return 1
   fi
 }
@@ -55,15 +48,6 @@ setup() {
   kubectl wait --for=condition=Ready --timeout=150s pods -l app=csi-secrets-store-provider-azure --namespace $NAMESPACE
 }
 
-@test "create azure k8s secret" {
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET}
-  assert_success
-
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
-  assert_success
-}
-
 @test "deploy azure secretproviderclass crd" {
   envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass.yaml | kubectl apply -f -
 
@@ -92,12 +76,6 @@ setup() {
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 }
 
-@test "CSI inline volume test with pod portability - read azure kv key from pod" {
-  result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
-}
-
 @test "CSI inline volume test with pod portability - unmount succeeds" {
   # On Linux a failure to unmount the tmpfs will block the pod from being
   # deleted.
@@ -140,10 +118,6 @@ setup() {
   result=$(kubectl exec $POD -- cat /mnt/secrets-store/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
-
   result=$(kubectl get secret foosecret -o jsonpath="{.data.username}" | base64 -d)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
@@ -180,13 +154,6 @@ setup() {
   run kubectl create ns test-ns
   assert_success
 
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns
-  assert_success
-
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns
-  assert_success
-
   envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass_ns.yaml | kubectl apply -f -
 
   kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
@@ -208,10 +175,6 @@ setup() {
   result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
-
   result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.username}" | base64 -d)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
@@ -234,13 +197,6 @@ setup() {
   run kubectl create ns negative-test-ns
   assert_success
 
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns
-  assert_success
-
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns
-  assert_success
-
   envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f -
   sleep 5
 
@@ -280,17 +236,9 @@ setup() {
   result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
-
   result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/secretalias)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
-  result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME)
-  result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
-  [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
-
   result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.username}" | base64 -d)
   [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
 
@@ -310,66 +258,11 @@ setup() {
   assert_success
 }
 
-@test "Test auto rotation of mount contents and K8s secrets - Create deployment" {
-  run kubectl create ns rotation
-  assert_success
-
-  run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n rotation
-  assert_success
-
-  # label the node publish secret ref secret
-  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n rotation
-  assert_success
-
-  run az login -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} -t ${TENANT_ID} --service-principal
-  assert_success
-
-  run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value secret
-  assert_success
-
-  envsubst < $BATS_TESTS_DIR/rotation/azure_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f -
-  envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-azure.yaml | kubectl apply -n rotation -f -
-
-  kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation
-
-  run kubectl get pod/secrets-store-inline-rotation -n rotation
-  assert_success
-}
-
-@test "Test auto rotation of mount contents and K8s secrets" {
-  result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
-  [[ "${result//$'\r'}" == "secret" ]]
-
-  result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
-  [[ "${result//$'\r'}" == "secret" ]]
-
-  run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value rotated
-  assert_success
-
-  sleep 60
-
-  result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
-  [[ "${result//$'\r'}" == "rotated" ]]
-
-  result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
-  [[ "${result//$'\r'}" == "rotated" ]]
-
-  run az keyvault secret delete --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME}
-  assert_success
-
-  run az logout
-  assert_success
-}
-
 teardown_file() {
   archive_provider "app=csi-secrets-store-provider-azure" || true
   archive_info || true
 
   #cleanup
-  run kubectl delete namespace rotation
   run kubectl delete namespace test-ns
-
-  run kubectl delete secret secrets-store-creds
-
   run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0
 }

test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml

@@ -13,7 +13,7 @@ spec:
     - objectName: secretalias                    # name of the mounted content to sync. this could be the object name or object alias 
       key: username
   parameters:
-    usePodIdentity: "false"                      # [OPTIONAL] if not provided, will default to "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"               # the name of the KeyVault
     objects: |
       array:
@@ -22,8 +22,4 @@ spec:
           objectType: secret                     # object types: secret, key or cert
           objectAlias: secretalias
           objectVersion: $SECRET_VERSION         # [OPTIONAL] object versions, default to latest if empty
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
+    tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault

test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml

@@ -11,7 +11,7 @@ spec:
     - objectName: secretalias
       key: username
   parameters:
-    usePodIdentity: "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"
     objects: |
       array:
@@ -20,11 +20,7 @@ spec:
           objectType: secret
           objectVersion: $SECRET_VERSION
           objectAlias: secretalias
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID"
+    tenantId: "$AZURE_TENANT_ID"
 ---
 apiVersion: $API_VERSION
 kind: SecretProviderClass
@@ -39,7 +35,7 @@ spec:
     - objectName: secretalias
       key: username
   parameters:
-    usePodIdentity: "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME"
     objects: |
       array:
@@ -48,8 +44,4 @@ spec:
           objectType: secret
           objectVersion: $SECRET_VERSION
           objectAlias: secretalias
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID"
+    tenantId: "$AZURE_TENANT_ID"

test/bats/tests/azure/azure_v1_secretproviderclass.yaml

@@ -5,16 +5,12 @@ metadata:
 spec:
   provider: azure
   parameters:
-    usePodIdentity: "false"         # [OPTIONAL] if not provided, will default to "false"
+    clientID: "$IDENTITY_CLIENT_ID"
     keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault
     objects: |
       array:
         - |
           objectName: $SECRET_NAME
           objectType: secret        # object types: secret, key or cert
           objectVersion: $SECRET_VERSION         # [OPTIONAL] object versions, default to latest if empty
-        - |
-          objectName: $KEY_NAME
-          objectType: key
-          objectVersion: $KEY_VERSION
-    tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
+    tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault