diff --git a/Makefile b/Makefile index 242335f90..d15def2c7 100644 --- a/Makefile +++ b/Makefile @@ -61,12 +61,12 @@ export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL # Generate all combination of all OS, ARCH, and OSVERSIONS for iteration ALL_OS = linux windows -ALL_ARCH.linux = amd64 arm64 -ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch)) -ALL_ARCH.windows = amd64 -ALL_OSVERSIONS.windows := 1809 ltsc2022 -ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-${osversion}-${arch})) -ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}}) +ALL_ARCH_linux ?= amd64 arm64 +ALL_OS_ARCH_linux = $(foreach arch, ${ALL_ARCH_linux}, linux-$(arch)) +ALL_ARCH_windows = amd64 +ALL_OSVERSIONS_windows := 1809 ltsc2022 +ALL_OS_ARCH_windows = $(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSIONS_windows}, windows-${osversion}-${arch})) +ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH_${os}}) # The current context of image building # The architecture of the image @@ -328,24 +328,24 @@ docker-buildx-builder: .PHONY: container-all container-all: docker-buildx-builder - for arch in $(ALL_ARCH.linux); do \ + for arch in $(ALL_ARCH_linux); do \ ARCH=$${arch} $(MAKE) container-linux; \ ARCH=$${arch} $(MAKE) crd-container-linux; \ done - for osversion in $(ALL_OSVERSIONS.windows); do \ + for osversion in $(ALL_OSVERSIONS_windows); do \ OSVERSION=$${osversion} $(MAKE) container-windows; \ done .PHONY: push-manifest push-manifest: docker manifest create --amend $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch}) - docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH.linux), $(CRD_IMAGE_TAG)-${osarch}) + docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH_linux), $(CRD_IMAGE_TAG)-${osarch}) # add "os.version" field to windows images (based on https://github.com/kubernetes/kubernetes/blob/master/build/pause/Makefile) set -x; \ registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq ".*[\/\.].*") && echo "" || echo "docker.io/"); \ manifest_image_folder=`echo "$${registry_prefix}${IMAGE_TAG}" | sed "s|/|_|g" | sed "s/:/-/"`; \ - for arch in $(ALL_ARCH.windows); do \ - for osversion in $(ALL_OSVERSIONS.windows); do \ + for arch in $(ALL_ARCH_windows); do \ + for osversion in $(ALL_OSVERSIONS_windows); do \ BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \ full_version=`docker manifest inspect $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \ sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":\"$${full_version}\"/" "${HOME}/.docker/manifests/$${manifest_image_folder}/$${manifest_image_folder}-windows-$${osversion}-$${arch}"; \ @@ -359,11 +359,12 @@ push-manifest: ## -------------------------------------- ## E2E Testing ## -------------------------------------- +.PHONY: e2e-install-prerequisites +e2e-install-prerequisites: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ) + .PHONY: e2e-bootstrap -e2e-bootstrap: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ) #setup all required binaries and kind cluster for testing -ifndef TEST_WINDOWS +e2e-bootstrap: e2e-install-prerequisites #setup all required binaries and kind cluster for testing $(MAKE) setup-kind -endif docker pull $(IMAGE_TAG) || $(MAKE) e2e-container .PHONY: setup-kind @@ -378,12 +379,8 @@ setup-eks-cluster: $(HELM) $(EKSCTL) $(BATS) $(ENVSUBST) $(YQ) .PHONY: e2e-container e2e-container: -ifdef TEST_WINDOWS - $(MAKE) container-all push-manifest -else $(MAKE) container kind load docker-image --name kind $(IMAGE_TAG) $(CRD_IMAGE_TAG) -endif .PHONY: e2e-mock-provider-container e2e-mock-provider-container: @@ -437,7 +434,9 @@ e2e-helm-deploy: --set enableSecretRotation=true \ --set rotationPollInterval=30s \ --set tokenRequests[0].audience="aud1" \ - --set tokenRequests[1].audience="aud2" + --set tokenRequests[1].audience="aud2" \ + --set tokenRequests[2].audience="conjur" \ + --set tokenRequests[3].audience="api://AzureADTokenExchange" .PHONY: e2e-helm-upgrade e2e-helm-upgrade: diff --git a/test/bats/azure.bats b/test/bats/azure.bats index dd2e03c04..81cc36276 100644 --- a/test/bats/azure.bats +++ b/test/bats/azure.bats @@ -16,21 +16,10 @@ if [ $TEST_WINDOWS ]; then NODE_SELECTOR_OS=windows fi -if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then - export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6) -fi - -if [ -z "$IS_YAML_TEST" ]; then - export IS_YAML_TEST=false -fi - -export KEYVAULT_NAME=${KEYVAULT_NAME:-csi-secrets-store-e2e} +export KEYVAULT_NAME=${KEYVAULT_NAME:-secrets-store-csi-e2e} export SECRET_NAME=${KEYVAULT_SECRET_NAME:-secret1} export SECRET_VERSION=${KEYVAULT_SECRET_VERSION:-""} export SECRET_VALUE=${KEYVAULT_SECRET_VALUE:-"test"} -export KEY_NAME=${KEYVAULT_KEY_NAME:-key1} -export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a} -export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"} export LABEL_VALUE=${LABEL_VALUE:-"test"} export NODE_SELECTOR_OS=$NODE_SELECTOR_OS @@ -39,8 +28,8 @@ export NODE_SELECTOR_OS=$NODE_SELECTOR_OS export API_VERSION=$(get_secrets_store_api_version) setup() { - if [[ -z "${AZURE_CLIENT_ID}" ]] || [[ -z "${AZURE_CLIENT_SECRET}" ]]; then - echo "Error: Azure service principal is not provided" >&2 + if [[ -z "${IDENTITY_CLIENT_ID}" ]]; then + echo "Error: Azure managed identity id is not provided" >&2 return 1 fi } @@ -59,15 +48,6 @@ setup() { kubectl wait --for=condition=Ready --timeout=150s pods -l app=csi-secrets-store-provider-azure --namespace $NAMESPACE } -@test "create azure k8s secret" { - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} - assert_success - - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true - assert_success -} - @test "deploy azure secretproviderclass crd" { envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass.yaml | kubectl apply -f - @@ -96,12 +76,6 @@ setup() { [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] } -@test "CSI inline volume test with pod portability - read azure kv key from pod" { - result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] -} - @test "CSI inline volume test with pod portability - unmount succeeds" { # On Linux a failure to unmount the tmpfs will block the pod from being # deleted. @@ -144,10 +118,6 @@ setup() { result=$(kubectl exec $POD -- cat /mnt/secrets-store/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] - result=$(kubectl get secret foosecret -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -184,13 +154,6 @@ setup() { run kubectl create ns test-ns assert_success - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns - assert_success - - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns - assert_success - envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass_ns.yaml | kubectl apply -f - kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io @@ -212,10 +175,6 @@ setup() { result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] - result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -238,13 +197,6 @@ setup() { run kubectl create ns negative-test-ns assert_success - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns - assert_success - - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns - assert_success - envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f - sleep 5 @@ -284,17 +236,9 @@ setup() { result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] - result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/secretalias) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] - result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] - result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] @@ -314,66 +258,11 @@ setup() { assert_success } -@test "Test auto rotation of mount contents and K8s secrets - Create deployment" { - run kubectl create ns rotation - assert_success - - run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n rotation - assert_success - - # label the node publish secret ref secret - run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n rotation - assert_success - - run az login -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} -t ${TENANT_ID} --service-principal - assert_success - - run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value secret - assert_success - - envsubst < $BATS_TESTS_DIR/rotation/azure_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f - - envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-azure.yaml | kubectl apply -n rotation -f - - - kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation - - run kubectl get pod/secrets-store-inline-rotation -n rotation - assert_success -} - -@test "Test auto rotation of mount contents and K8s secrets" { - result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias) - [[ "${result//$'\r'}" == "secret" ]] - - result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d) - [[ "${result//$'\r'}" == "secret" ]] - - run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value rotated - assert_success - - sleep 60 - - result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias) - [[ "${result//$'\r'}" == "rotated" ]] - - result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d) - [[ "${result//$'\r'}" == "rotated" ]] - - run az keyvault secret delete --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} - assert_success - - run az logout - assert_success -} - teardown_file() { archive_provider "app=csi-secrets-store-provider-azure" || true archive_info || true #cleanup - run kubectl delete namespace rotation run kubectl delete namespace test-ns - - run kubectl delete secret secrets-store-creds - run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0 } diff --git a/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml b/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml index 55e2979d5..a57650606 100644 --- a/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml +++ b/test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml @@ -13,7 +13,7 @@ spec: - objectName: secretalias # name of the mounted content to sync. this could be the object name or object alias key: username parameters: - usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault objects: | array: @@ -22,8 +22,4 @@ spec: objectType: secret # object types: secret, key or cert objectAlias: secretalias objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" # the tenant ID of the KeyVault + tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault diff --git a/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml b/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml index 5655b1eeb..441cad7be 100644 --- a/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml +++ b/test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml @@ -11,7 +11,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -20,11 +20,7 @@ spec: objectType: secret objectVersion: $SECRET_VERSION objectAlias: secretalias - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + tenantId: "$AZURE_TENANT_ID" --- apiVersion: $API_VERSION kind: SecretProviderClass @@ -39,7 +35,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -48,8 +44,4 @@ spec: objectType: secret objectVersion: $SECRET_VERSION objectAlias: secretalias - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + tenantId: "$AZURE_TENANT_ID" diff --git a/test/bats/tests/azure/azure_v1_secretproviderclass.yaml b/test/bats/tests/azure/azure_v1_secretproviderclass.yaml index 20ac1507c..a804c219f 100644 --- a/test/bats/tests/azure/azure_v1_secretproviderclass.yaml +++ b/test/bats/tests/azure/azure_v1_secretproviderclass.yaml @@ -5,7 +5,7 @@ metadata: spec: provider: azure parameters: - usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault objects: | array: @@ -13,8 +13,4 @@ spec: objectName: $SECRET_NAME objectType: secret # object types: secret, key or cert objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" # the tenant ID of the KeyVault + tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault diff --git a/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml b/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml index d48d5e588..5a0f32c04 100644 --- a/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml +++ b/test/bats/tests/azure/azure_v1_secretproviderclass_ns.yaml @@ -12,7 +12,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -21,11 +21,7 @@ spec: objectType: secret objectAlias: secretalias objectVersion: $SECRET_VERSION - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + tenantId: "$AZURE_TENANT_ID" --- apiVersion: $API_VERSION kind: SecretProviderClass @@ -41,7 +37,7 @@ spec: - objectName: secretalias key: username parameters: - usePodIdentity: "false" + clientID: "$IDENTITY_CLIENT_ID" keyvaultName: "$KEYVAULT_NAME" objects: | array: @@ -50,8 +46,4 @@ spec: objectType: secret objectAlias: secretalias objectVersion: $SECRET_VERSION - - | - objectName: $KEY_NAME - objectType: key - objectVersion: $KEY_VERSION - tenantId: "$TENANT_ID" + tenantId: "$AZURE_TENANT_ID" diff --git a/test/bats/tests/azure/deployment-synck8s-azure.yaml b/test/bats/tests/azure/deployment-synck8s-azure.yaml index 880931bb1..13818e39f 100644 --- a/test/bats/tests/azure/deployment-synck8s-azure.yaml +++ b/test/bats/tests/azure/deployment-synck8s-azure.yaml @@ -39,7 +39,5 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure-sync" - nodePublishSecretRef: - name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/deployment-two-synck8s-azure.yaml b/test/bats/tests/azure/deployment-two-synck8s-azure.yaml index 8c2d05e81..26c61c7cb 100644 --- a/test/bats/tests/azure/deployment-two-synck8s-azure.yaml +++ b/test/bats/tests/azure/deployment-two-synck8s-azure.yaml @@ -39,7 +39,5 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure-sync" - nodePublishSecretRef: - name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml b/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml index 53274cc4d..822101987 100644 --- a/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml +++ b/test/bats/tests/azure/pod-azure-inline-volume-multiple-spc.yaml @@ -36,15 +36,11 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure-spc-0" - nodePublishSecretRef: - name: secrets-store-creds - name: secrets-store-inline-1 csi: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: secretProviderClass: "azure-spc-1" - nodePublishSecretRef: - name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml b/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml index 8223e8b74..14bc4fc0c 100644 --- a/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml +++ b/test/bats/tests/azure/pod-secrets-store-inline-volume-crd.yaml @@ -22,7 +22,5 @@ spec: readOnly: true volumeAttributes: secretProviderClass: "azure" - nodePublishSecretRef: - name: secrets-store-creds nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/bats/tests/azure/rotation/azure_synck8s_v1_secretproviderclass.yaml b/test/bats/tests/azure/rotation/azure_synck8s_v1_secretproviderclass.yaml deleted file mode 100644 index a26e7e98b..000000000 --- a/test/bats/tests/azure/rotation/azure_synck8s_v1_secretproviderclass.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: $API_VERSION -kind: SecretProviderClass -metadata: - name: azure-auto-rotation -spec: - provider: azure - secretObjects: # [OPTIONAL] SecretObject defines the desired state of synced K8s secret objects - - secretName: rotationsecret - type: Opaque - labels: - environment: "test" - data: - - objectName: secretalias # name of the mounted content to sync. this could be the object name or object alias - key: username - parameters: - usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false" - keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault - objects: | - array: - - | - objectName: $AUTO_ROTATE_SECRET_NAME - objectType: secret - objectAlias: secretalias - tenantId: "$TENANT_ID" # the tenant ID of the KeyVault diff --git a/test/bats/tests/azure/rotation/pod-synck8s-azure.yaml b/test/bats/tests/azure/rotation/pod-synck8s-azure.yaml deleted file mode 100644 index ced17a854..000000000 --- a/test/bats/tests/azure/rotation/pod-synck8s-azure.yaml +++ /dev/null @@ -1,34 +0,0 @@ -kind: Pod -apiVersion: v1 -metadata: - name: secrets-store-inline-rotation -spec: - terminationGracePeriodSeconds: 0 - containers: - - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 - name: busybox - imagePullPolicy: IfNotPresent - command: - - "/bin/sleep" - - "10000" - env: - - name: SECRET_USERNAME - valueFrom: - secretKeyRef: - name: rotationsecret - key: username - volumeMounts: - - name: secrets-store-inline - mountPath: "/mnt/secrets-store" - readOnly: true - volumes: - - name: secrets-store-inline - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: "azure-auto-rotation" - nodePublishSecretRef: - name: secrets-store-creds - nodeSelector: - kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/scripts/run-e2e-azure.sh b/test/scripts/run-e2e-azure.sh new file mode 100755 index 000000000..6fe525436 --- /dev/null +++ b/test/scripts/run-e2e-azure.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +: "${REGISTRY:?Environment variable empty or not defined.}" + +readonly CLUSTER_NAME="${CLUSTER_NAME:-sscsi-e2e-$(openssl rand -hex 2)}" +readonly KEYVAULT_NAME="secrets-store-csi-e2e" + +IMAGE_VERSION=e2e-$(git rev-parse --short HEAD) +IMAGE_TAG=${REGISTRY}/driver:${IMAGE_VERSION} + +get_random_region() { + local REGIONS=("eastus" "eastus2" "uksouth") + echo "${REGIONS[${RANDOM} % ${#REGIONS[@]}]}" +} + +cleanup() { + echo "Deleting the AKS cluster ${CLUSTER_NAME}" + az login --service-principal -u "${AZURE_CLIENT_ID}" --t "${AZURE_TENANT_ID}" --federated-token "$(cat "${AZURE_FEDERATED_TOKEN_FILE}")" > /dev/null + az account set --subscription "${AZURE_SUBSCRIPTION_ID}" > /dev/null + az group delete --name "${CLUSTER_NAME}" --yes --no-wait || true +} +trap cleanup EXIT + +main() { + # install azure cli + curl -sL https://aka.ms/InstallAzureCLIDeb | bash > /dev/null + + echo "Logging into Azure" + az login --service-principal -u "${AZURE_CLIENT_ID}" --t "${AZURE_TENANT_ID}" --federated-token "$(cat "${AZURE_FEDERATED_TOKEN_FILE}")" > /dev/null + az account set --subscription "${AZURE_SUBSCRIPTION_ID}" > /dev/null + + LOCATION=$(get_random_region) + echo "Creating AKS cluster ${CLUSTER_NAME} in ${LOCATION}" + az group create --name "${CLUSTER_NAME}" --location "${LOCATION}" > /dev/null + az aks create \ + --resource-group "${CLUSTER_NAME}" \ + --name "${CLUSTER_NAME}" \ + --node-count 1 \ + --node-vm-size Standard_DS3_v2 \ + --enable-managed-identity \ + --network-plugin azure \ + --enable-oidc-issuer \ + --generate-ssh-keys > /dev/null + + # only add windows pool if TEST_WINDOWS is set and equal to true + if [[ "${TEST_WINDOWS:-}" == "true" ]]; then + echo "Adding windows nodepool" + # add windows nodepool + az aks nodepool add \ + --resource-group "${CLUSTER_NAME}" \ + --cluster-name "${CLUSTER_NAME}" \ + --os-type Windows \ + --name npwin \ + --node-count 1 > /dev/null + fi + + az aks get-credentials --resource-group "${CLUSTER_NAME}" --name "${CLUSTER_NAME}" --overwrite-existing + + # confirm the cluster is up and running + kubectl get nodes -o wide + kubectl get pods -A + + if [[ "${REGISTRY}" =~ \.azurecr\.io ]]; then + az acr login --name "${REGISTRY}" + fi + + AKS_CLUSTER_OIDC_ISSUER_URL=$(az aks show -g "${CLUSTER_NAME}" -n "${CLUSTER_NAME}" --query "oidcIssuerProfile.issuerUrl" -otsv) + # Create managed identity that'll be used by the provider to access keyvault + echo "Creating managed identity" + user_assigned_identity_name="sscsi-e2e-$(openssl rand -hex 2)" + az identity create --resource-group "${CLUSTER_NAME}" --name "${user_assigned_identity_name}" > /dev/null + IDENTITY_CLIENT_ID=$(az identity show --resource-group "${CLUSTER_NAME}" --name "${user_assigned_identity_name}" --query 'clientId' -otsv) + export IDENTITY_CLIENT_ID + IDENTITY_OBJECT_ID=$(az identity show --resource-group "${CLUSTER_NAME}" --name "${user_assigned_identity_name}" --query 'principalId' -otsv) + + # Create the federated identity credential (FIC) for the managed identity + echo "Creating federated identity credential for default:default" + az identity federated-credential create --name "kubernetes-federated-credential-default" \ + --identity-name "${user_assigned_identity_name}" \ + --resource-group "${CLUSTER_NAME}" \ + --issuer "${AKS_CLUSTER_OIDC_ISSUER_URL}" \ + --subject "system:serviceaccount:default:default" > /dev/null + + echo "Creating federated identity credential for test-ns:default" + az identity federated-credential create --name "kubernetes-federated-credential-test-ns" \ + --identity-name "${user_assigned_identity_name}" \ + --resource-group "${CLUSTER_NAME}" \ + --issuer "${AKS_CLUSTER_OIDC_ISSUER_URL}" \ + --subject "system:serviceaccount:test-ns:default" > /dev/null + + echo "Creating federated identity credential for negative-test-ns:default" + az identity federated-credential create --name "kubernetes-federated-credential-negative-test-ns" \ + --identity-name "${user_assigned_identity_name}" \ + --resource-group "${CLUSTER_NAME}" \ + --issuer "${AKS_CLUSTER_OIDC_ISSUER_URL}" \ + --subject "system:serviceaccount:negative-test-ns:default" > /dev/null + + # Assigning the managed identity the necessary permissions to access the keyvault + echo "Assigning managed identity permissions to get secrets from keyvault" + az keyvault set-policy --name "${KEYVAULT_NAME}" --secret-permissions get --object-id "${IDENTITY_OBJECT_ID}" > /dev/null + + docker pull "${IMAGE_TAG}" || ALL_ARCH_linux=amd64 make container-all push-manifest + make e2e-install-prerequisites + + if [[ ${RELEASE:-} == "true" ]]; then + make e2e-helm-deploy-release + else + make e2e-helm-deploy + fi + + # Run the e2e tests + make e2e-azure +} + +main diff --git a/test/scripts/run-e2e-windows.sh b/test/scripts/run-e2e-windows.sh deleted file mode 100755 index a214d1b99..000000000 --- a/test/scripts/run-e2e-windows.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2024 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -: "${REGISTRY:?Environment variable empty or not defined.}" - -readonly CLUSTER_NAME="${CLUSTER_NAME:-sscsi-e2e-$(openssl rand -hex 2)}" - -parse_cred() { - grep -E -o "\b$1[[:blank:]]*=[[:blank:]]*\"[^[:space:]\"]+\"" | cut -d '"' -f 2 -} - -get_random_region() { - local REGIONS=("eastus" "eastus2" "southcentralus" "westeurope" "uksouth") - echo "${REGIONS[${RANDOM} % ${#REGIONS[@]}]}" -} - -cleanup() { - echo "Deleting the AKS cluster ${CLUSTER_NAME}" - # login again because the bats tests might have logged out after rotating the secret for testing - az login --service-principal -u "${client_id}" -p "${client_secret}" --tenant "${tenant_id}" > /dev/null - az account set --subscription "${subscription_id}" > /dev/null - az group delete --name "${CLUSTER_NAME}" --yes --no-wait || true -} -trap cleanup EXIT - -main() { - # install azure cli - curl -sL https://aka.ms/InstallAzureCLIDeb | bash > /dev/null - - echo "Logging into Azure" - az login --service-principal -u "${client_id}" -p "${client_secret}" --tenant "${tenant_id}" > /dev/null - az account set --subscription "${subscription_id}" > /dev/null - - LOCATION=$(get_random_region) - echo "Creating AKS cluster ${CLUSTER_NAME} in ${LOCATION}" - az group create --name "${CLUSTER_NAME}" --location "${LOCATION}" > /dev/null - az aks create \ - --resource-group "${CLUSTER_NAME}" \ - --name "${CLUSTER_NAME}" \ - --node-count 1 \ - --node-vm-size Standard_DS3_v2 \ - --enable-managed-identity \ - --network-plugin azure \ - --generate-ssh-keys > /dev/null - - echo "Adding windows nodepool" - # add windows nodepool - az aks nodepool add \ - --resource-group "${CLUSTER_NAME}" \ - --cluster-name "${CLUSTER_NAME}" \ - --os-type Windows \ - --name npwin \ - --node-count 1 > /dev/null - - az aks get-credentials --resource-group "${CLUSTER_NAME}" --name "${CLUSTER_NAME}" --overwrite-existing - - if [[ "${REGISTRY}" =~ \.azurecr\.io ]]; then - az acr login --name "${REGISTRY}" - fi - - # build the driver image and run e2e tests - ALL_OS_ARCH=amd64 make e2e-test -} - -# for Prow we use the provided AZURE_CREDENTIALS file. -# the file is expected to be in toml format. -if [[ -n "${AZURE_CREDENTIALS:-}" ]]; then - subscription_id="$(parse_cred SubscriptionID < "${AZURE_CREDENTIALS}")" - tenant_id="$(parse_cred TenantID < "${AZURE_CREDENTIALS}")" - client_id="$(parse_cred ClientID < "${AZURE_CREDENTIALS}")" - client_secret="$(parse_cred ClientSecret < "${AZURE_CREDENTIALS}")" -fi - -main