test: add e2e test with e2e-provider and add helm config

Signed-off-by: Anish Ramasekar <[email protected]>

Makefile

@@ -427,7 +427,9 @@ e2e-helm-deploy:
 		--set linux.enabled=true \
 		--set syncSecret.enabled=true \
 		--set enableSecretRotation=true \
-		--set rotationPollInterval=30s
+		--set rotationPollInterval=30s \
+		--set tokenRequests[0].audience="aud1" \
+		--set tokenRequests[1].audience="aud2"
 
 .PHONY: e2e-helm-upgrade
 e2e-helm-upgrade:

manifest_staging/charts/secrets-store-csi-driver/templates/csidriver.yaml

@@ -8,3 +8,7 @@ spec:
   # Added in Kubernetes 1.16 with default mode of Persistent. Secrets store csi driver needs Ephermeral to be set.
   volumeLifecycleModes: 
   - Ephemeral
+  {{- if .Values.tokenRequests }}
+  tokenRequests:
+    {{- toYaml .Values.tokenRequests | nindent 2}}
+  {{- end }}

manifest_staging/charts/secrets-store-csi-driver/values.yaml

@@ -207,3 +207,9 @@ providerHealthCheck: false
 providerHealthCheckInterval: 2m
 
 imagePullSecrets: []
+
+## This allows CSI drivers to impersonate the pods that they mount the volumes for.
+# refer to https://kubernetes-csi.github.io/docs/token-requests.html for more details.
+tokenRequests: []
+# - audience: aud1
+# - audience: aud2

test/bats/e2e-provider.bats

@@ -34,13 +34,33 @@ export LABEL_VALUE=${LABEL_VALUE:-"test"}
 # export the secrets-store API version to be used
 export API_VERSION=$(get_secrets_store_api_version)
 
+# export the token requests audience configured in the CSIDriver
+# refer to https://kubernetes-csi.github.io/docs/token-requests.html for more information
+export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
+
+@test "setup mock provider validation config" {
+  if [[ -n "${VALIDATE_TOKENS_AUDIENCE}" ]]; then
+    # configure the mock provider to validate the token requests
+    kubectl create ns enable-token-requests
+    local curl_pod_name=curl-$(openssl rand -hex 5)
+    kubectl run ${curl_pod_name} -n enable-token-requests --image=curlimages/curl:7.75.0 --labels="util=enable-token-requests" -- tail -f /dev/null
+    kubectl wait -n enable-token-requests --for=condition=Ready --timeout=60s pod ${curl_pod_name}
+    local pod_ip=$(kubectl get pod -n kube-system -l app=csi-secrets-store-e2e-provider -o jsonpath="{.items[0].status.podIP}")
+    run kubectl exec ${curl_pod_name} -n enable-token-requests -- curl http://${pod_ip}:8080/validate-token-requests?audience=${VALIDATE_TOKENS_AUDIENCE}
+    kubectl delete pod -l util=enable-token-requests -n enable-token-requests --force --grace-period 0
+    kubectl delete ns enable-token-requests
+  fi
+
+  log_secrets_store_api_version
+  log_token_requests_audience
+}
+
+
 @test "secretproviderclasses crd is established" {
   kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
 
   run kubectl get crd/secretproviderclasses.secrets-store.csi.x-k8s.io
   assert_success
-
-  log_secrets_store_api_version
 }
 
 @test "secretproviderclasspodstatuses crd is established" {

test/bats/helpers.bash

@@ -137,3 +137,12 @@ get_secrets_store_api_version() {
 log_secrets_store_api_version() {
   echo "Testing secrets-store API version $API_VERSION" >&3
 }
+
+get_token_requests_audience() {
+  local token_requests_audience=$(kubectl get csidriver secrets-store.csi.k8s.io -o go-template --template='{{range $i, $v := .spec.tokenRequests}}{{if $i}},{{end}}{{printf "%s" .audience}}{{end}}') 
+  echo "${token_requests_audience}"
+}
+
+log_token_requests_audience() {
+  echo "Testing token requests audience $VALIDATE_TOKENS_AUDIENCE" >&3
+}

test/e2eprovider/e2e_provider.go

@@ -69,6 +69,7 @@ func main() {
 		os.Setenv("ROTATION_ENABLED", "false")
 
 		http.HandleFunc("/rotation", server.RotationHandler)
+		http.HandleFunc("/validate-token-requests", server.ValidateTokenAudienceHandler)
 		klog.Fatal(http.ListenAndServe(":8080", nil))
 	}()
 

test/e2eprovider/server/server.go

@@ -48,7 +48,8 @@ This is mock key
 	// podCache is a map of pod UID to check if secret has been rotated.
 	podCache = map[string]bool{}
 
-	podUIDAttribute = "csi.storage.k8s.io/pod.uid"
+	podUIDAttribute               = "csi.storage.k8s.io/pod.uid"
+	serviceAccountTokensAttribute = "csi.storage.k8s.io/serviceAccount.tokens" //nolint
 
 	// RWMutex is to safely access podCache
 	m sync.RWMutex
@@ -168,6 +169,23 @@ func (s *Server) Mount(ctx context.Context, req *v1alpha1.MountRequest) (*v1alph
 		resp.ObjectVersion = append(resp.ObjectVersion, version)
 	}
 
+	// if validate token flag is set, we want to check the service account tokens as passed
+	// as part of the mount attributes.
+	// In case of 1.21+, kubelet will generate the token and pass it as part of the volume context.
+	// The driver will pass this to the provider as part of the mount request.
+	// For 1.20, the driver will generate the token and pass it to the provider as part of the mount request.
+	// Irrespective of the kubernetes version, the rotation handler in the driver will generate the token
+	// and pass it to the provider as part of the mount request.
+	// VALIDATE_TOKENS_AUDIENCE environment variable will be a comma separated list of audiences configured in the csidriver object
+	// If this env var is not set, this could mean we are running an older version of driver.
+	tokenAudiences := os.Getenv("VALIDATE_TOKENS_AUDIENCE")
+	klog.InfoS("tokenAudiences", "tokenAudiences", tokenAudiences)
+	if tokenAudiences != "" {
+		if err := validateTokens(tokenAudiences, attrib[serviceAccountTokensAttribute]); err != nil {
+			return nil, fmt.Errorf("failed to validate token, error: %w", err)
+		}
+	}
+
 	m.Lock()
 	podCache[attrib[podUIDAttribute]] = true
 	m.Unlock()
@@ -219,3 +237,32 @@ func RotationHandler(w http.ResponseWriter, r *http.Request) {
 	os.Setenv("ROTATION_ENABLED", r.FormValue("rotated"))
 	klog.InfoS("Rotation response enabled")
 }
+
+// ValidateTokenAudienceHandler enables token validation for the mock provider
+// This is only required because older version of the driver don't generate a token
+// TODO(aramase): remove this after the supported driver releases are v1.1.0+
+func ValidateTokenAudienceHandler(w http.ResponseWriter, r *http.Request) {
+	// enable rotation response
+	os.Setenv("VALIDATE_TOKENS_AUDIENCE", r.FormValue("audience"))
+	klog.InfoS("Validation for token requests audience", "audience", os.Getenv("VALIDATE_TOKENS_AUDIENCE"))
+}
+
+// validateTokens checks there are tokens for distinct audiences in the
+// service account token attribute.
+func validateTokens(tokenAudiences, saTokens string) error {
+	ta := strings.Split(strings.TrimSpace(tokenAudiences), ",")
+	if saTokens == "" {
+		return fmt.Errorf("service account tokens is not set")
+	}
+	tokens := make(map[string]interface{})
+	if err := json.Unmarshal([]byte(saTokens), &tokens); err != nil {
+		return fmt.Errorf("failed to unmarshal service account tokens, error: %w", err)
+	}
+	for _, a := range ta {
+		if _, ok := tokens[a]; !ok {
+			return fmt.Errorf("service account token for audience %s is not set", a)
+		}
+		klog.InfoS("Validated service account token", "audience", a)
+	}
+	return nil
+}

test/e2eprovider/server/server_test.go

@@ -213,3 +213,33 @@ func TestRotation(t *testing.T) {
 		t.Errorf("didn't get expected results: (-want +got):\n%s", diff)
 	}
 }
+
+func TestValidateTokens(t *testing.T) {
+	tests := []struct {
+		name      string
+		tokens    string
+		audiences string
+		wantErr   bool
+	}{
+		{
+			name:      "no tokens",
+			tokens:    "",
+			audiences: "aud1,aud2",
+			wantErr:   true,
+		},
+		{
+			name:      "matching tokens for audience",
+			tokens:    `{"aud1":{"token":"eyJhbGciOiJSUzI1NiIsImtpZCI6InRhVDBxbzhQVEZ1ajB1S3BYUUxIclRsR01XakxjemJNOTlzWVMxSlNwbWcifQ.eyJhdWQiOlsiYXBpOi8vQXp1cmVBRGlUb2tlbkV4Y2hhbmdlIl0sImV4cCI6MTY0MzIzNDY0NywiaWF0IjoxNjQzMjMxMDQ3LCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbCIsImt1YmVybmV0ZXMuaW8iOnsibmFtZXNwYWNlIjoidGVzdC12MWFscGhhMSIsInBvZCI6eyJuYW1lIjoic2VjcmV0cy1zdG9yZS1pbmxpbmUtY3JkIiwidWlkIjoiYjBlYmZjMzUtZjEyNC00ZTEyLWI3N2UtYjM0MjM2N2IyMDNmIn0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkZWZhdWx0IiwidWlkIjoiMjViNGY1NzgtM2U4MC00NTczLWJlOGQtZTdmNDA5ZDI0MmI2In19LCJuYmYiOjE2NDMyMzEwNDcsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDp0ZXN0LXYxYWxwaGExOmRlZmF1bHQifQ.ALE46aKmtTV7dsuFOwDZqvEjdHFUTNP-JVjMxexTemmPA78fmPTUZF0P6zANumA03fjX3L-MZNR3PxmEZgKA9qEGIDsljLsUWsVBEquowuBh8yoBYkGkMJmRfmbfS3y7_4Q7AU3D9Drw4iAHcn1GwedjOQC0i589y3dkNNqf8saqHfXkbSSLtSE0f2uzI-PjuTKvR1kuojEVNKlEcA4wsKfoiRpkua17sHkHU0q9zxCMDCr_1f8xbigRnRx0wscU3vy-8KhF3zQtpcWkk3r4C5YSXut9F3xjz5J9DUQn2vNMfZg4tOdcR-9Xv9fbY5iujiSlS58GEktSEa3SE9wrCw\",\"expirationTimestamp\":\"2022-01-26T22:04:07Z\"},\"gcp\":{\"token\":\"eyJhbGciOiJSUzI1NiIsImtpZCI6InRhVDBxbzhQVEZ1ajB1S3BYUUxIclRsR01XakxjemJNOTlzWVMxSlNwbWcifQ.eyJhdWQiOlsiZ2NwIl0sImV4cCI6MTY0MzIzNDY0NywiaWF0IjoxNjQzMjMxMDQ3LCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbCIsImt1YmVybmV0ZXMuaW8iOnsibmFtZXNwYWNlIjoidGVzdC12MWFscGhhMSIsInBvZCI6eyJuYW1lIjoic2VjcmV0cy1zdG9yZS1pbmxpbmUtY3JkIiwidWlkIjoiYjBlYmZjMzUtZjEyNC00ZTEyLWI3N2UtYjM0MjM2N2IyMDNmIn0sInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkZWZhdWx0IiwidWlkIjoiMjViNGY1NzgtM2U4MC00NTczLWJlOGQtZTdmNDA5ZDI0MmI2In19LCJuYmYiOjE2NDMyMzEwNDcsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDp0ZXN0LXYxYWxwaGExOmRlZmF1bHQifQ.BT0YGI7bGdSNaIBqIEnVL0Ky5t-fynaemSGxjGdKOPl0E22UIVGDpAMUhaS19i20c-Dqs-Kn0N-R5QyDNpZg8vOL5KIFqu2kSYNbKxtQW7TPYIsV0d9wUZjLSr54DKrmyXNMGRoT2bwcF4yyfmO46eMmZSaXN8Y4lgapeabg6CBVVQYHD-GrgXf9jVLeJfCQkTuojK1iXOphyD6NqlGtVCaY1jWxbBMibN0q214vKvQboub8YMuvclGdzn_l_ZQSTjvhBj9I-W1t-JArVjqHoIb8_FlR9BSgzgL7V3Jki55vmiOdEYqMErJWrIZPP3s8qkU5hhO9rSVEd3LJHponvQ","expirationTimestamp":"2022-01-26T22:04:07Z"}}`,
+			audiences: "aud1",
+			wantErr:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := validateTokens(tt.audiences, tt.tokens); (err != nil) != tt.wantErr {
+				t.Errorf("validateTokens() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}