Skip to content

Commit

Permalink
validate metrics as part of e2e
Browse files Browse the repository at this point in the history 
Signed-off-by: Anish Ramasekar <[email protected]>
  • Loading branch information
@aramase
aramase committed May 18, 2023
1 parent 46ce725 commit 4e853c9
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 17 deletions.
33 changes: 24 additions & 9 deletions test/bats/e2e-provider.bats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export SECRET_VALUE=${SECRET_VALUE:-"secret"}
export KEY_NAME=${KEY_NAME:-fookey}
# defualt version value returned by mock provider
export KEY_VERSION=${KEY_VERSION:-"v1"}
# default key value returned by mock provider.
# default key value returned by mock provider.
# base64 encoded content comparision is easier in case of very long multiline string.
export KEY_VALUE_CONTAINS=${KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KVGhpcyBpcyBtb2NrIGtleQotLS0tLUVORCBQVUJMSUMgS0VZLS0tLS0K"}

Expand Down Expand Up @@ -118,7 +118,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)

@test "[v1alpha1] CSI inline volume test with pod portability" {
envsubst < $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml | kubectl apply -n test-v1alpha1 -f -

kubectl wait --for=condition=Ready -n test-v1alpha1 --timeout=180s pod/secrets-store-inline-crd

run kubectl get pod/secrets-store-inline-crd -n test-v1alpha1
Expand Down Expand Up @@ -149,7 +149,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)

@test "CSI inline volume test with pod portability" {
envsubst < $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml | kubectl apply -f -

kubectl wait --for=condition=Ready --timeout=180s pod/secrets-store-inline-crd

run kubectl get pod/secrets-store-inline-crd
Expand Down Expand Up @@ -192,7 +192,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
}

@test "Sync with K8s secrets - create deployment" {
envsubst < $BATS_TESTS_DIR/e2e_provider_synck8s_v1_secretproviderclass.yaml | kubectl apply -f -
envsubst < $BATS_TESTS_DIR/e2e_provider_synck8s_v1_secretproviderclass.yaml | kubectl apply -f -

kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io

Expand Down Expand Up @@ -231,7 +231,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
assert_success
}

@test "Sync with K8s secrets - delete deployment, check owner ref updated, check secret deleted" {
@test "Sync with K8s secrets - delete deployment, check owner ref updated, check secret deleted" {
if [[ "${INPLACE_UPGRADE_TEST}" == "true" ]]; then
skip
fi
Expand Down Expand Up @@ -316,7 +316,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
if [[ "${INPLACE_UPGRADE_TEST}" != "true" ]]; then
run kubectl delete ns negative-test-ns
assert_success
fi
fi
}

@test "deploy multiple e2e provier secretproviderclass crd" {
Expand All @@ -333,7 +333,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)

@test "deploy pod with multiple secret provider class" {
envsubst < $BATS_TESTS_DIR/pod-e2e-provider-inline-volume-multiple-spc.yaml | kubectl apply -f -

kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-inline-multiple-crd

run kubectl get pod/secrets-store-inline-multiple-crd
Expand Down Expand Up @@ -392,7 +392,7 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)

result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
[[ "${result//$'\r'}" == "secret" ]]

# enable rotation response in mock server
local curl_pod_name=curl-$(openssl rand -hex 5)
kubectl run ${curl_pod_name} -n rotation --image=curlimages/curl:7.75.0 --labels="test=rotation" -- tail -f /dev/null
Expand All @@ -414,13 +414,28 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience)
fi
}

@test "Validate metrics" {
kubectl create ns metrics
local curl_pod_name=curl-$(openssl rand -hex 5)
kubectl run ${curl_pod_name} -n metrics --image=curlimages/curl:7.75.0 --labels="test=metrics" -- tail -f /dev/null
kubectl wait -n metrics --for=condition=Ready --timeout=60s pod ${curl_pod_name}
for pod_ip in $(kubectl get pod -n kube-system -l app=secrets-store-csi-driver -o jsonpath="{.items[0].status.podIP}")
do
run kubectl exec ${curl_pod_name} -n metrics -- curl http://${pod_ip}:8095/metrics
assert_match "node_publish_total" "${output}"
assert_match "node_unpublish_total" "${output}"
assert_match "rotation_reconcile_total" "${output}"
done
}

teardown_file() {
if [[ "${INPLACE_UPGRADE_TEST}" != "true" ]]; then
#cleanup
run kubectl delete namespace rotation
run kubectl delete namespace test-ns
run kubectl delete namespace test-v1alpha1
run kubectl delete namespace metrics

run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0
fi
fi
}
8 changes: 4 additions & 4 deletions test/bats/gcp.bats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@ export RESOURCE_NAME=${RESOURCE_NAME:-"projects/735463103342/secrets/test-secret
export FILE_NAME=${FILE_NAME:-"secret"}
export SECRET_VALUE=${SECRET_VALUE:-"aHVudGVyMg=="}

@test "install gcp provider" {
@test "install gcp provider" {
run kubectl apply -f $PROVIDER_YAML --namespace $PROVIDER_NAMESPACE
assert_success
assert_success

kubectl wait --for=condition=Ready --timeout=120s pod -l app=csi-secrets-store-provider-gcp --namespace $PROVIDER_NAMESPACE

GCP_PROVIDER_POD=$(kubectl get pod --namespace $PROVIDER_NAMESPACE -l app=csi-secrets-store-provider-gcp -o jsonpath="{.items[0].metadata.name}")
GCP_PROVIDER_POD=$(kubectl get pod --namespace $PROVIDER_NAMESPACE -l app=csi-secrets-store-provider-gcp -o jsonpath="{.items[0].metadata.name}")

run kubectl get pod/$GCP_PROVIDER_POD --namespace $PROVIDER_NAMESPACE
assert_success
Expand All @@ -35,7 +35,7 @@ export SECRET_VALUE=${SECRET_VALUE:-"aHVudGVyMg=="}

@test "CSI inline volume test with pod portability" {
envsubst < $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml | kubectl apply --namespace=$NAMESPACE -f -

kubectl wait --for=condition=Ready --timeout=60s --namespace=$NAMESPACE pod/secrets-store-inline-crd

run kubectl get pod/secrets-store-inline-crd --namespace=$NAMESPACE
Expand Down
8 changes: 4 additions & 4 deletions test/bats/helpers.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,15 @@ archive_info() {

# collect metrics
local curl_pod_name=curl-$(openssl rand -hex 5)
kubectl run ${curl_pod_name} --image=curlimages/curl:7.75.0 --labels="test=metrics_test" -- tail -f /dev/null
kubectl wait --for=condition=Ready --timeout=60s pod ${curl_pod_name}
kubectl run ${curl_pod_name} -n default --image=curlimages/curl:7.75.0 --labels="test=metrics_test" --overrides='{"spec": { "nodeSelector": {"kubernetes.io/os": "linux"}}}' -- tail -f /dev/null
kubectl wait --for=condition=Ready --timeout=60s -n default pod ${curl_pod_name}

for pod_ip in $(kubectl get pod -n kube-system -l app=secrets-store-csi-driver -o jsonpath="{.items[*].status.podIP}")
do
kubectl exec ${curl_pod_name} -- curl -s http://${pod_ip}:8095/metrics > ${LOGS_DIR}/${pod_ip}.metrics
kubectl exec -n default ${curl_pod_name} -- curl -s http://${pod_ip}:8095/metrics > ${LOGS_DIR}/${pod_ip}.metrics
done

kubectl delete pod ${curl_pod_name}
kubectl delete pod -n default ${curl_pod_name}
}

# get_secrets_store_api_version returns the API version of the secrets-store API
Expand Down

0 comments on commit 4e853c9

Please sign in to comment.