-
Notifications
You must be signed in to change notification settings - Fork 267
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
optionally delete sqs message when node not found (#801)
* optionally delete sqs message when node not found * remove unused variables
- Loading branch information
Showing
7 changed files
with
194 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
166 changes: 166 additions & 0 deletions
166
test/e2e/rebalance-recommendation-sqs-node-not-found-test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| #!/bin/bash | ||
| set -euo pipefail | ||
|
|
||
| # Available env vars: | ||
| # $TMP_DIR | ||
| # $CLUSTER_NAME | ||
| # $KUBECONFIG | ||
| # $NODE_TERMINATION_HANDLER_DOCKER_REPO | ||
| # $NODE_TERMINATION_HANDLER_DOCKER_TAG | ||
| # $WEBHOOK_DOCKER_REPO | ||
| # $WEBHOOK_DOCKER_TAG | ||
| # $AEMM_URL | ||
| # $AEMM_VERSION | ||
|
|
||
| function fail_and_exit { | ||
| echo "❌ Rebalance Recommendation SQS Test failed $CLUSTER_NAME ❌" | ||
| exit "${1:-1}" | ||
| } | ||
|
|
||
| echo "Starting Rebalance Recommendation SQS Test for Node Termination Handler" | ||
| START_TIME=$(date -u +"%Y-%m-%dT%TZ") | ||
|
|
||
| SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" | ||
|
|
||
| common_helm_args=() | ||
|
|
||
| localstack_helm_args=( | ||
| upgrade | ||
| --install | ||
| --namespace default | ||
| "$CLUSTER_NAME-localstack" | ||
| "$SCRIPTPATH/../../config/helm/localstack/" | ||
| --set nodeSelector."${NTH_CONTROL_LABEL}" | ||
| --set defaultRegion="${AWS_REGION}" | ||
| --wait | ||
| ) | ||
|
|
||
| set -x | ||
| helm "${localstack_helm_args[@]}" | ||
| set +x | ||
|
|
||
| sleep 10 | ||
|
|
||
| localstack_pod=$(kubectl get pods --selector app=localstack --field-selector="status.phase=Running" \ | ||
| -o go-template --template '{{range .items}}{{.metadata.name}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' \ | ||
| | awk '$2 >= "'"${START_TIME//+0000/Z}"'" { print $1 }') | ||
| echo "🥑 Using localstack pod ${localstack_pod}" | ||
|
|
||
| run_instances_resp=$(kubectl exec -i "${localstack_pod}" -- bash -c "awslocal ec2 run-instances --private-ip-address 192.168.0.4 --region ${AWS_REGION}") | ||
| instance_id=$(echo "${run_instances_resp}" | jq -r '.Instances[] .InstanceId') | ||
| echo "🥑 Created awslocal EC2 instance (${instance_id}) not backed by a node" | ||
|
|
||
| CREATE_SQS_CMD="awslocal sqs create-queue --queue-name "${CLUSTER_NAME}-queue" --attributes MessageRetentionPeriod=300 --region ${AWS_REGION}" | ||
| queue_url=$(kubectl exec -i "${localstack_pod}" -- bash -c "${CREATE_SQS_CMD}" | jq -r .QueueUrl) | ||
|
|
||
| echo "🥑 Created SQS Queue ${queue_url}" | ||
|
|
||
| anth_helm_args=( | ||
| upgrade | ||
| --install | ||
| --namespace kube-system | ||
| "$CLUSTER_NAME-acth" | ||
| "$SCRIPTPATH/../../config/helm/aws-node-termination-handler/" | ||
| --set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" | ||
| --set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" | ||
| --set nodeSelector."${NTH_CONTROL_LABEL}" | ||
| --set tolerations[0].operator=Exists | ||
| --set awsAccessKeyID=foo | ||
| --set awsSecretAccessKey=bar | ||
| --set awsRegion="${AWS_REGION}" | ||
| --set awsEndpoint="http://localstack.default" | ||
| --set checkTagBeforeDraining=false | ||
| --set enableSqsTerminationDraining=true | ||
| --set deleteSqsMsgIfNodeNotFound=true | ||
| --set taintNode="true" | ||
| --set "queueURL=${queue_url}" | ||
| --wait | ||
| ) | ||
| [[ -n "${NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY-}" ]] && | ||
| anth_helm_args+=(--set image.pullPolicy="$NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY") | ||
| [[ ${#common_helm_args[@]} -gt 0 ]] && | ||
| anth_helm_args+=("${common_helm_args[@]}") | ||
|
|
||
| set -x | ||
| helm "${anth_helm_args[@]}" | ||
| set +x | ||
|
|
||
| emtp_helm_args=( | ||
| upgrade | ||
| --install | ||
| --namespace default | ||
| "$CLUSTER_NAME-emtp" | ||
| "$SCRIPTPATH/../../config/helm/webhook-test-proxy/" | ||
| --set webhookTestProxy.image.repository="$WEBHOOK_DOCKER_REPO" | ||
| --set webhookTestProxy.image.tag="$WEBHOOK_DOCKER_TAG" | ||
| --wait | ||
| ) | ||
| [[ -n "${WEBHOOK_DOCKER_PULL_POLICY-}" ]] && | ||
| emtp_helm_args+=(--set webhookTestProxy.image.pullPolicy="$WEBHOOK_DOCKER_PULL_POLICY") | ||
| [[ ${#common_helm_args[@]} -gt 0 ]] && | ||
| emtp_helm_args+=("${common_helm_args[@]}") | ||
|
|
||
| set -x | ||
| helm "${emtp_helm_args[@]}" | ||
| set +x | ||
|
|
||
| CHECK_CYCLES=15 | ||
| CHECK_SLEEP=15 | ||
|
|
||
| DEPLOYED=0 | ||
|
|
||
| for i in $(seq 1 $CHECK_CYCLES); do | ||
| if [[ $(kubectl get deployments regular-pod-test -o jsonpath='{.status.unavailableReplicas}') -eq 0 ]]; then | ||
| echo "✅ Verified regular-pod-test pod was scheduled and started!" | ||
| DEPLOYED=1 | ||
| break | ||
| fi | ||
| echo "Setup Loop $i/$CHECK_CYCLES, sleeping for $CHECK_SLEEP seconds" | ||
| sleep $CHECK_SLEEP | ||
| done | ||
|
|
||
| if [[ $DEPLOYED -eq 0 ]]; then | ||
| echo "❌ regular-pod-test pod deployment failed" | ||
| fail_and_exit 2 | ||
| fi | ||
|
|
||
| REBALANCE_EVENT=$(cat <<EOF | ||
| { | ||
| "version": "0", | ||
| "id": "5d5555d5-dd55-5555-5555-5555dd55d55d", | ||
| "detail-type": "EC2 Instance Rebalance Recommendation", | ||
| "source": "aws.ec2", | ||
| "account": "123456789012", | ||
| "time": "$(date -u +"%Y-%m-%dT%TZ")", | ||
| "region": "us-east-1", | ||
| "resources": [ | ||
| "arn:aws:ec2:us-east-1b:instance/${instance_id}" | ||
| ], | ||
| "detail": { | ||
| "instance-id": "${instance_id}" | ||
| } | ||
| } | ||
| EOF | ||
| ) | ||
|
|
||
| REBALANCE_EVENT_ONE_LINE=$(echo "${REBALANCE_EVENT}" | tr -d '\n' |sed 's/\"/\\"/g') | ||
| SEND_SQS_CMD="awslocal sqs send-message --queue-url ${queue_url} --message-body \"${REBALANCE_EVENT_ONE_LINE}\" --region ${AWS_REGION}" | ||
| kubectl exec -i "${localstack_pod}" -- bash -c "${SEND_SQS_CMD}" | ||
| echo "✅ Sent Rebalance Recommendation to SQS queue: ${queue_url}" | ||
|
|
||
| GET_ATTRS_SQS_CMD="awslocal sqs get-queue-attributes --queue-url ${queue_url} --attribute-names All --region ${AWS_REGION}" | ||
|
|
||
| for i in $(seq 1 $CHECK_CYCLES); do | ||
| if [[ $(kubectl exec -i "${localstack_pod}" -- bash -c "${GET_ATTRS_SQS_CMD}" | jq '(.Attributes.ApproximateNumberOfMessagesNotVisible|tonumber) + (.Attributes.ApproximateNumberOfMessages|tonumber)' ) -eq 0 ]]; then | ||
| kubectl exec -i "${localstack_pod}" -- bash -c "${GET_ATTRS_SQS_CMD}" | ||
| echo "✅ Verified the message was deleted from the queue after processing!" | ||
| echo "✅ Rebalance Recommendation SQS Test Passed $CLUSTER_NAME! ✅" | ||
| exit 0 | ||
| fi | ||
|
|
||
| echo "Assertion Loop $i/$CHECK_CYCLES, sleeping for $CHECK_SLEEP seconds" | ||
| sleep $CHECK_SLEEP | ||
| done | ||
|
|
||
| echo "❌ message was not removed from the queue after processing" | ||
| fail_and_exit 3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters