feat: Token encryption #401
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'K8S: Stream Service' | |
| on: | |
| # This workflow is also part of the release pipeline, | |
| # in that case, the actual version is deployed twice. | |
| workflow_call: | |
| # In PR the previous version is deployed first and then the actual version. | |
| pull_request: | |
| paths: | |
| - .github/workflows/test-k8s-service-stream.yml | |
| - provisioning/stream/** | |
| - provisioning/common/** | |
| # When the config structure is changed it may be necessary to adjust k8s configmap. | |
| - internal/pkg/service/stream/config/** | |
| env: | |
| MINIKUBE_PROFILE: stream | |
| MINIKUBE_DRIVER: docker | |
| KUBERNETES_NAMESPACE: stream | |
| KUBERNETES_ROLLOUT_WAIT: 200s | |
| REMOVE_RESOURCES_LIMITS: true | |
| SERVICE_NAME: Stream | |
| ETCD_RELEASE_NAME: stream-etcd | |
| ETCD_ENDPOINT: stream-etcd-headless.stream.svc.cluster.local:2379 | |
| METRICS_PORT: 9000 | |
| defaults: | |
| run: | |
| working-directory: provisioning/stream | |
| jobs: | |
| test: | |
| name: "K8S test: Stream Service" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: false | |
| docker-images: false | |
| swap-storage: false | |
| - name: Create artifacts directory | |
| run: mkdir -p /tmp/artifacts | |
| - name: Copy latest scripts to a temp dir | |
| run: cp -R ${{ github.workspace }}/provisioning/common/scripts /tmp/latest-scripts | |
| - name: Install gron tool | |
| run: | | |
| url="https://github.com/tomnomnom/gron/releases/download/v0.7.1/gron-linux-amd64-0.7.1.tgz" | |
| curl -L "$url" | tar -xz -C /usr/local/bin | |
| - name: Install MiniKube | |
| run: /tmp/latest-scripts/minikube/install.sh | |
| - name: Start MiniKube | |
| run: /tmp/latest-scripts/minikube/start.sh | |
| - name: Set Kubernetes namespace | |
| run: kubectl config set-context --current "--namespace=$KUBERNETES_NAMESPACE" | |
| - name: Checkout BASE branch (or HEAD if it is not a pull request) | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref }} | |
| - name: Deploy the old version, from the BASE branch | |
| continue-on-error: true | |
| run: ./deploy_local.sh | |
| - name: Dump the old version (for diff) | |
| continue-on-error: true | |
| run: sleep 10 && /tmp/latest-scripts/k8s/dump.sh /tmp/artifacts/test-k8s-state.old.json | |
| - name: Checkout HEAD branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| - name: Deploy the new version, from the HEAD branch | |
| run: ./deploy_local.sh | |
| - name: Dump the new version (for diff) | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| # Delete empty old replica sets | |
| emptyReplicaSets=$(kubectl get replicasets --ignore-not-found | tail --lines=+2 | awk '{if ($2 + $3 + $4 == 0) print $1}') | |
| echo -e "Found empty replica sets:" | |
| echo "$emptyReplicaSets" | |
| echo "-------------------------" | |
| echo "$emptyReplicaSets" | xargs --no-run-if-empty -I {} kubectl delete replicaset "{}" | |
| # Wait for pod startup/termination | |
| sleep 10 | |
| # Dump objects | |
| /tmp/latest-scripts/k8s/dump.sh /tmp/artifacts/test-k8s-state.new.json | |
| - name: Check deployment of the etcd | |
| if: always() | |
| run: kubectl rollout status "sts/$ETCD_RELEASE_NAME" --timeout=10s | |
| - name: Check deployment of the API nodes | |
| if: always() | |
| run: kubectl rollout status "deployment/stream-api" --timeout=10s | |
| - name: Check deployment of the HTTP source nodes | |
| if: always() | |
| run: kubectl rollout status "deployment/stream-http-source" --timeout=10s | |
| - name: Check deployment of the storage writer/reader nodes | |
| if: always() | |
| run: kubectl rollout status "sts/stream-storage-writer-reader" --timeout=10s | |
| - name: Check deployment of the storage coordinator nodes | |
| if: always() | |
| run: kubectl rollout status "deployment/stream-storage-coordinator" --timeout=10s | |
| - name: Check access to the metrics from the DataDog Agent | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| kubectl create namespace datadog || true | |
| for APP_WITH_METRICS_PORT in "stream-api:9000" "stream-http-source:9000" "stream-storage-writer-reader:9001" "stream-storage-writer-reader:9002" "stream-storage-coordinator:9000" | |
| do | |
| IFS=: read -r APP METRICS_PORT <<< "$APP_WITH_METRICS_PORT" | |
| echo "---------------------------------" | |
| echo "Checking $APP:$METRICS_PORT" | |
| export POD_IP=`kubectl get pod -l "app=$APP" -o=jsonpath='{.items[0].status.podIP}'` | |
| echo "Pod IP: $POD_IP" | |
| kubectl run --attach --rm --restart=Never check-api-datadog \ | |
| --namespace datadog \ | |
| --image docker.io/alpine/curl \ | |
| --labels="app=datadog-agent" \ | |
| --env="POD_IP=$POD_IP" \ | |
| --env="METRICS_PORT=$METRICS_PORT" \ | |
| --command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail" | |
| echo "---------------------------------" | |
| done | |
| - name: Check forbidden access to the metrics from other places | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| for APP_WITH_METRICS_PORT in "stream-api:9000" "stream-http-source:9000" "stream-storage-writer-reader:9001" "stream-storage-writer-reader:9002" "stream-storage-coordinator:9000" | |
| do | |
| IFS=: read -r APP METRICS_PORT <<< "$APP_WITH_METRICS_PORT" | |
| echo "---------------------------------" | |
| echo "Checking $APP:$METRICS_PORT" | |
| export POD_IP=`kubectl get pod -l "app=$APP" -o=jsonpath='{.items[0].status.podIP}'` | |
| echo "Pod IP: $POD_IP" | |
| if kubectl run --attach --rm --restart=Never check-api-other \ | |
| --image docker.io/alpine/curl \ | |
| --env="POD_IP=$POD_IP" \ | |
| --env="METRICS_PORT=$METRICS_PORT" \ | |
| --command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$POD_IP:$METRICS_PORT/metrics" | tail"; then | |
| echo "The command did not fail, but it should have." | |
| exit 1 | |
| else | |
| echo "The command failed, OK." | |
| exit 0 | |
| fi | |
| echo "---------------------------------" | |
| done | |
| - name: Check access to the etcd from a client | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| export ETCD_ROOT_PASSWORD=$(kubectl get secret "$ETCD_RELEASE_NAME" -o jsonpath="{.data.etcd-root-password}" 2>/dev/null | base64 -d) | |
| kubectl run --attach --rm --restart=Never check-etcd-client \ | |
| --image docker.io/bitnami/etcd \ | |
| --labels="${ETCD_RELEASE_NAME}-client=true" \ | |
| --env="ETCD_ROOT_PASSWORD=$ETCD_ROOT_PASSWORD" \ | |
| --env="ETCDCTL_ENDPOINTS=$ETCD_ENDPOINT" \ | |
| --command -- etcdctl --dial-timeout=10s --user root:$ETCD_ROOT_PASSWORD put /message Hello | |
| - name: Check access to the etcd from the DataDog Agent | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| kubectl create namespace datadog || true | |
| kubectl run --attach --rm --restart=Never check-etcd-other-datadog \ | |
| --namespace datadog \ | |
| --image docker.io/alpine/curl \ | |
| --labels="app=datadog-agent" \ | |
| --env="ETCD_ENDPOINT=$ETCD_ENDPOINT" \ | |
| --command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$ETCD_ENDPOINT/metrics" | tail" | |
| - name: Check forbidden access to the etcd from other places | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| if kubectl run --attach --rm --restart=Never check-etcd-other \ | |
| --image docker.io/alpine/curl \ | |
| --env="ETCD_ENDPOINT=$ETCD_ENDPOINT" \ | |
| --command -- sh -c "set -eo pipefail; curl -f -L --max-time 5 "$ETCD_ENDPOINT/metrics" | tail"; then | |
| echo "The command did not fail, but it should have." | |
| exit 1 | |
| else | |
| echo "The command failed, OK." | |
| exit 0 | |
| fi | |
| - name: Check API response | |
| if: always() | |
| run: | | |
| hostPort=$(minikube service --url stream-api --namespace $KUBERNETES_NAMESPACE) | |
| echo "$hostPort" | |
| curl --fail -s --max-time 5 --retry 5 --retry-connrefused "$hostPort/health-check" | |
| - name: Check HTTP source response | |
| if: always() | |
| run: | | |
| hostPort=$(minikube service --url stream-http-source --namespace $KUBERNETES_NAMESPACE) | |
| curl --fail -s --max-time 5 --retry 5 --retry-connrefused "$hostPort/health-check" | |
| - name: Test etcd defragmentation cron job | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| # Create a job from the cron job | |
| kubectl create job "--from=cronjob/${ETCD_RELEASE_NAME}-defrag" test-defrag-job | |
| # Wait for the job | |
| if kubectl wait --for=condition=complete --timeout=30s job/test-defrag-job; then | |
| echo "The job succeed, logs:" | |
| kubectl logs --selector="job-name=test-defrag-job" | |
| exit 0 | |
| else | |
| echo "The job failed, logs:" | |
| kubectl logs --selector="job-name=test-defrag-job" | |
| exit 1 | |
| fi | |
| - name: Diff the old and the new Kubernetes state | |
| if: always() | |
| run: | | |
| set -Eeuo pipefail | |
| # Diff JSON states | |
| /tmp/latest-scripts/k8s/diff.sh \ | |
| /tmp/artifacts/test-k8s-state.old.json \ | |
| /tmp/artifacts/test-k8s-state.new.json \ | |
| /tmp/artifacts/test-k8s-state.diff | |
| # Remove ANSI sequences | |
| sed -e 's/\x1b\[[0-9;]*m//g' -i /tmp/artifacts/test-k8s-state.diff || true | |
| # Prepare PR comment message | |
| echo -e "### ${{ env.SERVICE_NAME }} Kubernetes Diff [CI]\n\n" >> /tmp/artifacts/test-k8s-state.diff.message | |
| echo -e "Between \`base\` ${{ github.event.pull_request.base.sha }} :arrow_left: \`head\` ${{ github.event.pull_request.head.sha }}.\n\n" >> /tmp/artifacts/test-k8s-state.diff.message | |
| echo -e "<details>\n<summary>Expand</summary>\n\n\`\`\`diff\n" >> /tmp/artifacts/test-k8s-state.diff.message | |
| head -c 50000 /tmp/artifacts/test-k8s-state.diff >> /tmp/artifacts/test-k8s-state.diff.message | |
| echo -e "\n\n(see artifacts in the Github Action for more information)\n\`\`\`\n</details>" >> /tmp/artifacts/test-k8s-state.diff.message | |
| - name: Dump logs | |
| if: always() | |
| run: | | |
| /tmp/latest-scripts/minikube/logs.sh /tmp/artifacts && | |
| /tmp/latest-scripts/k8s/logs.sh /tmp/artifacts | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-k8s-state-stream | |
| path: /tmp/artifacts | |
| if-no-files-found: error | |
| - name: Send PR comment | |
| uses: marocchino/sticky-pull-request-comment@v2 | |
| with: | |
| header: "${{ env.KUBERNETES_NAMESPACE }}-kubernetes-state-diff" | |
| recreate: true | |
| path: /tmp/artifacts/test-k8s-state.diff.message |