Use Skaffold + hack/ scripts for dev/test loops

.dockerignore

@@ -4,11 +4,14 @@ Dockerfile
 .github/
 .gitlab-ci.yml
 .idea/
+.vscode/
 CHANGES.md
 LICENSE
 README.md
 e2etests/
 deploy/
+hack/
 mock/
 script/
 **/*_test.go
+skaffold.yaml

.github/workflows/test_e2e.yml

@@ -1,12 +1,23 @@
 name: Run e2e tests
-on: [push]
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        k8s: [ k8s-1.19.10, k8s-1.20.6, k8s-1.21.0, k8s-1.22.0 ]
-    name: k8s ${{ matrix.k8s }}
+        k8s:
+          - test_version: v1.19.14
+            major: 19
+          - test_version: v1.20.10
+            major: 20
+          - test_version: v1.21.4
+            major: 21
+    name: k8s v1.${{ matrix.k8s.major }}
     steps:
     - uses: actions/setup-go@v2
       with:
@@ -31,22 +42,59 @@ jobs:
       env:
         HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
       run: |
+        # If HCLOUD_TOKEN has multiple items (delimited with a comma) then grab a random one.
+        HCLOUD_TOKEN=$(echo $HCLOUD_TOKEN | tr ',' '\n' | shuf -n1)
         echo "HCLOUD_TOKEN=$HCLOUD_TOKEN" >> $GITHUB_ENV
+        echo "::add-mask::$HCLOUD_TOKEN"
     - name: Blow up if no HCLOUD_TOKEN or TTS_TOKEN set.
       if: steps.check_tts.outputs.hcloud_token_set == 'false' && steps.check_tts.outputs.tts_token_set == 'false'
       run: |
         echo "::error ::Couldn't determine HCLOUD_TOKEN. Check your repository secrets are setup correctly."
-    - uses: actions/cache@v2
+    - name: cache deps
+      uses: actions/cache@v2
       with:
         path: |
-          ~/go/pkg/mod
-          ~/.cache/go-build
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-    - name: Run tests
+          ~/bin/k3sup
+          ~/bin/skaffold
+        # Use this workflow file as cache key in case we change the versions we download below.
+        key: bin-deps-${{ hashFiles('.github/workflows/test_e2e.yml') }}
+    - name: install deps
+      run: |
+        mkdir -p ~/bin
+        [[ ! -f ~/bin/skaffold ]] && curl -Lo ~/bin/skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && chmod +x ~/bin/skaffold
+        [[ ! -f ~/bin/k3sup ]] && curl -Lo ~/bin/k3sup https://github.com/alexellis/k3sup/releases/download/0.11.0/k3sup && chmod +x ~/bin/k3sup
+        echo "$HOME/bin" >> $GITHUB_PATH
+    - name: Log in to the Container registry
+      uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+      if: ${{ github.repository_owner != 'hetznercloud' }}
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - uses: 3bit/setup-hcloud@v1
+    - if: ${{ github.repository_owner != 'hetznercloud' }}
+      env:
+        REPO_OWNER: ${{ github.repository_owner }}
+      run: |
+        echo "SKAFFOLD_DEFAULT_REPO=ghcr.io/$REPO_OWNER" >> $GITHUB_ENV
+    - name: run e2e tests
       env:
-        K8S_VERSION: ${{ matrix.k8s }}
+        K3S_CHANNEL: v1.${{ matrix.k8s.major }}
+        SCOPE: ci-${{ matrix.k8s.major }}-${{ github.run_id }}
+        SSH_KEYS: ${{ secrets.SSH_KEYS }}
+        KUBE_VERSION: ${{ matrix.k8s.test_version }}
       run: |
-        go test $(go list ./... | grep e2etests) -v -timeout 60m
-        ./script/delete-token.sh $HCLOUD_TOKEN
+        set -uex -o pipefail
+        eval $(INSTANCES=1 hack/dev-up.sh)
+        skaffold run
+        hack/run-e2e-tests.sh
+    - name: cleanup environment
+      if: always()
+      env:
+        SCOPE: ci-${{ matrix.k8s.major }}-${{ github.run_id }}
+      run: hack/dev-down.sh
+    - name: Cleanup TTS token
+      if: always() && steps.check_tts.outputs.tts_token_set == 'true'
+      env:
+        TTS_TOKEN: ${{ secrets.TTS_TOKEN }}
+      run: ./script/delete-token.sh $HCLOUD_TOKEN

Dockerfile

@@ -3,9 +3,14 @@ WORKDIR /csi
 ADD go.mod go.sum /csi/
 RUN go mod download
 ADD . /csi/
-RUN CGO_ENABLED=0 go build -o driver.bin github.com/hetznercloud/csi-driver/cmd/driver
+RUN ls -al
+# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG SKAFFOLD_GO_GCFLAGS
+
+RUN CGO_ENABLED=0 go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o driver.bin github.com/hetznercloud/csi-driver/cmd/driver
 
 FROM alpine:3.13
 RUN apk add --no-cache ca-certificates e2fsprogs xfsprogs blkid xfsprogs-extra e2fsprogs-extra btrfs-progs
+ENV GOTRACEBACK=all
 COPY --from=builder /csi/driver.bin /bin/hcloud-csi-driver
 ENTRYPOINT ["/bin/hcloud-csi-driver"]

README.md

@@ -104,26 +104,56 @@ tests for a specific version. You can run the tests with the following
 commands. Keep in mind, that these tests run on real cloud servers and
 will create volumes that will be billed.
 
-**Test Server Setup**:
+**Development/Testing**
 
-1x CPX21 (Ubuntu 18.04)
+For local development, you will need the following tools installed:
 
-**Requirements: Docker and Go 1.16**
+ * Docker
+ * Golang 1.16
+ * [Skaffold](https://skaffold.dev/)
+ * [k3sup](https://github.com/alexellis/k3sup#readme)
+ * [hcloud CLI](https://github.com/hetznercloud/cli#readme)
 
-1. Configure your environment correctly
-   ```bash
-   export HCLOUD_TOKEN=<specifiy a project token>
-   export K8S_VERSION=1.21.0 # The specific (latest) version is needed here
-   export USE_SSH_KEYS=key1,key2 # Name or IDs of your SSH Keys within the Hetzner Cloud, the servers will be accessable with that keys
-   ```
-2. Run the tests
-   ```bash
-   go test $(go list ./... | grep e2etests) -v -timeout 60m
-   ```
+You will also need to set a `HCLOUD_TOKEN` in your shell session:
+
+```sh
+ $ export HCLOUD_TOKEN=<token>
+```
+
+You can quickly bring up a dev cluster test environment in Hetzner Cloud.
+
+```sh
+  $ eval $(INSTANCES=3 hack/dev-up.sh)
+  # In about a minute, you should have a 3 node cluster of CPX11 instances (cost is around 2 cents per hour)
+  $ kubectl get nodes
+  # Now let's run the app.
+  $ SKAFFOLD_DEFAULT_REPO=my_dockerhub_username skaffold dev
+  # In a minute or two, the project should be built into an image, deployed into the test cluster.
+  # Logs will now be tailing out to your shell.
+  # If you make changes to the project, the image will be rebuilt and pushed to the cluster, restarting pods as needed.
+  # You can even debug the containers running remotely in The Cloud(tm) using standard Golang delve.
+  ^C
+  $ skaffold debug
+  # The logs will indicate which debug ports are available.
+  # IMPORTANT! The created servers are not automatically cleaned up. You must remember to delete everything yourself:
+  $ hack/dev-down.sh
+```
 
-The tests will now run, this will take a while (~30 min).
+### A note about `SKAFFOLD_DEFAULT_REPO`
 
-**If the tests fail, make sure to clean up the project with the Hetzner Cloud Console or the hcloud cli.**
+When you use Skaffold to deploy the driver to a remote cluster in Hetzner Cloud, you need somewhere to host the images. The default image repository is owned by Hetzner, and thus cannot be used for local development purposes. Instead, you can point Skaffold at your own Docker Hub, ghcr.io, Quay.io, or whatever. The Skaffold docks talk more about [Image Repository Handling](https://skaffold.dev/docs/environment/image-registries/) in gory detail, if you need more information.
+
+Please see the [Skaffold Documentation](https://skaffold.dev/docs/) for more information on the things you can do with Skaffold.
+
+### Running end-to-end tests
+
+Note, these tests will create and detach a *lot* of volumes. You will likely run into API request limits if you run this too frequently.
+The tests take 10-20 minutes.
+
+
+```sh
+hack/run-e2e-tests.sh
+```
 
 ## License
 

deploy/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - kubernetes/hcloud-csi.yml