ci: add initial integration tests

.github/workflows/.reusable-integration-test.yml

@@ -29,13 +29,210 @@ env:
   IMAGEPULLSECRET: dockerconfigjson-ghcr
 
 jobs:
-  do-nothing:
+  integration-test:
     name: functional
     runs-on: ubuntu-latest
     if: inputs.skip != 'all'
     #    permissions:  #TODO: reactivate for non-private
     #      packages: read
+    env:
+      IMAGE: ${{ inputs.build_image_repository }}
+      TAG: ${{ inputs.build_tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        integration-test-arg:
+          [
+            "basic",
+          ]
     steps:
-      - name: Do nothing
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Login with registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ inputs.build_registry }}
+          username: ${{ inputs.repo_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install yq
         run: |
-          sleep 1
+          sudo snap install yq
+      - uses: ./.github/actions/k8s-version-config
+        name: Setup k8s cluster
+        with:
+          k8s-version: v1.25
+      - name: Run test
+        run: |
+          bash tests/integration/main.sh "${{ matrix.integration-test-arg }}"
+      - name: Display semgr8s configuration
+        if: always()
+        run: |
+          echo "::group::values.yaml"
+          yq e '... comments=""' charts/semgr8s/values.yaml
+          echo "::endgroup::"
+      - name: Display k8s state if integration test failed
+        if: failure()
+        run: |
+          kubectl describe deployments.apps -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+          kubectl describe pods -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+      - name: Display logs if integration test failed
+        if: failure()
+        run: |
+          kubectl logs -n semgr8ns -lapp.kubernetes.io/name=semgr8s --prefix=true --tail=-1
+
+  optional-integration-test:
+    name: optional
+    runs-on: ubuntu-latest
+    if: |
+      inputs.skip != 'non-required' &&
+      inputs.skip != 'all'
+    #    permissions:  #TODO: reactivate for non-private
+    #      packages: read
+    env:
+      IMAGE: ${{ inputs.build_image_repository }}
+      TAG: ${{ inputs.build_tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        integration-test-arg:
+          [
+            "rules",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Login with registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ inputs.build_registry }}
+          username: ${{ inputs.repo_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install yq
+        run: |
+          sudo snap install yq
+      - uses: ./.github/actions/k8s-version-config
+        name: Setup k8s cluster
+        with:
+          k8s-version: v1.25
+      - name: Run test
+        run: |
+          bash tests/integration/main.sh "${{ matrix.integration-test-arg }}"
+      - name: Display semgr8s configuration
+        if: always()
+        run: |
+          echo "::group::values.yaml"
+          yq e '... comments=""' charts/semgr8s/values.yaml
+          echo "::endgroup::"
+      - name: Display k8s state if integration test failed
+        if: failure()
+        run: |
+          kubectl describe deployments.apps -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+          kubectl describe pods -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+      - name: Display logs if integration test failed
+        if: failure()
+        run: |
+          kubectl logs -n semgr8ns -lapp.kubernetes.io/name=semgr8s --prefix=true --tail=-1
+
+  k8s-versions:
+    name: k8s versions
+    runs-on: ubuntu-latest
+    if: inputs.skip != 'all'
+    #    permissions:  #TODO: reactivate for non-private
+    #      packages: read
+    env:
+      IMAGE: ${{ inputs.build_image_repository }}
+      TAG: ${{ inputs.build_tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: [
+            "v1.25",
+            "v1.26",
+            "v1.27",
+            "v1.28",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Login with registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ inputs.build_registry }}
+          username: ${{ inputs.repo_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install yq
+        run: |
+          sudo snap install yq
+      - uses: ./.github/actions/k8s-version-config
+        name: Setup k8s cluster
+        with:
+          k8s-version: ${{ matrix.k8s-version }}
+      - name: Run pre-config and workload integration tests
+        run: |
+          bash tests/integration/main.sh "basic"
+      - name: Display k8s state and logs if integration test failed
+        if: failure()
+        run: |
+          kubectl describe deployments.apps -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+          kubectl describe pods -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+          kubectl logs -n semgr8ns -lapp.kubernetes.io/name=semgr8s --prefix=true --tail=-1
+      - name: Display semgr8s configuration
+        if: always()
+        run: |
+          echo "::group::values.yaml"
+          yq e '... comments=""' charts/semgr8s/values.yaml
+          echo "::endgroup::"
+
+  optional-k8s-versions:
+    name: optional k8s versions
+    runs-on: ubuntu-latest
+    if: |
+      inputs.skip != 'non-required' &&
+      inputs.skip != 'all'
+    #    permissions:  #TODO: reactivate for non-private
+    #      packages: read
+    env:
+      IMAGE: ${{ inputs.build_image_repository }}
+      TAG: ${{ inputs.build_tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version: [
+            "v1.20",
+            "v1.21",
+            "v1.22",
+            "v1.23",
+            "v1.24",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Login with registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ inputs.build_registry }}
+          username: ${{ inputs.repo_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install yq
+        run: |
+          sudo snap install yq
+      - uses: ./.github/actions/k8s-version-config
+        name: Setup k8s cluster
+        with:
+          k8s-version: ${{ matrix.k8s-version }}
+      - name: Run pre-config and workload integration tests
+        run: |
+          bash tests/integration/main.sh "basic"
+      - name: Display k8s state and logs if integration test failed
+        if: failure()
+        run: |
+          kubectl describe deployments.apps -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+          kubectl describe pods -n semgr8ns -lapp.kubernetes.io/name=semgr8s
+          kubectl logs -n semgr8ns -lapp.kubernetes.io/name=semgr8s --prefix=true --tail=-1
+      - name: Display semgr8s configuration
+        if: always()
+        run: |
+          echo "::group::values.yaml"
+          yq e '... comments=""' charts/semgr8s/values.yaml
+          echo "::endgroup::"
+

.github/workflows/pr.yml

@@ -25,5 +25,5 @@ jobs:
       skip_sast: 'none'
       skip_sca: 'none'
       skip_docs: 'non-required'
-      skip_integration_tests: 'none'
+      skip_integration_tests: 'non-required'
       output_type: 'sarif'

.github/workflows/push.yml

@@ -25,5 +25,5 @@ jobs:
       skip_sast: 'non-required'
       skip_sca: 'non-required'
       skip_docs: 'none'
-      skip_integration_tests: 'non-required'
+      skip_integration_tests: 'none'
       output_type: 'sarif'

Makefile

@@ -26,7 +26,7 @@ install:
 	@echo "####################"
 	@echo "## $(@)"
 	@echo "####################"
-	helm install semgr8s charts/semgr8s --create-namespace --namespace $(ns)
+	helm install semgr8s charts/semgr8s --atomic --create-namespace --namespace $(ns)
 
 .PHONY:uninstall
 uninstall:
@@ -48,10 +48,10 @@ test:
 	@echo "####################"
 	@echo "## $(@)"
 	@echo "####################"
-	-kubectl create -f tests/
+	-kubectl create -f tests/demo
 	@echo
 	-kubectl get pods -n test-semgr8s-passing
 	@echo
 	-kubectl get pods -n test-semgr8s-failing
 	@echo
-	-kubectl delete -f tests/
+	-kubectl delete -f tests/demo

README.md

@@ -99,30 +99,30 @@ Once all resources are in `READY` state, you have successfully installed semgr8s
 
 ### Testing
 
-Several test resources are provided under `tests/`.
-Semgr8s denies creating pods with insecure configuration according to the rules in `charts/semgr8s/rules`:
+Several test resources are provided under `tests/demo/`.
+For namespaces with label `semgr8s/validation=enabled`, Semgr8s denies creating pods with insecure configuration according to the rules in `charts/semgr8s/rules`:
 
 ```bash
-kubectl create -f tests/failing_deployment.yaml
+kubectl create -f tests/demo/failing_deployment.yaml
 ```
 <details>
   <summary>output</summary>
 
   ```bash
   namespace/test-semgr8s-failing created
-  Error from server: error when creating "tests/failing_deployment.yaml": admission webhook "semgr8s-svc.semgr8ns.svc" denied the request: Found 1 violation(s) of the following policies: 
+  Error from server: error when creating "tests/demo/failing_deployment.yaml": admission webhook "semgr8s-svc.semgr8ns.svc" denied the request: Found 1 violation(s) of the following policies: 
   * rules.allow-privilege-escalation-no-securitycontext
-  Error from server: error when creating "tests/failing_deployment.yaml": admission webhook "semgr8s-svc.semgr8ns.svc" denied the request: Found 1 violation(s) of the following policies: 
+  Error from server: error when creating "tests/demo/failing_deployment.yaml": admission webhook "semgr8s-svc.semgr8ns.svc" denied the request: Found 1 violation(s) of the following policies: 
   * rules.privileged-container
-  Error from server: error when creating "tests/failing_deployment.yaml": admission webhook "semgr8s-svc.semgr8ns.svc" denied the request: Found 1 violation(s) of the following policies: 
+  Error from server: error when creating "tests/demo/failing_deployment.yaml": admission webhook "semgr8s-svc.semgr8ns.svc" denied the request: Found 1 violation(s) of the following policies: 
   * rules.hostnetwork-pod
   ```
 </details>
 
 Securely configured resources on the other hand are permitted to the cluster:
 
 ```bash
-kubectl create -f tests/passing_deployment.yaml
+kubectl create -f tests/demo/passing_deployment.yaml
 ```
 <details>
   <summary>output</summary>
@@ -153,7 +153,7 @@ kubectl delete ns semgr8ns
 Test resources are deleted via:
 
 ```bash
-kubectl delete -f tests/
+kubectl delete -f tests/demo/
 ```
 <details>
   <summary>output</summary>
@@ -162,9 +162,9 @@ kubectl delete -f tests/
   namespace "test-semgr8s-failing" deleted
   namespace "test-semgr8s-passing" deleted
   pod "passing-testpod-1" deleted
-  Error from server (NotFound): error when deleting "tests/failing_deployment.yaml": pods "failing-testpod-1" not found
-  Error from server (NotFound): error when deleting "tests/failing_deployment.yaml": pods "failing-testpod-2" not found
-  Error from server (NotFound): error when deleting "tests/failing_deployment.yaml": pods "failing-testpod-3" not found
+  Error from server (NotFound): error when deleting "tests/demo/failing_deployment.yaml": pods "failing-testpod-1" not found
+  Error from server (NotFound): error when deleting "tests/demo/failing_deployment.yaml": pods "failing-testpod-2" not found
+  Error from server (NotFound): error when deleting "tests/demo/failing_deployment.yaml": pods "failing-testpod-3" not found
 
   ```
 </details>

charts/semgr8s/templates/webhook.yaml

@@ -38,7 +38,7 @@ webhooks:
           - "*"
         operations:
           - CREATE
-        resources: ["pods"]
+          - UPDATE
     clientConfig:
       service:
         name: {{ include "semgr8s.serviceName" . }}

tests/failing_deployment.yaml → tests/demo/failing_deployment.yaml

@@ -33,7 +33,7 @@ spec:
         drop:
           - ALL
       privileged: true
-      readOnlyRootFilesystem: false
+      readOnlyRootFilesystem: true
       runAsNonRoot: true
 ---
 apiVersion: v1
@@ -52,5 +52,5 @@ spec:
         drop:
           - ALL
       privileged: false
-      readOnlyRootFilesystem: false
+      readOnlyRootFilesystem: true
   hostNetwork: true

tests/integration/README.md

@@ -0,0 +1,14 @@
+# Run integration tests
+
+Use the cluster of your choice, e.g. [kind](https://kind.sigs.k8s.io/).
+Specify which semgr8s image is to be used as environment variable, e.g.:
+
+```bash
+export IMAGE=ghcr.io/sse-secure-systems/semgr8s
+export TAG=v0.1.0
+```
+
+Run the desired integration test via:
+```bash
+tests/integration/main.sh "basic"
+```

tests/integration/data/00_namespaces.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: validatedns
+  labels:
+    semgr8s/validation: enabled
+    use: semgr8s-integration-test
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ignoredns
+  labels:
+    use: semgr8s-integration-test

tests/integration/data/20_compliant_pod.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: compliant-pod
+  namespace: validatedns
+spec:
+  containers:
+  - image: busybox
+    name: compliant-pod
+    command: ["/bin/sh", "-ec", "sleep 1000"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+          - ALL
+      privileged: false
+      readOnlyRootFilesystem: true
+      runAsNonRoot: true
+      runAsUser: 10001  # remove when using openshift or OKD 4
+      runAsGroup: 20001  # remove when using openshift or OKD 4
+      seccompProfile:  # remove when using Kubernetes prior v1.19, openshift or OKD 4
+        type: RuntimeDefault  # remove when using Kubernetes prior v1.19, openshift or OKD 4