some fixes to the runtime cleanup

.github/workflows/cleanup.yml

@@ -0,0 +1,39 @@
+name: push cleanup-bare-metal image
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/cleaup.yml
+      - packages/**
+
+env:
+  container_registry: ghcr.io/edgelesssys
+
+jobs:
+  push:
+    name: "push cleanup image"
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Log in to ghcr.io Container registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0
+      - name: Create justfile.env
+        run: |
+          cat <<EOF > justfile.env
+          container_registry=${{ env.container_registry }}
+          EOF
+      - name: Build and push cleanup-bare-metal image
+        run: |
+          just push cleanup-bare-metal

packages/cleanup-bm.sh → packages/cleanup-bare-metal.sh

@@ -24,7 +24,7 @@ done
 
 kubectl get pods --all-namespaces -o jsonpath='{.items[?(@.metadata.annotations.contrast\.edgeless\.systems/pod-role=="contrast-node-installer")].spec.containers[0].args[1]}' |
   tr ' ' '\n' |
-  grep -o "contrast-cc-.\+" >>usedRuntimeClasses
+  grep -o "contrast-cc-.\+" >>usedRuntimeClasses || true
 sort -u usedRuntimeClasses -o usedRuntimeClasses
 
 mapfile -t unusedRuntimeClasses < <(
@@ -56,3 +56,6 @@ for runtimeClass in "${unusedRuntimeClasses[@]}"; do
   dasel delete --file "${CONFIG}" --indent 0 --read toml --write toml "plugins.io\.containerd\.grpc\.v1\.cri.containerd.runtimes.${runtimeClass}" 2>/dev/null
   dasel delete --file "${CONFIG}" --indent 0 --read toml --write toml "proxy_plugins.${SNAPSHOTTER}-${runtimeClass}" 2>/dev/null
 done
+
+# Fix the state for removed snapshotters.
+cleanup-images

tools/cleanup-images.sh → packages/cleanup-images.sh

@@ -37,7 +37,7 @@ for image in "${pauseImages[@]}"; do
   ctr "${ctrOpts[@]}" content fetch "${image}"
 done
 
-if ctr "${ctrOpts[@]}" image check | grep --exit-code "incomplete"; then
+if ctr "${ctrOpts[@]}" image check | grep "incomplete"; then
   echo "Incomplete images detected"
   exit 1
 fi

packages/containers.nix

@@ -156,11 +156,14 @@ let
       };
     };
 
-    cleanup-bm = dockerTools.buildImage {
-      name = "cleanup-bm";
-      tag = "v0.0.1";
+    cleanup-bare-metal = dockerTools.buildImage {
+      name = "cleanup-bare-metal";
+      tag = "latest";
+      copyToRoot = with pkgs; [
+        cacert
+      ];
       config = {
-        Cmd = [ "${lib.getExe pkgs.scripts.cleanup-bm}" ];
+        Cmd = [ "${lib.getExe pkgs.scripts.cleanup-bare-metal}" ];
       };
     };
   };

packages/scripts.nix

@@ -491,13 +491,24 @@
     '';
   };
 
-  cleanup-bm = writeShellApplication {
-    name = "cleanup-bm";
+  cleanup-bare-metal = writeShellApplication {
+    name = "cleanup-bare-metal";
     runtimeInputs = with pkgs; [
       busybox
       kubectl
       dasel
+      scripts.cleanup-images
     ];
-    text = builtins.readFile ./cleanup-bm.sh;
+    text = builtins.readFile ./cleanup-bare-metal.sh;
+  };
+
+  cleanup-images = writeShellApplication {
+    name = "cleanup-images";
+    runtimeInputs = with pkgs; [
+      gnugrep
+      busybox
+      containerd
+    ];
+    text = builtins.readFile ./cleanup-images.sh;
   };
 }

tools/bm-maintenance/deployment_tdx_snp.yml

@@ -47,7 +47,7 @@ spec:
           serviceAccountName: cleanup-sa
           containers:
             - name: cleanup
-              image: ghcr.io/edgelesssys/contrast/cleanup-bm:v0.0.1
+              image: ghcr.io/edgelesssys/contrast/cleanup-bare-metal
               env:
                 - name: OPTEDGELESS
                   value: /opt/edgeless
@@ -62,6 +62,8 @@ spec:
                   mountPath: /var/lib/nydus-snapshotter
                 - name: containerd-config
                   mountPath: /var/lib/rancher/k3s/agent/etc/containerd
+                - name: containerd-run
+                  mountPath: /run/k3s/containerd/
           volumes:
             - name: opt-edgeless
               hostPath:
@@ -75,4 +77,8 @@ spec:
               hostPath:
                 path: /var/lib/rancher/k3s/agent/etc/containerd
                 type: Directory
+            - name: containerd-run
+              hostPath:
+                path: /run/k3s/containerd/
+                type: Directory
           restartPolicy: OnFailure