e2e: add a test for checking AKS upstream

.github/workflows/e2e_aks_runtime.yml

@@ -0,0 +1,23 @@
+name: e2e test aks runtime
+
+on:
+  schedule:
+    - cron: "16 6 * * 6" # 6:16 on Saturdays
+  pull_request:
+    paths:
+      - e2e/aks-runtime/**
+
+jobs:
+  test_matrix:
+    name: Test aks runtime
+    uses: ./.github/workflows/e2e.yml
+    with:
+      skip-undeploy: false
+      test-name: aks-runtime
+      platform: AKS-CLH-SNP
+      runner: ubuntu-22.04
+      self-hosted: false
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write

e2e/aks-runtime/aks_runtime_test.go

@@ -0,0 +1,141 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package aksruntime
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/contrast/e2e/internal/az"
+	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
+	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
+	"github.com/edgelesssys/contrast/internal/kubeapi"
+	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const testContainer = "testcontainer"
+
+var (
+	imageReplacementsFile, namespaceFile, _platformStr string
+	skipUndeploy                                       bool
+)
+
+func TestAKSRuntime(t *testing.T) {
+	require := require.New(t)
+
+	workdir := t.TempDir()
+
+	f, err := os.Open(imageReplacementsFile)
+	require.NoError(err)
+	imageReplacements, err := kuberesource.ImageReplacementsFromFile(f)
+	require.NoError(err)
+	namespace := contrasttest.MakeNamespace(t)
+
+	// Log versions
+	kataPolicyGenV, err := az.KataPolicyGenVersion()
+	require.NoError(err)
+	rg := os.Getenv("azure_resource_group")
+	nodeImageV, err := az.NodeImageVersion(rg, rg)
+	require.NoError(err)
+	t.Log("katapolicygen version: ", kataPolicyGenV)
+	t.Log("node image version: ", nodeImageV)
+
+	c := kubeclient.NewForTest(t)
+
+	// create the namespace
+	ns, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(namespace)})
+	require.NoError(err)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	err = c.Apply(ctx, ns...)
+	cancel()
+	require.NoError(err)
+	if namespaceFile != "" {
+		require.NoError(os.WriteFile(namespaceFile, []byte(namespace), 0o644))
+	}
+
+	// simple deployment that logs the kernel version and then sleeps
+	deployment := kuberesource.Deployment(testContainer, "").
+		WithSpec(kuberesource.DeploymentSpec().
+			WithReplicas(1).
+			WithSelector(kuberesource.LabelSelector().WithMatchLabels(
+				map[string]string{"app.kubernetes.io/name": testContainer},
+			)).
+			WithTemplate(kuberesource.PodTemplateSpec().
+				WithLabels(map[string]string{"app.kubernetes.io/name": testContainer}).
+				WithSpec(kuberesource.PodSpec().
+					WithContainers(kuberesource.Container().
+						WithName(testContainer).
+						WithImage("docker.io/bash@sha256:ce062497c248eb1cf4d32927f8c1780cce158d3ed0658c586a5be7308d583cbb").
+						WithCommand("/usr/local/bin/bash", "-c", "uname -r; sleep infinity"),
+					),
+				),
+			),
+		)
+
+	// define resources
+	resources := []any{deployment}
+	resources = kuberesource.PatchRuntimeHandlers(resources, "kata-cc-isolation")
+	resources = kuberesource.PatchNamespaces(resources, namespace)
+	resources = kuberesource.PatchImages(resources, imageReplacements)
+
+	toWrite, err := kuberesource.ResourcesToUnstructured(resources)
+	require.NoError(err)
+
+	// generate policies
+	resourceBytes, err := kuberesource.EncodeUnstructured(toWrite)
+	require.NoError(err)
+	require.NoError(os.WriteFile(path.Join(workdir, "resources.yaml"), resourceBytes, 0o644))
+	require.NoError(az.KataPolicyGen(path.Join(workdir, "resources.yaml")))
+
+	// load in generated resources
+	resourceBytes, err = os.ReadFile(path.Join(workdir, "resources.yaml"))
+	require.NoError(err)
+	toApply, err := kubeapi.UnmarshalUnstructuredK8SResource(resourceBytes)
+	require.NoError(err)
+
+	ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+	err = c.Apply(ctx, toApply...)
+	require.NoError(err)
+	require.NoError(c.WaitFor(ctx, kubeclient.Ready, kubeclient.Deployment{}, namespace, testContainer))
+
+	t.Cleanup(func() {
+		if skipUndeploy {
+			return
+		}
+
+		// delete the deployment
+		deletePolicy := metav1.DeletePropagationForeground
+		require.NoError(c.Client.AppsV1().Deployments(namespace).Delete(context.Background(), testContainer, metav1.DeleteOptions{
+			PropagationPolicy: &deletePolicy,
+		}))
+	})
+
+	pods, err := c.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	require.NoError(err)
+	pod := pods.Items[0] // only one pod was deployed
+
+	logs, err := c.Client.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx)
+	require.NoError(err)
+	t.Logf("kernel version in pod %s: %s", pod.Name, string(logs))
+}
+
+func TestMain(m *testing.M) {
+	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
+	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&_platformStr, "platform", "", "Deployment platform")
+	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
+	flag.Parse()
+
+	os.Exit(m.Run())
+}

e2e/internal/az/aks.go

@@ -0,0 +1,30 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package az
+
+import (
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+// NodeImageVersion gets the node image version from the specified cluster
+// and resource group.
+func NodeImageVersion(clusterName string, rg string) (string, error) {
+	out, err := exec.Command("az", "aks", "nodepool", "list", "--cluster-name", clusterName, "--resource-group", rg).Output()
+	if err != nil {
+		return "", err
+	}
+
+	var outMap []map[string]interface{}
+	err = json.Unmarshal(out, &outMap)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(fmt.Sprintf("%s", outMap[0]["nodeImageVersion"])), nil
+}

e2e/internal/az/confcom.go

@@ -0,0 +1,25 @@
+// Copyright 2024 Edgeless Systems GmbH
+// SPDX-License-Identifier: AGPL-3.0-only
+
+//go:build e2e
+
+package az
+
+import (
+	"os/exec"
+	"strings"
+)
+
+// KataPolicyGenVersion gets the version string of `az confcom katapolicygen`.
+func KataPolicyGenVersion() (string, error) {
+	out, err := exec.Command("az", "confcom", "katapolicygen", "--print-version").Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+// KataPolicyGen executes `az confcom katapolicygen --yaml <resourcePath>`.
+func KataPolicyGen(resourcePath string) error {
+	return exec.Command("az", "confcom", "katapolicygen", "--yaml", resourcePath).Run()
+}

e2e/internal/contrasttest/contrasttest.go

@@ -52,7 +52,7 @@ type ContrastTest struct {
 // New creates a new contrasttest.T object bound to the given test.
 func New(t *testing.T, imageReplacements, namespaceFile string, platform platforms.Platform, skipUndeploy bool) *ContrastTest {
 	return &ContrastTest{
-		Namespace:             makeNamespace(t),
+		Namespace:             MakeNamespace(t),
 		WorkDir:               t.TempDir(),
 		ImageReplacementsFile: imageReplacements,
 		Platform:              platform,
@@ -372,7 +372,8 @@ func (ct *ContrastTest) FactorPlatformTimeout(timeout time.Duration) time.Durati
 	}
 }
 
-func makeNamespace(t *testing.T) string {
+// MakeNamespace creates a namespace string using a given *testing.T.
+func MakeNamespace(t *testing.T) string {
 	buf := make([]byte, 4)
 	re := regexp.MustCompile("[a-z0-9-]+")
 	n, err := rand.Reader.Read(buf)

flake.nix

@@ -31,6 +31,10 @@
           overlays = [ (import ./overlays/nixpkgs.nix) ];
           config.allowUnfree = true;
           config.nvidia.acceptLicense = true;
+          # TODO(miampf): REMOVE AGAIN ONCE UNNEEDED
+          config.permittedInsecurePackages = [
+            "openssl-1.1.1w"
+          ];
         };
         inherit (pkgs) lib;
         treefmtEval = treefmt-nix.lib.evalModule pkgs ./treefmt.nix;

overlays/nixpkgs.nix

@@ -1,7 +1,61 @@
 # Copyright 2024 Edgeless Systems GmbH
 # SPDX-License-Identifier: AGPL-3.0-only
 
-final: prev: {
+final: prev:
+# TODO(miampf): Remove unneccessary block once https://github.com/NixOS/nixpkgs/pull/345326 is merged into unstable nixpkgs
+let
+  # Builder for Azure CLI extensions. Extensions are Python wheels that
+  # outside of nix would be fetched by the CLI itself from various sources.
+  mkAzExtension =
+    {
+      pname,
+      url,
+      sha256,
+      description,
+      ...
+    }@args:
+    prev.python3.pkgs.buildPythonPackage (
+      {
+        format = "wheel";
+        src = prev.fetchurl { inherit url sha256; };
+        meta = {
+          inherit description;
+          inherit (prev.azure-cli.meta) platforms maintainers;
+          homepage = "https://github.com/Azure/azure-cli-extensions";
+          changelog = "https://github.com/Azure/azure-cli-extensions/blob/main/src/${pname}/HISTORY.rst";
+          license = prev.lib.licenses.mit;
+          sourceProvenance = [ prev.lib.sourceTypes.fromSource ];
+        } // args.meta or { };
+      }
+      // (removeAttrs args [
+        "url"
+        "sha256"
+        "description"
+        "meta"
+      ])
+    );
+
+  confcom = mkAzExtension rec {
+    pname = "confcom";
+    version = "1.0.0";
+    url = "https://azcliprod.blob.core.windows.net/cli-extensions/confcom-${version}-py3-none-any.whl";
+    sha256 = "73823e10958a114b4aca84c330b4debcc650c4635e74c568679b6c32c356411d";
+    description = "Microsoft Azure Command-Line Tools Confidential Container Security Policy Generator Extension";
+    nativeBuildInputs = [ prev.autoPatchelfHook ];
+    buildInputs = [ prev.openssl_1_1 ];
+    propagatedBuildInputs = with prev.python3Packages; [
+      pyyaml
+      deepdiff
+      docker
+      tqdm
+    ];
+    postInstall = ''
+      chmod +x $out/${prev.python3.sitePackages}/azext_confcom/bin/genpolicy-linux
+    '';
+    meta.maintainers = with prev.lib.maintainers; [ miampf ];
+  };
+in
+{
   # Use when a version of Go is needed that is not available in the nixpkgs yet.
   # go_1_xx = prev.go_1_xx.overrideAttrs (finalAttrs: _prevAttrs: {
   #   version = "";
@@ -13,7 +67,10 @@ final: prev: {
 
   # Add the required extensions to the Azure CLI.
   azure-cli = prev.azure-cli.override {
-    withExtensions = with final.azure-cli.extensions; [ aks-preview ];
+    withExtensions = with final.azure-cli.extensions; [
+      aks-preview
+      confcom
+    ];
   };
 
   # Use a newer uplosi that has fixes for private galleries.

packages/by-name/contrast/package.nix

@@ -3,6 +3,8 @@
 
 {
   lib,
+  azure-cli,
+  makeWrapper,
   buildGoModule,
   buildGoTest,
   microsoft,
@@ -26,7 +28,13 @@ let
 
     tags = [ "e2e" ];
 
-    ldflags = [ "-s" ];
+    nativeBuildInputs = [
+      makeWrapper
+    ];
+
+    ldflags = [
+      "-s"
+    ];
 
     subPackages = [
       "e2e/genpolicy"
@@ -38,7 +46,13 @@ let
       "e2e/workloadsecret"
       "e2e/volumestatefulset"
       "e2e/regression"
+      "e2e/aks-runtime"
     ];
+
+    postInstall = ''
+      wrapProgram $out/bin/aks-runtime.test \
+        --prefix PATH : ${lib.makeBinPath [ azure-cli ]}
+    '';
   };
 
   # Reference values that we embed into the Contrast CLI for