-
Notifications
You must be signed in to change notification settings - Fork 7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
e2e: add a test for checking AKS upstream #939
base: main
Are you sure you want to change the base?
Changes from 11 commits
fef7b4f
bd42684
895bd72
6407e0f
e48ec8c
509dffa
e978836
0128a58
aa70b82
4156f7e
807b8b3
3861654
7426c13
71b3501
e0222b4
15ecc5c
51aaff3
1850c67
0a6c0c3
b06a87e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
name: e2e test aks runtime | ||
|
||
on: | ||
schedule: | ||
- cron: "16 6 * * 6" # 6:16 on Saturdays | ||
pull_request: | ||
paths: | ||
- e2e/aks-runtime/** | ||
|
||
jobs: | ||
test_matrix: | ||
name: Test aks runtime | ||
uses: ./.github/workflows/e2e.yml | ||
with: | ||
skip-undeploy: false | ||
test-name: aks-runtime | ||
platform: AKS-CLH-SNP | ||
runner: ubuntu-22.04 | ||
self-hosted: false | ||
secrets: inherit | ||
permissions: | ||
contents: read | ||
packages: write |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,141 @@ | ||
// Copyright 2024 Edgeless Systems GmbH | ||
// SPDX-License-Identifier: AGPL-3.0-only | ||
|
||
//go:build e2e | ||
|
||
package aksruntime | ||
|
||
import ( | ||
"context" | ||
"flag" | ||
"os" | ||
"path" | ||
"testing" | ||
"time" | ||
|
||
"github.com/edgelesssys/contrast/e2e/internal/az" | ||
"github.com/edgelesssys/contrast/e2e/internal/contrasttest" | ||
"github.com/edgelesssys/contrast/e2e/internal/kubeclient" | ||
"github.com/edgelesssys/contrast/internal/kubeapi" | ||
"github.com/edgelesssys/contrast/internal/kuberesource" | ||
"github.com/stretchr/testify/require" | ||
corev1 "k8s.io/api/core/v1" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
) | ||
|
||
const testContainer = "testcontainer" | ||
|
||
var ( | ||
imageReplacementsFile, namespaceFile, _platformStr string | ||
skipUndeploy bool | ||
) | ||
|
||
func TestAKSRuntime(t *testing.T) { | ||
require := require.New(t) | ||
|
||
workdir := t.TempDir() | ||
|
||
f, err := os.Open(imageReplacementsFile) | ||
require.NoError(err) | ||
imageReplacements, err := kuberesource.ImageReplacementsFromFile(f) | ||
require.NoError(err) | ||
namespace := contrasttest.MakeNamespace(t) | ||
|
||
// Log versions | ||
kataPolicyGenV, err := az.KataPolicyGenVersion() | ||
require.NoError(err) | ||
rg := os.Getenv("azure_resource_group") | ||
nodeImageV, err := az.NodeImageVersion(rg, rg) | ||
require.NoError(err) | ||
t.Log("katapolicygen version: ", kataPolicyGenV) | ||
t.Log("node image version: ", nodeImageV) | ||
|
||
c := kubeclient.NewForTest(t) | ||
|
||
// create the namespace | ||
ns, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(namespace)}) | ||
require.NoError(err) | ||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) | ||
err = c.Apply(ctx, ns...) | ||
cancel() | ||
require.NoError(err) | ||
if namespaceFile != "" { | ||
require.NoError(os.WriteFile(namespaceFile, []byte(namespace), 0o644)) | ||
} | ||
|
||
// simple deployment that logs the kernel version and then sleeps | ||
deployment := kuberesource.Deployment(testContainer, ""). | ||
WithSpec(kuberesource.DeploymentSpec(). | ||
WithReplicas(1). | ||
WithSelector(kuberesource.LabelSelector().WithMatchLabels( | ||
map[string]string{"app.kubernetes.io/name": testContainer}, | ||
)). | ||
WithTemplate(kuberesource.PodTemplateSpec(). | ||
WithLabels(map[string]string{"app.kubernetes.io/name": testContainer}). | ||
WithSpec(kuberesource.PodSpec(). | ||
WithContainers(kuberesource.Container(). | ||
WithName(testContainer). | ||
WithImage("docker.io/bash@sha256:ce062497c248eb1cf4d32927f8c1780cce158d3ed0658c586a5be7308d583cbb"). | ||
WithCommand("/usr/local/bin/bash", "-c", "uname -r; sleep infinity"), | ||
), | ||
), | ||
), | ||
) | ||
|
||
// define resources | ||
resources := []any{deployment} | ||
resources = kuberesource.PatchRuntimeHandlers(resources, "kata-cc-isolation") | ||
resources = kuberesource.PatchNamespaces(resources, namespace) | ||
resources = kuberesource.PatchImages(resources, imageReplacements) | ||
|
||
toWrite, err := kuberesource.ResourcesToUnstructured(resources) | ||
require.NoError(err) | ||
|
||
// generate policies | ||
resourceBytes, err := kuberesource.EncodeUnstructured(toWrite) | ||
require.NoError(err) | ||
require.NoError(os.WriteFile(path.Join(workdir, "resources.yaml"), resourceBytes, 0o644)) | ||
require.NoError(az.KataPolicyGen(path.Join(workdir, "resources.yaml"))) | ||
|
||
// load in generated resources | ||
resourceBytes, err = os.ReadFile(path.Join(workdir, "resources.yaml")) | ||
require.NoError(err) | ||
toApply, err := kubeapi.UnmarshalUnstructuredK8SResource(resourceBytes) | ||
require.NoError(err) | ||
|
||
ctx, cancel = context.WithTimeout(context.Background(), 3*time.Minute) | ||
defer cancel() | ||
err = c.Apply(ctx, toApply...) | ||
require.NoError(err) | ||
require.NoError(c.WaitFor(ctx, kubeclient.Ready, kubeclient.Deployment{}, namespace, testContainer)) | ||
|
||
t.Cleanup(func() { | ||
if skipUndeploy { | ||
return | ||
} | ||
|
||
// delete the deployment | ||
deletePolicy := metav1.DeletePropagationForeground | ||
require.NoError(c.Client.AppsV1().Deployments(namespace).Delete(context.Background(), testContainer, metav1.DeleteOptions{ | ||
PropagationPolicy: &deletePolicy, | ||
})) | ||
}) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why not just delete the namespace? |
||
|
||
pods, err := c.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) | ||
require.NoError(err) | ||
pod := pods.Items[0] // only one pod was deployed | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Consider asserting this. |
||
|
||
logs, err := c.Client.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{}).DoRaw(ctx) | ||
require.NoError(err) | ||
t.Logf("kernel version in pod %s: %s", pod.Name, string(logs)) | ||
} | ||
|
||
func TestMain(m *testing.M) { | ||
flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file") | ||
flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in") | ||
flag.StringVar(&_platformStr, "platform", "", "Deployment platform") | ||
flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test") | ||
flag.Parse() | ||
|
||
os.Exit(m.Run()) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
// Copyright 2024 Edgeless Systems GmbH | ||
// SPDX-License-Identifier: AGPL-3.0-only | ||
|
||
//go:build e2e | ||
|
||
package az | ||
|
||
import ( | ||
"encoding/json" | ||
"fmt" | ||
"os/exec" | ||
"strings" | ||
) | ||
|
||
// NodeImageVersion gets the node image version from the specified cluster | ||
// and resource group. | ||
func NodeImageVersion(clusterName string, rg string) (string, error) { | ||
out, err := exec.Command("az", "aks", "nodepool", "list", "--cluster-name", clusterName, "--resource-group", rg).Output() | ||
if err != nil { | ||
return "", err | ||
} | ||
|
||
var outMap []map[string]interface{} | ||
err = json.Unmarshal(out, &outMap) | ||
if err != nil { | ||
return "", err | ||
} | ||
|
||
return strings.TrimSpace(fmt.Sprintf("%s", outMap[0]["nodeImageVersion"])), nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
// Copyright 2024 Edgeless Systems GmbH | ||
// SPDX-License-Identifier: AGPL-3.0-only | ||
|
||
//go:build e2e | ||
|
||
package az | ||
|
||
import ( | ||
"os/exec" | ||
"strings" | ||
) | ||
|
||
// KataPolicyGenVersion gets the version string of `az confcom katapolicygen`. | ||
func KataPolicyGenVersion() (string, error) { | ||
out, err := exec.Command("az", "confcom", "katapolicygen", "--print-version").Output() | ||
if err != nil { | ||
return "", err | ||
} | ||
return strings.TrimSpace(string(out)), nil | ||
} | ||
|
||
// KataPolicyGen executes `az confcom katapolicygen --yaml <resourcePath>`. | ||
func KataPolicyGen(resourcePath string) error { | ||
return exec.Command("az", "confcom", "katapolicygen", "--yaml", resourcePath).Run() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does this potentially swallow interesting command output? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not really, at most it swallows stderr which could be interesting if the command fails. I've redirected the commands stderr output to the stderr output of the OS as a fix. |
||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Let's use
quay.io
orghcr.io
.