From b747cfd686caf9b399936fbbbdab8c96241b43bd Mon Sep 17 00:00:00 2001
From: Ike Ma <ike.shibin.ma@gmail.com>
Date: Thu, 30 Mar 2023 18:34:11 +0000
Subject: [PATCH] Add ARM64 node_e2e test

Run node_e2e serial tests on GCP T2A (arm64) machine
---
 .../kubernetes/sig-node/node-kubelet.yaml     | 43 +++++++++++++++++++
 jobs/e2e_node/arm/image-config-serial.yaml    | 10 +++++
 jobs/e2e_node/arm/init.yaml                   | 28 ++++++++++++
 3 files changed, 81 insertions(+)
 create mode 100644 jobs/e2e_node/arm/image-config-serial.yaml
 create mode 100644 jobs/e2e_node/arm/init.yaml

diff --git a/config/jobs/kubernetes/sig-node/node-kubelet.yaml b/config/jobs/kubernetes/sig-node/node-kubelet.yaml
index 9df7e9c2e25a..acc8d96d9f99 100644
--- a/config/jobs/kubernetes/sig-node/node-kubelet.yaml
+++ b/config/jobs/kubernetes/sig-node/node-kubelet.yaml
@@ -120,6 +120,49 @@ periodics:
     testgrid-alert-email: skclark@redhat.com
     description: "Uses kubetest to run serial node-e2e tests (+Serial, -Flaky|Benchmark|Node*Feature)"
 
+- name: ci-kubernetes-node-arm64-ubuntu-serial
+  interval: 4h
+  cluster: k8s-infra-prow-build
+  annotations:
+    testgrid-dashboards: sig-node-kubelet
+    testgrid-tab-name: kubelet-gce-e2e-arm64-ubuntu-serial
+    description: "Run serial node e2e tests on ARM64 environment on Ubuntu"
+  labels:
+    preset-service-account: "true"
+    preset-k8s-ssh: "true"
+    preset-dind-enabled: "true"
+  spec:
+    containers:
+      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20230513-7e1db2f1bb-master
+        args:
+          - --repo=k8s.io/kubernetes=master
+          - --timeout=240
+          - --root=/go/src
+          - --scenario=kubernetes_e2e
+          - --
+          - --deployment=node
+          - --gcp-zone=us-central1-a
+          - --use-dockerized-build=true
+          - --target-build-arch=linux/arm64
+          - --node-args=--image-config-file=/workspace/test-infra/jobs/e2e_node/arm/image-config-serial.yaml
+          - '--node-test-args=--container-runtime-endpoint=unix:///run/containerd/containerd.sock --container-runtime-process-name=/usr/bin/containerd --container-runtime-pid-file= --kubelet-flags="--cgroup-driver=systemd --cgroups-per-qos=true --cgroup-root=/ --runtime-cgroups=/system.slice/containerd.service" --extra-log="{\"name\": \"containerd.log\", \"journalctl\": [\"-u\", \"containerd*\"]}"'
+          - --node-tests=true
+          - --provider=gce
+          - --test_args=--nodes=1 --focus="\[Serial\]" --skip="\[Flaky\]|\[Slow\]|\[Benchmark\]|\[NodeSpecialFeature:.+\]|\[NodeSpecialFeature\]|\[NodeAlphaFeature:.+\]|\[NodeAlphaFeature\]|\[NodeFeature:Eviction\]|\[NodeFeature:NodeProblemDetector\]|\[NodeFeature:OOMScoreAdj\]|\[NodeFeature:DevicePluginProbe\]|\[NodeConformance\]"
+          - --timeout=180m
+        securityContext:
+          privileged: true
+        env:
+          - name: GOPATH
+            value: /go
+        resources:
+          limits:
+            cpu: 4
+            memory: 6Gi
+          requests:
+            cpu: 4
+            memory: 6Gi
+
 - name: ci-kubernetes-node-swap-fedora
   cluster: k8s-infra-prow-build
   interval: 4h
diff --git a/jobs/e2e_node/arm/image-config-serial.yaml b/jobs/e2e_node/arm/image-config-serial.yaml
new file mode 100644
index 000000000000..915845fce33e
--- /dev/null
+++ b/jobs/e2e_node/arm/image-config-serial.yaml
@@ -0,0 +1,10 @@
+# To copy an image between projects:
+# `gcloud compute --project <to-project> disks create <image name> --image=https://www.googleapis.com/compute/v1/projects/<from-project>/global/images/<image-name>`
+# `gcloud compute --project <to-project> images create <image-name> --source-disk=<image-name>`
+images:
+  ubuntu:
+    image_family: pipeline-1-25-arm64
+    machine: t2a-standard-2 # These tests need a lot of memory
+    metadata: "user-data</workspace/test-infra/jobs/e2e_node/arm/init.yaml,cni-template</workspace/test-infra/jobs/e2e_node/containerd/cni.template,containerd-config</workspace/test-infra/jobs/e2e_node/containerd/config.toml"
+    project: ubuntu-os-gke-cloud
+
diff --git a/jobs/e2e_node/arm/init.yaml b/jobs/e2e_node/arm/init.yaml
new file mode 100644
index 000000000000..414f3bade643
--- /dev/null
+++ b/jobs/e2e_node/arm/init.yaml
@@ -0,0 +1,28 @@
+#cloud-config
+
+runcmd:
+  - echo "Test run from /tmp folder, remounting it"
+  - mount /tmp /tmp -o remount,exec,suid
+
+  - echo "This will configure built-in containerd for k8s tests. Containerd version is:"
+  - ctr version # current version of containerd
+
+  - echo "Download and install CNI configuration to /home/containerd"
+  - mkdir -p /home/containerd
+  - mount --bind /home/containerd /home/containerd
+  - mount -o remount,exec /home/containerd
+  - 'curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/cni.template http://metadata.google.internal/computeMetadata/v1/instance/attributes/cni-template'
+
+  - echo "Download and install CNI to /home/containerd"
+  - 'curl -sSL --fail --retry 5 --retry-delay 3 --silent --show-error -o /home/containerd/cni.tgz https://storage.googleapis.com/k8s-artifacts-cni/release/v1.0.1/cni-plugins-linux-arm64-v1.0.1.tgz'
+  - tar xzf /home/containerd/cni.tgz -C /home/containerd --overwrite
+
+  - echo "Set containerd configuration"
+  - mkdir -p /etc/containerd
+  - 'curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /etc/containerd/config.toml http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-config'
+
+  - echo "Restarting containerd"
+  - systemctl restart containerd
+
+  - echo "Configuration complete"
+