Add e2e tests for rootless control-plane.

kinder/ci/tools/update-workflows/config.yaml

@@ -166,3 +166,15 @@ jobGroups:
     - ./templates/workflows/upgrade-latest-no-addon-config-maps.yaml
   jobs:
   - kubernetesVersion: latest
+
+- name: rootless
+  testInfraJobSpec:
+    targetFile: kubeadm-kinder-rootless.yaml
+    template: ./templates/testinfra/kubeadm-kinder-rootless.yaml
+  kinderWorkflowSpec:
+    targetFile: rootless-{{ .KubernetesVersion }}.yaml
+    template: ./templates/workflows/rootless.yaml
+    additionalFiles:
+    - ./templates/workflows/rootless-tasks.yaml
+  jobs:
+  - kubernetesVersion: latest

kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-rootless.yaml

@@ -0,0 +1,38 @@
+- name: ci-kubernetes-e2e-kubeadm-kinder-rootless-{{ dashVer .KubernetesVersion }}
+  interval: {{ .JobInterval }}
+  decorate: true
+  labels:
+    preset-dind-enabled: "true"
+    preset-kind-volume-mounts: "true"
+  annotations:
+    testgrid-dashboards: sig-cluster-lifecycle-kubeadm
+    testgrid-tab-name: kubeadm-kinder-rootless-{{ dashVer .KubernetesVersion }}
+    testgrid-alert-email: [email protected]
+    description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster with rootless control-plane and run kubeadm-e2e and the conformance suite"
+    testgrid-num-columns-recent: "20"
+{{ .AlertAnnotations }}
+  decoration_config:
+    timeout: 60m
+  extra_refs:
+  - org: kubernetes
+    repo: kubernetes
+    base_ref: {{ branchFor .KubernetesVersion }}
+    path_alias: k8s.io/kubernetes
+  - org: kubernetes
+    repo: kubeadm
+    base_ref: master
+    path_alias: k8s.io/kubeadm
+  spec:
+    containers:
+    - image: gcr.io/k8s-testimages/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }}
+      command:
+      - runner.sh
+      - "../kubeadm/kinder/ci/kinder-run.sh"
+      args:
+      - {{ .WorkflowFile }}
+      securityContext:
+        privileged: true
+      resources:
+        requests:
+          memory: "9000Mi"
+          cpu: 2000m

kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml

@@ -0,0 +1,280 @@
+# IMPORTANT! this workflow is imported by rootless-* workflows.
+version: 1
+summary: |
+  This workflow implements a sequence of tasks used test the proper functioning
+  of kubeadm with the control-plane running as non-root.
+vars:
+  # vars defines default values for variable used by tasks in this workflow;
+  # those values might be overridden when importing this files.
+  kubernetesVersion: v1.13.5
+  controlPlaneNodes: 3
+  workerNodes: 2
+  baseImage: kindest/base:v20191105-ee880e9b # has containerd
+  image: kindest/node:test
+  clusterName: kinder-rootless
+  kubeadmVerbosity: 6
+tasks:
+- name: pull-base-image
+  description: |
+    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
+  cmd: docker
+  args:
+    - pull
+    - "{{ .vars.baseImage }}"
+- name: add-kubernetes-versions
+  description: |
+    creates a node-image-variant by adding a Kubernetes version
+  cmd: kinder
+  args:
+    - build
+    - node-image-variant
+    - --base-image={{ .vars.baseImage }}
+    - --image={{ .vars.image }}
+    - --with-init-artifacts={{ .vars.kubernetesVersion }}
+    - --loglevel=debug
+  timeout: 15m
+- name: create-cluster
+  description: |
+    create a set of nodes ready for hosting the Kubernetes cluster
+  cmd: kinder
+  args:
+    - create
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --image={{ .vars.image }}
+    - --control-plane-nodes={{ .vars.controlPlaneNodes }}
+    - --worker-nodes={{ .vars.workerNodes }}
+    - --loglevel=debug
+  timeout: 5m
+- name: prepare verify-rootless.sh script
+  cmd: /bin/sh
+  args:
+    - -c
+    - |
+      cat <<EOF >/tmp/verify-rootless.sh
+      #!/usr/bin/env bash
+      res=0
+      users=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd")
+      for d in ${users[@]}; do
+        if grep -q "^\$d:" /etc/passwd ; then
+            echo "/etc/passwd has user \$d!"
+        else
+            echo "ERROR: /etc/passwd does not have user \$d"
+            res=1
+        fi
+      done
+
+      groups=("kubeadm-kas" "kubeadm-ks" "kubeadm-kcm" "kubeadm-etcd" "kubeadm-sa-key-readers")
+      for d in ${groups[@]}; do
+        if grep -q "^\$d:" /etc/group ; then
+            echo "/etc/group has user \$d!"
+        else
+            echo "ERROR: /etc/group does not have user \$d"
+            res=1
+        fi
+      done
+
+      # Here pgrep will return the PID of the process and we will pass the PID using xargs
+      # to the ps command as an argument.
+      # `ps o user:16 --no-headers -p <PID>` prints the name of the user that the process with PID is running as.
+      if pgrep kube-apiserver | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kas ; then
+          echo "kube-apiserver is running as user kubeadm-kas"
+      else
+          echo "ERROR: kube-apiserver is not running as user kubeadm-kas"
+          res=1
+      fi
+
+      if pgrep kube-apiserver | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kas ; then
+          echo "kube-apiserver is running as user kubeadm-kas"
+      else
+          echo "ERROR: kube-apiserver is not running as user kubeadm-kas"
+          res=1
+      fi
+
+      if pgrep kube-apiserver | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then
+          echo "kube-apiserver is running as supplemental group kubeadm-sa-key-readers"
+      else
+          echo "ERROR: kube-apiserver is not running as supplemental group kubeadm-sa-key-readers"
+          res=1
+      fi
+
+      if pgrep kube-controller-manager | xargs ps o user:16 --no-headers -p | grep -q kubeadm-kcm ; then
+          echo "kube-controller-manager is running as user kubeadm-kcm"
+      else
+          echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm"
+          res=1
+      fi
+
+      if pgrep kube-controller-manager | xargs ps o group:16 --no-headers -p | grep -q kubeadm-kcm ; then
+          echo "kube-controller-manager is running as user kubeadm-kcm"
+      else
+          echo "ERROR: kube-controller-manager is not running as user kubeadm-kcm"
+          res=1
+      fi
+
+      if pgrep kube-controller-manager | xargs ps o supgrp:16 --no-headers -p | grep -q kubeadm-sa-key-readers ; then
+          echo "kube-controller-manager is running as supplemental group kubeadm-sa-key-readers"
+      else
+          echo "ERROR: kube-controller-manager is not running as supplemental group kubeadm-sa-key-readers"
+          res=1
+      fi
+
+      if pgrep kube-scheduler | xargs ps o user:16 --no-headers -p | grep -q kubeadm-ks ; then
+          echo "kube-scheduler is running as user kubeadm-ks"
+      else
+          echo "ERROR: kube-scheduler is not running as user kubeadm-ks"
+          res=1
+      fi
+
+      if pgrep kube-scheduler | xargs ps o group:16 --no-headers -p | grep -q kubeadm-ks ; then
+          echo "kube-scheduler is running as user kubeadm-ks"
+      else
+          echo "ERROR: kube-scheduler is not running as user kubeadm-ks"
+          res=1
+      fi
+
+      if pgrep etcd | xargs ps o user:16 --no-headers -p | grep -q kubeadm-etcd ; then
+          echo "etcd is running as user kubeadm-etcd"
+      else
+          echo "ERROR: etcd is not running as user kubeadm-etcd"
+          res=1
+      fi
+
+      if pgrep etcd | xargs ps o group:16 --no-headers -p | grep -q kubeadm-etcd ; then
+          echo "etcd is running as user kubeadm-etcd"
+      else
+          echo "ERROR: etcd is not running as user kubeadm-etcd"
+          res=1
+      fi
+
+      if [[ "\${res}" = 0 ]]; then
+        echo "All verify checks passed, congrats!"
+        echo ""
+      else
+        echo "One or more verify checks failed! See output above..."
+        echo ""
+        exit 1
+      fi
+      EOF
+
+      chmod +x /tmp/verify-rootless.sh
+- name: copy verify-rootless.sh on controlplane nodes
+  cmd: kinder
+  args:
+    - cp
+    - --name={{ .vars.clusterName }}
+    - /tmp/verify-rootless.sh
+    - "@cp*:/kinder/verify-rootless.sh"
+    - --loglevel=debug
+- name: init
+  description: |
+    Initializes the Kubernetes cluster with version "initVersion"
+    by starting the boostrap control-plane nodes
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-init
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+    - --kubeadm-feature-gate="RootlessControlPlane=true"
+  timeout: 5m
+- name: join
+  description: |
+    Join the other nodes to the Kubernetes cluster
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-join
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  timeout: 10m
+- name: run verify-rootless.sh on controlplane nodes before upgrades
+  cmd: kinder
+  args:
+    - exec
+    - --name={{ .vars.clusterName }}
+    - "@cp*"
+    - /kinder/verify-rootless.sh
+    - --loglevel=debug
+- name: e2e-kubeadm
+  description: |
+    Runs kubeadm e2e tests
+  cmd: kinder
+  args:
+    - test
+    - e2e-kubeadm
+    - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  timeout: 10m
+- name: e2e
+  description: |
+    Runs Kubernetes e2e test (conformance)
+  cmd: kinder
+  args:
+    - test
+    - e2e
+    - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e
+    - --parallel
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  timeout: 35m
+- name: upgrade
+  description: |
+    upgrades the cluster to Kubernetes "upgradeVersion"
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-upgrade
+    - --upgrade-version={{ .vars.kubernetesVersion }}
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  timeout: 15m
+- name: run verify-rootless.sh on controlplane nodes after upgrades
+  cmd: kinder
+  args:
+    - exec
+    - --name={{ .vars.clusterName }}
+    - "@cp*"
+    - /kinder/verify-rootless.sh
+    - --loglevel=debug
+- name: get-logs
+  description: |
+    Collects all the test logs
+  cmd: kinder
+  args:
+    - export
+    - logs
+    - --loglevel=debug
+    - --name={{ .vars.clusterName }}
+    - "{{ .env.ARTIFACTS }}"
+  force: true
+  timeout: 5m
+  # kind export log is know to be flaky, so we are temporary ignoring errors in order
+  # to make the test pass in case everything else passed
+  # see https://github.com/kubernetes-sigs/kind/issues/456
+  ignoreError: true
+- name: reset
+  description: |
+    Exec kubeadm reset
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-reset
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  force: true
+- name: delete
+  description: |
+    Deletes the cluster
+  cmd: kinder
+  args:
+    - delete
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  force: true

kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml

@@ -0,0 +1,10 @@
+version: 1
+summary: |
+  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes with
+  the control-plane running as non-root.
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-rootless{{ dashVer .KubernetesVersion }}
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
+vars:
+  kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}"
+tasks:
+- import: rootless-tasks.yaml

kinder/ci/workflows/rootless-latest.yaml

@@ -0,0 +1,11 @@
+# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
+version: 1
+summary: |
+  This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes with
+  the control-plane running as non-root.
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-rootlesslatest
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-rootless.yaml
+vars:
+  kubernetesVersion: "{{ resolve `ci/latest` }}"
+tasks:
+- import: rootless-tasks.yaml