Add e2e tests for rootless control-plane.

kubernetes · Jun 19, 2021 · 00069bb · 00069bb
1 parent b300f48
commit 00069bb
Show file tree

Hide file tree

Showing 6 changed files with 186 additions and 7 deletions.
diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml
@@ -166,3 +166,15 @@ jobGroups:
     - ./templates/workflows/upgrade-latest-no-addon-config-maps.yaml
   jobs:
   - kubernetesVersion: latest
+
+- name: rootless
+  testInfraJobSpec:
+    targetFile: kubeadm-kinder-rootless.yaml
+    template: ./templates/testinfra/kubeadm-kinder-rootless.yaml
+  kinderWorkflowSpec:
+    targetFile: rootless-{{ .KubernetesVersion }}.yaml
+    template: ./templates/workflows/rootless.yaml
+    additionalFiles:
+    - ./templates/workflows/rootless-tasks.yaml
+  jobs:
+  - kubernetesVersion: latest
diff --git a/kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/rootless-tasks.yaml
@@ -0,0 +1,141 @@
+# IMPORTANT! this workflow is imported by regular-* workflows.
+version: 1
+summary: |
+  This workflow implements a sequence of tasks used test the proper functioning
+  of kubeadm version X with Kubernetes version X.
+vars:
+  # vars defines default values for variable used by tasks in this workflow;
+  # those values might be overridden when importing this files.
+  kubernetesVersion: v1.13.5
+  controlPlaneNodes: 3
+  workerNodes: 2
+  baseImage: kindest/base:v20191105-ee880e9b # has containerd
+  image: kindest/node:test
+  clusterName: kinder-regular
+  kubeadmVerbosity: 6
+tasks:
+- name: pull-base-image
+  description: |
+    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
+  cmd: docker
+  args:
+    - pull
+    - "{{ .vars.baseImage }}"
+- name: add-kubernetes-versions
+  description: |
+    creates a node-image-variant by adding a Kubernetes version
+  cmd: kinder
+  args:
+    - build
+    - node-image-variant
+    - --base-image={{ .vars.baseImage }}
+    - --image={{ .vars.image }}
+    - --with-init-artifacts={{ .vars.kubernetesVersion }}
+    - --loglevel=debug
+  timeout: 15m
+- name: create-cluster
+  description: |
+    create a set of nodes ready for hosting the Kubernetes cluster
+  cmd: kinder
+  args:
+    - create
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --image={{ .vars.image }}
+    - --control-plane-nodes={{ .vars.controlPlaneNodes }}
+    - --worker-nodes={{ .vars.workerNodes }}
+    - --loglevel=debug
+  timeout: 5m
+- name: init
+  description: |
+    Initializes the Kubernetes cluster with version "initVersion"
+    by starting the boostrap control-plane nodes
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-init
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+    - --feature-gates="RootlessControlPlane=true"
+  timeout: 5m
+- name: join
+  description: |
+    Join the other nodes to the Kubernetes cluster
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-join
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  timeout: 10m
+- name: cluster-info
+  description: |
+    Runs cluster-info
+  cmd: kinder
+  args:
+    - do
+    - cluster-info
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+- name: e2e-kubeadm
+  description: |
+    Runs kubeadm e2e tests
+  cmd: kinder
+  args:
+    - test
+    - e2e-kubeadm
+    - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  timeout: 10m
+- name: e2e
+  description: |
+    Runs Kubernetes e2e test (conformance)
+  cmd: kinder
+  args:
+    - test
+    - e2e
+    - --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e
+    - --parallel
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  timeout: 35m
+- name: get-logs
+  description: |
+    Collects all the test logs
+  cmd: kinder
+  args:
+    - export
+    - logs
+    - --loglevel=debug
+    - --name={{ .vars.clusterName }}
+    - "{{ .env.ARTIFACTS }}"
+  force: true
+  timeout: 5m
+  # kind export log is know to be flaky, so we are temporary ignoring errors in order
+  # to make the test pass in case everything else passed
+  # see https://github.com/kubernetes-sigs/kind/issues/456
+  ignoreError: true
+- name: reset
+  description: |
+    Exec kubeadm reset
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-reset
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  force: true
+- name: delete
+  description: |
+    Deletes the cluster
+  cmd: kinder
+  args:
+    - delete
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  force: true
diff --git a/kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml b/kinder/ci/tools/update-workflows/templates/workflows/rootless.yaml
@@ -0,0 +1,9 @@
+version: 1
+summary: |
+  This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-rootless{{ dashVer .KubernetesVersion }}
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
+vars:
+  kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}"
+tasks:
+- import: rootless-tasks.yaml
diff --git a/kinder/cmd/kinder/do/do.go b/kinder/cmd/kinder/do/do.go
@@ -43,6 +43,7 @@ type flagpole struct {
 	Wait                  time.Duration
 	IgnorePreflightErrors string
 	KubeadmConfigVersion  string
+	FeatureGates          string
 }
 
 // NewCommand returns a new cobra.Command for exec
@@ -122,6 +123,11 @@ func NewCommand() *cobra.Command {
 			"If not set, kubeadm will automatically choose the kubeadm config version "+
 			"according to the Kubernetes version in use",
 	)
+	cmd.Flags().StringVar(
+		&flags.FeatureGates,
+		"kubeadm-feature-gates", flags.FeatureGates,
+		"the kubeadm feature-gates to be used for init, join and upgrade. ",
+	)
 	return cmd
 }
 
@@ -175,6 +181,7 @@ func runE(flags *flagpole, cmd *cobra.Command, args []string) (err error) {
 		actions.PatchesDir(flags.PatchesDir),
 		actions.IgnorePreflightErrors(flags.IgnorePreflightErrors),
 		actions.KubeadmConfigVersion(flags.KubeadmConfigVersion),
+		actions.FeatureGates(flags.FeatureGates),
 	)
 	if err != nil {
 		return errors.Wrapf(err, "failed to exec action %s", action)

diff --git a/kinder/pkg/cluster/manager/actions/actions.go b/kinder/pkg/cluster/manager/actions/actions.go
@@ -42,7 +42,7 @@ var actionRegistry = map[string]func(*status.Cluster, *RunOptions) error{
 		return KubeadmConfig(c, flags.kubeadmConfigVersion, flags.copyCertsMode, flags.discoveryMode, c.K8sNodes().EligibleForActions()...)
 	},
 	"kubeadm-init": func(c *status.Cluster, flags *RunOptions) error {
-		return KubeadmInit(c, flags.usePhases, flags.copyCertsMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.wait, flags.vLevel)
+		return KubeadmInit(c, flags.usePhases, flags.copyCertsMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.featureGates, flags.wait, flags.vLevel)
 	},
 	"kubeadm-join": func(c *status.Cluster, flags *RunOptions) error {
 		return KubeadmJoin(c, flags.usePhases, flags.copyCertsMode, flags.discoveryMode, flags.kubeadmConfigVersion, flags.patchesDir, flags.ignorePreflightErrors, flags.wait, flags.vLevel)
@@ -145,6 +145,13 @@ func KubeadmConfigVersion(kubeadmConfigVersion string) Option {
 	}
 }
 
+// FeatureGates option sets the kubeadm feature-gates for the kubeadm commands
+func FeatureGates(featureGates string) Option {
+	return func(r *RunOptions) {
+		r.featureGates = featureGates
+	}
+}
+
 // RunOptions holds options supplied to actions.Run
 type RunOptions struct {
 	usePhases             bool
@@ -156,6 +163,7 @@ type RunOptions struct {
 	patchesDir            string
 	ignorePreflightErrors string
 	kubeadmConfigVersion  string
+	featureGates          string
 }
 
 // DiscoveryMode defines discovery mode supported by kubeadm join

diff --git a/kinder/pkg/cluster/manager/actions/kubeadm-init.go b/kinder/pkg/cluster/manager/actions/kubeadm-init.go
@@ -37,7 +37,7 @@ import (
 
 // KubeadmInit executes the kubeadm init workflow including also post init task
 // like installing the CNI network plugin
-func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode, kubeadmConfigVersion, patchesDir, ignorePreflightErrors string, wait time.Duration, vLevel int) (err error) {
+func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode, kubeadmConfigVersion, patchesDir, ignorePreflightErrors, featureGates string, wait time.Duration, vLevel int) (err error) {
 	cp1 := c.BootstrapControlPlane()
 
 	// if patcheDir is defined, copy the patches to the node
@@ -72,9 +72,9 @@ func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode,
 
 	// execs the kubeadm init workflow
 	if usePhases {
-		err = kubeadmInitWithPhases(cp1, copyCertsMode, patchesDir, ignorePreflightErrors, vLevel)
+		err = kubeadmInitWithPhases(cp1, copyCertsMode, patchesDir, ignorePreflightErrors, featureGates, vLevel)
 	} else {
-		err = kubeadmInit(cp1, copyCertsMode, patchesDir, ignorePreflightErrors, vLevel)
+		err = kubeadmInit(cp1, copyCertsMode, patchesDir, ignorePreflightErrors, featureGates, vLevel)
 	}
 	if err != nil {
 		return err
@@ -88,12 +88,13 @@ func KubeadmInit(c *status.Cluster, usePhases bool, copyCertsMode CopyCertsMode,
 	return nil
 }
 
-func kubeadmInit(cp1 *status.Node, copyCertsMode CopyCertsMode, patchesDir, ignorePreflightErrors string, vLevel int) error {
+func kubeadmInit(cp1 *status.Node, copyCertsMode CopyCertsMode, patchesDir, ignorePreflightErrors, featureGates string, vLevel int) error {
 	initArgs := []string{
 		"init",
 		fmt.Sprintf("--ignore-preflight-errors=%s", ignorePreflightErrors),
 		fmt.Sprintf("--config=%s", constants.KubeadmConfigPath),
 		fmt.Sprintf("--v=%d", vLevel),
+		fmt.Sprintf("--feature-gates=%s", featureGates),
 	}
 	if copyCertsMode == CopyCertsModeAuto {
 		initArgs = append(initArgs,
@@ -114,7 +115,7 @@ func kubeadmInit(cp1 *status.Node, copyCertsMode CopyCertsMode, patchesDir, igno
 	return nil
 }
 
-func kubeadmInitWithPhases(cp1 *status.Node, copyCertsMode CopyCertsMode, patchesDir, ignorePreflightErrors string, vLevel int) error {
+func kubeadmInitWithPhases(cp1 *status.Node, copyCertsMode CopyCertsMode, patchesDir, ignorePreflightErrors, featureGates string, vLevel int) error {
 	if err := cp1.Command(
 		"kubeadm", "init", "phase", "preflight", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel),
 		fmt.Sprintf("--ignore-preflight-errors=%s", ignorePreflightErrors),
@@ -142,6 +143,7 @@ func kubeadmInitWithPhases(cp1 *status.Node, copyCertsMode CopyCertsMode, patche
 
 	controlplaneArgs := []string{
 		"init", "phase", "control-plane", "all", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel),
+		fmt.Sprintf("--feature-gates=%s", featureGates),
 	}
 	if patchesDir != "" {
 		controlplaneArgs = append(controlplaneArgs, "--experimental-patches", constants.PatchesDir)
@@ -153,7 +155,7 @@ func kubeadmInitWithPhases(cp1 *status.Node, copyCertsMode CopyCertsMode, patche
 	}
 
 	etcdArgs := []string{
-		"init", "phase", "etcd", "local", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel),
+		"init", "phase", "etcd", "local", fmt.Sprintf("--config=%s", constants.KubeadmConfigPath), fmt.Sprintf("--v=%d", vLevel), fmt.Sprintf("--feature-gates=%s", featureGates),
 	}
 	if patchesDir != "" {
 		etcdArgs = append(etcdArgs, "--experimental-patches", constants.PatchesDir)