Skip to content

Commit

Permalink
use kustomize for e2e templates
Browse files Browse the repository at this point in the history
  • Loading branch information
fabriziopandini committed Sep 16, 2020
1 parent 624ddb0 commit 8cdceba
Show file tree
Hide file tree
Showing 20 changed files with 615 additions and 469 deletions.
4 changes: 2 additions & 2 deletions scripts/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ source "${REPO_ROOT}/hack/ensure-kustomize.sh"
# Configure provider images generation;
# please ensure the generated image name matches image names used in the E2E_CONF_FILE
export REGISTRY=gcr.io/k8s-staging-cluster-api
export TAG=ci
export TAG=dev
export ARCH=amd64
export PULL_POLICY=IfNotPresent

Expand All @@ -54,7 +54,7 @@ docker pull kindest/node:v1.17.2
export GINKGO_NODES=3
export GINKGO_NOCOLOR=true
export GINKGO_ARGS="--failFast" # Other ginkgo args that need to be appended to the command.
export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/docker-ci.yaml"
export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/docker.yaml"
export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
export SKIP_RESOURCE_CLEANUP=false
export USE_EXISTING_CLUSTER=false
Expand Down
31 changes: 23 additions & 8 deletions test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export GOPROXY
REPO_ROOT := $(shell git rev-parse --show-toplevel)

help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

## --------------------------------------
## Binaries
Expand All @@ -38,28 +38,43 @@ TOOLS_DIR := $(REPO_ROOT)/hack/tools
BIN_DIR := bin
TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
GINKGO := $(TOOLS_BIN_DIR)/ginkgo
KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize

.PHONY: ginkgo
ginkgo:
$(GINKGO): # Build ginkgo from tools folder.
cd $(TOOLS_DIR) && go build -tags=tools -o $(BIN_DIR)/ginkgo github.com/onsi/ginkgo/ginkgo

$(KUSTOMIZE): # Build kustomize from tools folder.
cd $(TOOLS_DIR) && go build -tags=tools -o $(BIN_DIR)/kustomize sigs.k8s.io/kustomize/kustomize/v3

## --------------------------------------
## Testing
## Templates
## --------------------------------------

TEST_E2E_DIR := $(REPO_ROOT)/test/e2e
DOCKER_TEMPLATES := $(REPO_ROOT)/test/e2e/data/infrastructure-docker

.PHONY: cluster-templates
cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template --load_restrictor none > $(DOCKER_TEMPLATES)/cluster-template.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-mhc --load_restrictor none > $(DOCKER_TEMPLATES)/cluster-template-mhc.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption/step1 --load_restrictor none > $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption.yaml
echo "---" >> $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption/step2 --load_restrictor none >> $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption.yaml

## --------------------------------------
## Testing
## --------------------------------------

GINKGO_FOCUS ?=
GINKGO_NODES ?= 1
E2E_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker-dev.yaml
E2E_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker.yaml
ARTIFACTS ?= ${REPO_ROOT}/_artifacts
SKIP_RESOURCE_CLEANUP ?= false
USE_EXISTING_CLUSTER ?= false
GINKGO_NOCOLOR ?= false

.PHONY: run
run: ginkgo ## Run the end-to-end tests
cd $(TEST_E2E_DIR); $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) . -- \
run: $(GINKGO) cluster-templates ## Run the end-to-end tests
$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) . -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE)" \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
123 changes: 0 additions & 123 deletions test/e2e/config/docker-dev.yaml

This file was deleted.

16 changes: 7 additions & 9 deletions test/e2e/config/docker-ci.yaml → test/e2e/config/docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@
# - control-plane kubeadm
# - docker

# For creating local dev images run ./scripts/ci-e2e.sh
# For creating local dev images run make docker-build-e2e from the main CAPI repository

images:
# Use local dev images built source tree;
- name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci
- name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:dev
loadBehavior: mustLoad
- name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:ci
- name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:dev
loadBehavior: mustLoad
- name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:ci
- name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:dev
loadBehavior: mustLoad
- name: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:ci
- name: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev
loadBehavior: mustLoad
- name: quay.io/jetstack/cert-manager-cainjector:v0.16.1
loadBehavior: tryLoad
Expand Down Expand Up @@ -66,11 +66,9 @@ providers:
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
# Add a metadata for docker provider
- sourcePath: "../data/infrastructure-docker/metadata.yaml"
# Add cluster templates
- sourcePath: "../data/infrastructure-docker/cluster-template-ci.yaml"
targetName: "cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/cluster-template-mhc.yaml"
- sourcePath: "../data/infrastructure-docker/cluster-template-kcp-adoption.yaml"

variables:
Expand Down
1 change: 0 additions & 1 deletion test/e2e/data/cni/kindnet/kindnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,4 +110,3 @@ spec:
- name: lib-modules
hostPath:
path: /lib/modules
---
70 changes: 70 additions & 0 deletions test/e2e/data/infrastructure-docker/bases/cluster-with-kcp.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
---
# DockerCluster object referenced by the Cluster object
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerCluster
metadata:
name: '${CLUSTER_NAME}'
---
# Cluster object with
# - Reference to the KubeadmControlPlane object
# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
apiVersion: cluster.x-k8s.io/v1alpha3
kind: Cluster
metadata:
name: '${CLUSTER_NAME}'
labels:
cni: "${CLUSTER_NAME}-crs-0"
spec:
clusterNetwork:
services:
cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
pods:
cidrBlocks: ['${DOCKER_POD_CIDRS}']
serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
infrastructureRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerCluster
name: '${CLUSTER_NAME}'
controlPlaneRef:
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
name: "${CLUSTER_NAME}-control-plane"
---
# DockerMachineTemplate object referenced by the KubeadmControlPlane object
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-control-plane"
spec:
template:
spec: {}
---
# KubeadmControlPlane referenced by the Cluster object with
# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
kind: KubeadmControlPlane
apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
metadata:
name: "${CLUSTER_NAME}-control-plane"
labels:
kcp-adoption.step2: ""
spec:
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
infrastructureTemplate:
kind: DockerMachineTemplate
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
name: "${CLUSTER_NAME}-control-plane"
kubeadmConfigSpec:
clusterConfiguration:
controllerManager:
extraArgs: {enable-hostpath-provisioner: 'true'}
apiServer:
certSANs: [localhost, 127.0.0.1]
initConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
joinConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
version: "${KUBERNETES_VERSION}"
24 changes: 24 additions & 0 deletions test/e2e/data/infrastructure-docker/bases/crs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
# ConfigMap object referenced by the ClusterResourceSet object and with
# the CNI resource defined in the test config file
apiVersion: v1
kind: ConfigMap
metadata:
name: "cni-${CLUSTER_NAME}-crs-0"
data: ${CNI_RESOURCES}
binaryData:
---
# ClusterResourceSet object with
# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
apiVersion: addons.cluster.x-k8s.io/v1alpha3
kind: ClusterResourceSet
metadata:
name: "${CLUSTER_NAME}-crs-0"
spec:
strategy: ApplyOnce
clusterSelector:
matchLabels:
cni: "${CLUSTER_NAME}-crs-0"
resources:
- name: "cni-${CLUSTER_NAME}-crs-0"
kind: ConfigMap
54 changes: 54 additions & 0 deletions test/e2e/data/infrastructure-docker/bases/md.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
---
# DockerMachineTemplate referenced by the MachineDeployment and with
# - extraMounts for the docker sock, thus allowing self-hosting test
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerMachineTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
extraMounts:
- containerPath: "/var/run/docker.sock"
hostPath: "/var/run/docker.sock"
---
# KubeadmConfigTemplate referenced by the MachineDeployment
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
kind: KubeadmConfigTemplate
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
template:
spec:
joinConfiguration:
nodeRegistration:
criSocket: /var/run/containerd/containerd.sock
kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
---
# MachineDeployment object with
# - the label nodepool=pool1 that applies to all the machines, so those machine can be targeted by the MachineHealthCheck object
apiVersion: cluster.x-k8s.io/v1alpha3
kind: MachineDeployment
metadata:
name: "${CLUSTER_NAME}-md-0"
spec:
clusterName: "${CLUSTER_NAME}"
replicas: ${WORKER_MACHINE_COUNT}
selector:
matchLabels:
template:
metadata:
labels:
"nodepool": "pool1"
spec:
clusterName: "${CLUSTER_NAME}"
version: "${KUBERNETES_VERSION}"
bootstrap:
configRef:
name: "${CLUSTER_NAME}-md-0"
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
kind: KubeadmConfigTemplate
infrastructureRef:
name: "${CLUSTER_NAME}-md-0"
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerMachineTemplate
Loading

0 comments on commit 8cdceba

Please sign in to comment.