From 967325070dad36cd555175c6689fa18993bb3107 Mon Sep 17 00:00:00 2001 From: Arvinderpal Wander Date: Tue, 4 Feb 2020 18:32:19 -0800 Subject: [PATCH] Added a test that creates a workload cluster. Incorporated local-overrides and kind config file into run-e2e.sh. Incorporated review feedback. --- .../test/e2e/config_cluster_test.go | 194 ++++++++++-------- cmd/clusterctl/test/e2e/e2e_suite_test.go | 12 +- cmd/clusterctl/test/e2e/go.mod | 2 + cmd/clusterctl/test/e2e/go.sum | 15 ++ cmd/clusterctl/test/e2e/helpers.go | 26 +-- cmd/clusterctl/test/e2e/init_test.go | 90 ++++---- cmd/clusterctl/test/run-e2e.sh | 27 ++- 7 files changed, 214 insertions(+), 152 deletions(-) diff --git a/cmd/clusterctl/test/e2e/config_cluster_test.go b/cmd/clusterctl/test/e2e/config_cluster_test.go index b10ce53618c3..076438d39ad4 100644 --- a/cmd/clusterctl/test/e2e/config_cluster_test.go +++ b/cmd/clusterctl/test/e2e/config_cluster_test.go @@ -21,110 +21,122 @@ package e2e import ( "fmt" "os" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client" clusterctlrepo "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/repository" + "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/management/kind" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kind/pkg/cluster" ) -var _ = Describe("clusterctl config cluster", func() { - Describe("basic", func() { - var ( - cfgFile string // default is $HOME/clusterctl.yaml - kindCluster *kind.Cluster - kindClient client.Client - clusterName string - kubernetesVersion string - coreProvider string - coreVersion string - bootstrapProvider string - bootstrapVersion string - infrastructureProvider string - infrastructureVersion string - tmpDir string - template clusterctlrepo.Template - ) - BeforeEach(func() { - clusterName = "e2e-workload-cluster" - kubernetesVersion = "1.14.2" - coreProvider = "cluster-api" - coreVersion = "v0.3.0" - bootstrapProvider = "kubeadm-bootstrap" - bootstrapVersion = "v0.3.0" - infrastructureProvider = "docker" - infrastructureVersion = "v0.3.0" - var err error - - // Create the mgmt cluster and client - kindCluster, kindClient, err = CreateKindClusterAndClient() - Expect(err).ToNot(HaveOccurred()) - - // Create clusterctl.yaml - tmpDir = createTempDir() - cfgFile = createLocalTestClusterCtlConfig(tmpDir, "clusterctl.yaml", "DOCKER_SERVICE_DOMAIN: \"docker.cluster.local\"") - // Let's setup some varibles for the workload cluster template - os.Setenv("DOCKER_SERVICE_CIDRS", "\"10.96.0.0/12\"") - os.Setenv("DOCKER_POD_CIDRS", "\"192.168.0.0/16\"") - - c, err := clusterctlclient.New(cfgFile) - Expect(err).ToNot(HaveOccurred()) - initOpt := clusterctlclient.InitOptions{ - Kubeconfig: kindCluster.KubeconfigPath, - CoreProvider: coreProvider + ":" + coreVersion, - BootstrapProviders: []string{bootstrapProvider + ":" + bootstrapVersion}, - InfrastructureProviders: []string{infrastructureProvider + ":" + infrastructureVersion}, - } - _, _, err = c.Init(initOpt) - Expect(err).ToNot(HaveOccurred()) - // Confirm controllers exists - CheckAndWaitDeploymentExists(kindClient, "capi-system", "capi-controller-manager") - CheckAndWaitDeploymentExists(kindClient, "capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager") - CheckAndWaitDeploymentExists(kindClient, "capd-system", "capd-controller-manager") - - options := clusterctlclient.GetClusterTemplateOptions{ - Kubeconfig: kindCluster.KubeconfigPath, - InfrastructureProvider: infrastructureProvider, - BootstrapProvider: bootstrapProvider, - ClusterName: clusterName, - Flavor: "", - // TargetNamespace: targetNamespace, - KubernetesVersion: kubernetesVersion, - ControlPlaneMachineCount: 1, - WorkerMachineCount: 0, - } - - template, err = c.GetClusterTemplate(options) - Expect(err).ToNot(HaveOccurred()) +var _ = Describe("clusterctl create cluster", func() { + var ( + cfgFile string // default is $HOME/clusterctl.yaml + mgmtCluster *kind.Cluster + mgmtClient client.Client + workloadClusterName string + kubernetesVersion string + coreProvider string + bootstrapProviders []string + controlPlaneProviders []string + infrastructureProviders []string + tmpDir string + template clusterctlrepo.Template + ) + BeforeEach(func() { + workloadClusterName = "e2e-workload-cluster" + kubernetesVersion = "1.14.2" + coreProvider = "cluster-api:v0.3.0" + bootstrapProviders = []string{"kubeadm-bootstrap:v0.3.0"} + controlPlaneProviders = []string{"kubeadm-control-plane:v0.3.0"} + infrastructureProviders = []string{"docker:v0.3.0"} + var err error + + // Create the mgmt cluster and client + mgmtCluster, err = CreateKindCluster(kindConfigFile) + Expect(err).ToNot(HaveOccurred()) + mgmtClient, err = mgmtCluster.GetClient() + Expect(err).NotTo(HaveOccurred()) + + // Let's setup some varibles for the workload cluster template + os.Setenv("DOCKER_SERVICE_CIDRS", "\"10.96.0.0/12\"") + os.Setenv("DOCKER_POD_CIDRS", "\"192.168.0.0/16\"") + // Create clusterctl.yaml + tmpDir = createTempDir() + cfgFile = createLocalTestClusterCtlConfig(tmpDir, "clusterctl.yaml", "DOCKER_SERVICE_DOMAIN: \"docker.cluster.local\"") + + c, err := clusterctlclient.New(cfgFile) + Expect(err).ToNot(HaveOccurred()) + initOpt := clusterctlclient.InitOptions{ + Kubeconfig: mgmtCluster.KubeconfigPath, + CoreProvider: coreProvider, + BootstrapProviders: bootstrapProviders, + ControlPlaneProviders: controlPlaneProviders, + InfrastructureProviders: infrastructureProviders, + } + _, _, err = c.Init(initOpt) + Expect(err).ToNot(HaveOccurred()) + framework.WaitForAPIServiceAvailable(ctx, mgmtCluster, "v1beta1.webhook.cert-manager.io") + // Confirm controllers exists + CheckAndWaitDeploymentExists(mgmtClient, "capi-system", "capi-controller-manager") + CheckAndWaitDeploymentExists(mgmtClient, "capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager") + CheckAndWaitDeploymentExists(mgmtClient, "capi-kubeadm-control-plane-system", "capi-kubeadm-control-plane-controller-manager") + CheckAndWaitDeploymentExists(mgmtClient, "capd-system", "capd-controller-manager") + + options := clusterctlclient.GetClusterTemplateOptions{ + Kubeconfig: mgmtCluster.KubeconfigPath, + InfrastructureProvider: infrastructureProviders[0], + ClusterName: workloadClusterName, + Flavor: "", + // TargetNamespace: targetNamespace, + KubernetesVersion: kubernetesVersion, + ControlPlaneMachineCount: 1, + WorkerMachineCount: 0, + } + + template, err = c.GetClusterTemplate(options) + Expect(err).ToNot(HaveOccurred()) + yaml, err := template.Yaml() + Expect(err).ToNot(HaveOccurred()) + // Create our workload cluster + err = mgmtCluster.Apply(ctx, yaml) + Expect(err).ToNot(HaveOccurred()) - }) + }) - AfterEach(func() { - fmt.Fprintf(GinkgoWriter, "Tearing down kind cluster\n") - kindCluster.Teardown(ctx) - os.RemoveAll(tmpDir) - }) + AfterEach(func() { + fmt.Fprintf(GinkgoWriter, "Tearing down kind mgmt cluster\n") + mgmtCluster.Teardown(ctx) + fmt.Fprintf(GinkgoWriter, "Tearing down kind workload cluster\n") + if err := cluster.NewProvider().Delete(workloadClusterName, ""); err != nil { + // Treat this as a non critical error + fmt.Fprintf(GinkgoWriter, "Deleting the kind cluster %q failed. You may need to remove this by hand.\n", workloadClusterName) + } + os.RemoveAll(tmpDir) + }) - Context("using default infra and bootstrap provider", func() { - It("should generate a yaml file for a workload cluster", func() { - provider := template.Name() - Expect(provider).To(Equal(infrastructureProvider)) - version := template.Version() - Expect(version).To(Equal(infrastructureVersion)) - bootstrap := template.Bootstrap() - Expect(bootstrap).To(Equal(bootstrapProvider)) - - variables := template.Variables() - fmt.Fprintf(GinkgoWriter, "%v", variables) - yaml, err := template.Yaml() - Expect(err).ToNot(HaveOccurred()) - yaml = append(yaml, '\n') - fmt.Fprintf(GinkgoWriter, "%v", string(yaml)) - - }) + Context("using specific core, control-plane, bootstrap, and capd provider version", func() { + It("should create a workload cluster", func() { + Eventually(func() ([]v1.Node, error) { + workloadClient, err := mgmtCluster.GetWorkloadClient(ctx, "default", workloadClusterName) + if err != nil { + return nil, errors.Wrap(err, "failed to get workload client") + } + nodeList := v1.NodeList{} + if err := workloadClient.List(ctx, &nodeList); err != nil { + return nil, err + } + return nodeList.Items, nil + }, 5*time.Minute, 10*time.Second).Should(HaveLen(2)) + + // TODO: Check config file and env variables defined above. }) }) }) diff --git a/cmd/clusterctl/test/e2e/e2e_suite_test.go b/cmd/clusterctl/test/e2e/e2e_suite_test.go index 81c0bf70f625..6af654f9148b 100644 --- a/cmd/clusterctl/test/e2e/e2e_suite_test.go +++ b/cmd/clusterctl/test/e2e/e2e_suite_test.go @@ -20,6 +20,7 @@ package e2e import ( "context" + "fmt" "os" "testing" @@ -37,8 +38,9 @@ const ( ) var ( - ctx context.Context - managerImage string + ctx context.Context + managerImage string + kindConfigFile string ) var _ = BeforeSuite(func() { @@ -48,6 +50,12 @@ var _ = BeforeSuite(func() { if managerImage == "" { managerImage = "gcr.io/k8s-staging-capi-docker/capd-manager-amd64:dev" } + kindConfigFile = os.Getenv("KIND_CONFIG_FILE") + if kindConfigFile == "" { + fmt.Fprintf(GinkgoWriter, "KIND_CONFIG_FILE not found, capd mgmt cluster wil be created without any special kind configuration.\n") + } else { + fmt.Fprintf(GinkgoWriter, "Using KIND_CONFIG_FILE: %v\n", kindConfigFile) + } }, setupTimeout) var _ = AfterSuite(func() { diff --git a/cmd/clusterctl/test/e2e/go.mod b/cmd/clusterctl/test/e2e/go.mod index 1c1490cd292d..f1128f996ea9 100644 --- a/cmd/clusterctl/test/e2e/go.mod +++ b/cmd/clusterctl/test/e2e/go.mod @@ -5,12 +5,14 @@ go 1.13 require ( github.com/onsi/ginkgo v1.11.0 github.com/onsi/gomega v1.8.1 + github.com/pkg/errors v0.9.0 k8s.io/api v0.0.0-20191121015604-11707872ac1c k8s.io/client-go v11.0.0+incompatible sigs.k8s.io/cluster-api v0.2.9 sigs.k8s.io/cluster-api/test/framework v0.0.0-20200125173702-54f26d7fd2b5 sigs.k8s.io/cluster-api/test/infrastructure/docker v0.0.0-20200125173702-54f26d7fd2b5 sigs.k8s.io/controller-runtime v0.4.0 + sigs.k8s.io/kind v0.7.0 ) replace ( diff --git a/cmd/clusterctl/test/e2e/go.sum b/cmd/clusterctl/test/e2e/go.sum index e944ad9a3cec..8705a78569a5 100644 --- a/cmd/clusterctl/test/e2e/go.sum +++ b/cmd/clusterctl/test/e2e/go.sum @@ -11,6 +11,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -22,6 +23,7 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI= github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -29,6 +31,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -71,6 +74,7 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= @@ -128,6 +132,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 h1:u4bArs140e9+AfE52mFHOXVFnOSBJBRlzTHrOPLOIhE= github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -212,6 +217,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -265,16 +271,22 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -381,6 +393,7 @@ golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= @@ -458,6 +471,7 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -524,6 +538,7 @@ sigs.k8s.io/cluster-api/test/infrastructure/docker v0.0.0-20200125173702-54f26d7 sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= sigs.k8s.io/controller-runtime v0.4.0 h1:wATM6/m+3w8lj8FXNaO6Fs/rq/vqoOjO1Q116Z9NPsg= sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= +sigs.k8s.io/kind v0.7.0 h1:7y7a8EYtGHM+auHmsvzuK5o84SrxPYGidlvfql7j/k4= sigs.k8s.io/kind v0.7.0/go.mod h1:An/AbWHT6pA/Lm0Og8j3ukGhfJP3RiVN/IBU6Lo3zl8= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= diff --git a/cmd/clusterctl/test/e2e/helpers.go b/cmd/clusterctl/test/e2e/helpers.go index 7f89e5b2a611..e2949c8fc3c7 100644 --- a/cmd/clusterctl/test/e2e/helpers.go +++ b/cmd/clusterctl/test/e2e/helpers.go @@ -30,31 +30,27 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - capiv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/management/kind" - infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha3" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" ) -func CreateKindClusterAndClient() (*kind.Cluster, client.Client, error) { - - Expect(corev1.AddToScheme(clientgoscheme.Scheme)).To(Succeed()) - Expect(capiv1.AddToScheme(clientgoscheme.Scheme)).To(Succeed()) - Expect(infrav1.AddToScheme(clientgoscheme.Scheme)).To(Succeed()) - Expect(clusterctlv1.AddToScheme(clientgoscheme.Scheme)).To(Succeed()) - +func CreateKindCluster(configFile string) (*kind.Cluster, error) { + var kindCluster *kind.Cluster + var err error + framework.TryAddDefaultSchemes(clientgoscheme.Scheme) // create kind cluster and client kindClusterName := "clusterctl-e2e-test-" + util.RandomString(6) - kindCluster, err := kind.NewCluster(ctx, kindClusterName, clientgoscheme.Scheme, managerImage) + if configFile == "" { + kindCluster, err = kind.NewCluster(ctx, kindClusterName, clientgoscheme.Scheme, managerImage) + } else { + kindCluster, err = kind.NewClusterWithConfig(ctx, kindClusterName, configFile, clientgoscheme.Scheme, managerImage) + } Expect(err).NotTo(HaveOccurred()) Expect(kindCluster).NotTo(BeNil()) - kindClient, err := kindCluster.GetClient() - Expect(err).NotTo(HaveOccurred()) - return kindCluster, kindClient, err + return kindCluster, err } func CheckAndWaitDeploymentExists(c client.Client, namespace, name string) { diff --git a/cmd/clusterctl/test/e2e/init_test.go b/cmd/clusterctl/test/e2e/init_test.go index 7ee0737ee667..be3a35c10377 100644 --- a/cmd/clusterctl/test/e2e/init_test.go +++ b/cmd/clusterctl/test/e2e/init_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/management/kind" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,51 +32,56 @@ import ( ) var _ = Describe("clusterctl init", func() { - Describe("basic", func() { - var ( - cfgFile string // default is $HOME/.clusterctl.yaml - kindCluster *kind.Cluster - kindClient client.Client - ) + var ( + cfgFile string // default is $HOME/.clusterctl.yaml + mgmtCluster *kind.Cluster + mgmtClient client.Client + ) - BeforeEach(func() { - var err error - kindCluster, kindClient, err = CreateKindClusterAndClient() - Expect(err).ToNot(HaveOccurred()) - c, err := clusterctlclient.New(cfgFile) - Expect(err).ToNot(HaveOccurred()) - initOpt := clusterctlclient.InitOptions{ - Kubeconfig: kindCluster.KubeconfigPath, - CoreProvider: "cluster-api:v0.3.0", - BootstrapProviders: []string{"kubeadm-bootstrap:v0.3.0"}, - InfrastructureProviders: []string{"docker:v0.3.0"}, - } - componentList, firstExecution, err := c.Init(initOpt) - Expect(err).ToNot(HaveOccurred()) - // TODO: remove below print statements - for _, components := range componentList { - fmt.Fprintf(GinkgoWriter, " - %s %s installed (%s)\n", components.Name(), components.Type(), components.Version()) - } - if firstExecution { - fmt.Fprintf(GinkgoWriter, "\nYour cluster API management cluster has been initialized successfully!") - fmt.Fprintf(GinkgoWriter, "\nYou can now create your first workload cluster by running the following:") - fmt.Fprintf(GinkgoWriter, "\n clusterctl config cluster [name] --kubernetes-version [version] | kubectl apply -f -") - fmt.Fprintf(GinkgoWriter, "\n") - } - }, setupTimeout) + BeforeEach(func() { + var err error + // Create the mgmt cluster and client + mgmtCluster, err = CreateKindCluster(kindConfigFile) + Expect(err).ToNot(HaveOccurred()) + mgmtClient, err = mgmtCluster.GetClient() + Expect(err).NotTo(HaveOccurred()) - AfterEach(func() { - fmt.Fprintf(GinkgoWriter, "Tearing down kind cluster\n") - kindCluster.Teardown(ctx) - }) + c, err := clusterctlclient.New(cfgFile) + Expect(err).ToNot(HaveOccurred()) + initOpt := clusterctlclient.InitOptions{ + Kubeconfig: mgmtCluster.KubeconfigPath, + CoreProvider: "cluster-api:v0.3.0", + BootstrapProviders: []string{"kubeadm-bootstrap:v0.3.0"}, + ControlPlaneProviders: []string{"kubeadm-control-plane:v0.3.0"}, + InfrastructureProviders: []string{"docker:v0.3.0"}, + } + componentList, firstExecution, err := c.Init(initOpt) + Expect(err).ToNot(HaveOccurred()) + // TODO: remove below print statements + for _, components := range componentList { + fmt.Fprintf(GinkgoWriter, " - %s %s installed (%s)\n", components.Name(), components.Type(), components.Version()) + } + if firstExecution { + fmt.Fprintf(GinkgoWriter, "\nYour cluster API management cluster has been initialized successfully!") + fmt.Fprintf(GinkgoWriter, "\nYou can now create your first workload cluster by running the following:") + fmt.Fprintf(GinkgoWriter, "\n clusterctl config cluster [name] --kubernetes-version [version] | kubectl apply -f -") + fmt.Fprintf(GinkgoWriter, "\n") + } + }, setupTimeout) + + AfterEach(func() { + fmt.Fprintf(GinkgoWriter, "Tearing down kind cluster\n") + mgmtCluster.Teardown(ctx) + }) - Context("mgmt cluster", func() { - It("should install core, bootstrap and capd components and CRDs", func() { - // Confirm controllers exists - CheckAndWaitDeploymentExists(kindClient, "capi-system", "capi-controller-manager") - CheckAndWaitDeploymentExists(kindClient, "capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager") - CheckAndWaitDeploymentExists(kindClient, "capd-system", "capd-controller-manager") - }) + Context("mgmt cluster", func() { + It("should install core, bootstrap and capd components and CRDs", func() { + framework.WaitForAPIServiceAvailable(ctx, mgmtCluster, "v1beta1.webhook.cert-manager.io") + // Confirm controllers exists + CheckAndWaitDeploymentExists(mgmtClient, "capi-system", "capi-controller-manager") + CheckAndWaitDeploymentExists(mgmtClient, "capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager") + CheckAndWaitDeploymentExists(mgmtClient, "capi-kubeadm-control-plane-system", "capi-kubeadm-control-plane-controller-manager") + CheckAndWaitDeploymentExists(mgmtClient, "capd-system", "capd-controller-manager") }) }) }) diff --git a/cmd/clusterctl/test/run-e2e.sh b/cmd/clusterctl/test/run-e2e.sh index 4c1014d8fe7d..b58f260b5722 100755 --- a/cmd/clusterctl/test/run-e2e.sh +++ b/cmd/clusterctl/test/run-e2e.sh @@ -24,18 +24,41 @@ REPO_ROOT_ABS=${PWD} ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" mkdir -p "$ARTIFACTS/logs/" -cd "${REPO_ROOT_ABS}/test/infrastructure/docker" +cat < "clusterctl-settings.json" +{ + "providers": [ "cluster-api", "kubeadm-bootstrap", "kubeadm-control-plane", "docker"] +} +EOF + +./cmd/clusterctl/hack/local-overrides.py + +export KIND_CONFIG_FILE="${ARTIFACTS}/kind-cluster-with-extramounts.yaml" +cat < "${KIND_CONFIG_FILE}" +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + extraMounts: + - hostPath: /var/lib/docker + containerPath: /var/lib/docker + - hostPath: /var/run/docker.sock + containerPath: /var/run/docker.sock +EOF + export REGISTRY=gcr.io/"$(gcloud config get-value project)" export DOCKER_MANAGER_IMAGE=docker-provider-manager export TAG=dev export ARCH=amd64 -# Update the images used in the e2e tests. + # This will load the capd image into the kind mgmt cluster. export MANAGER_IMAGE=${REGISTRY}/${DOCKER_MANAGER_IMAGE}-${ARCH}:${TAG} export PULL_POLICY=IfNotPresent + if [ -z "${SKIP_DOCKER_BUILD-}" ]; then + cd "${REPO_ROOT_ABS}/test/infrastructure/docker" CONTROLLER_IMG=${REGISTRY}/${DOCKER_MANAGER_IMAGE} make docker-build fi + GINKGO_FOCUS=${GINKGO_FOCUS:-""} cd "${REPO_ROOT_ABS}/cmd/clusterctl/test/e2e"