Skip to content

Commit

Permalink
* E2E tests are now self contained -- use a local respository
Browse files Browse the repository at this point in the history 
instead of local overrides
* run-e2e.sh script updated to setup local repo in _artifacts dir
and also create a custom clusterctl.yaml
* Removed init test (merged with cluster create test)
* Created util funcs to init a mgmt cluster and create a workload
cluster
* Added new clusterctl delete tests
  • Loading branch information
@Arvinderpal
Arvinderpal committed Feb 11, 2020
1 parent d7b627b commit 5df11f0
Show file tree
Hide file tree
Showing 10 changed files with 841 additions and 329 deletions.
95 changes: 27 additions & 68 deletions cmd/clusterctl/test/e2e/config_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,100 +28,59 @@ import (
"github.com/pkg/errors"

v1 "k8s.io/api/core/v1"
clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client"
clusterctlrepo "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/repository"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/management/kind"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/kind/pkg/cluster"
)

var _ = Describe("clusterctl create cluster", func() {
var (
cfgFile string // default is $HOME/clusterctl.yaml
mgmtCluster *kind.Cluster
mgmtClient client.Client
workloadClusterName string
kubernetesVersion string
coreProvider string
bootstrapProviders []string
controlPlaneProviders []string
infrastructureProviders []string
tmpDir string
template clusterctlrepo.Template
mgmtInfo testMgmtClusterInfo
workloadInfo testWorkloadClusterInfo
)
BeforeEach(func() {
workloadClusterName = "e2e-workload-cluster"
kubernetesVersion = "1.14.2"
coreProvider = "cluster-api:v0.3.0"
bootstrapProviders = []string{"kubeadm-bootstrap:v0.3.0"}
controlPlaneProviders = []string{"kubeadm-control-plane:v0.3.0"}
infrastructureProviders = []string{"docker:v0.3.0"}
var err error

// Mgmt cluster info object
mgmtInfo = testMgmtClusterInfo{
clusterctlConfigFile: clusterctlConfigFile,
coreProvider: "cluster-api:v0.3.0",
bootstrapProviders: []string{"kubeadm-bootstrap:v0.3.0"},
controlPlaneProviders: []string{"kubeadm-control-plane:v0.3.0"},
infrastructureProviders: []string{"docker:v0.3.0"},
}
// Create the mgmt cluster and client
mgmtCluster, err = CreateKindCluster(kindConfigFile)
mgmtInfo.mgmtCluster, err = CreateKindCluster(kindConfigFile)
Expect(err).ToNot(HaveOccurred())
mgmtClient, err = mgmtCluster.GetClient()
mgmtInfo.mgmtClient, err = mgmtInfo.mgmtCluster.GetClient()
Expect(err).NotTo(HaveOccurred())

// Let's setup some varibles for the workload cluster template
os.Setenv("DOCKER_SERVICE_CIDRS", "\"10.96.0.0/12\"")
os.Setenv("DOCKER_POD_CIDRS", "\"192.168.0.0/16\"")
// Create clusterctl.yaml
tmpDir = createTempDir()
cfgFile = createLocalTestClusterCtlConfig(tmpDir, "clusterctl.yaml", "DOCKER_SERVICE_DOMAIN: \"docker.cluster.local\"")

c, err := clusterctlclient.New(cfgFile)
Expect(err).ToNot(HaveOccurred())
initOpt := clusterctlclient.InitOptions{
Kubeconfig: mgmtCluster.KubeconfigPath,
CoreProvider: coreProvider,
BootstrapProviders: bootstrapProviders,
ControlPlaneProviders: controlPlaneProviders,
InfrastructureProviders: infrastructureProviders,
}
_, err = c.Init(initOpt)
Expect(err).ToNot(HaveOccurred())
framework.WaitForAPIServiceAvailable(ctx, mgmtCluster, "v1beta1.webhook.cert-manager.io")
CheckAndWaitCAPITestDeploymentsExist(mgmtClient)
initTestMgmtCluster(ctx, mgmtInfo)

options := clusterctlclient.GetClusterTemplateOptions{
Kubeconfig: mgmtCluster.KubeconfigPath,
InfrastructureProvider: infrastructureProviders[0],
ClusterName: workloadClusterName,
Flavor: "",
// TargetNamespace: targetNamespace,
KubernetesVersion: kubernetesVersion,
ControlPlaneMachineCount: 1,
WorkerMachineCount: 0,
// Workload cluster info object
workloadInfo = testWorkloadClusterInfo{
workloadClusterName: "e2e-workload-cluster",
kubernetesVersion: "1.14.2",
controlPlaneMachineCount: 1,
workerMachineCount: 0,
}

template, err = c.GetClusterTemplate(options)
Expect(err).ToNot(HaveOccurred())
yaml, err := template.Yaml()
Expect(err).ToNot(HaveOccurred())
// Create our workload cluster
err = mgmtCluster.Apply(ctx, yaml)
Expect(err).ToNot(HaveOccurred())

// Let's setup some varibles for the workload cluster template
Expect(os.Setenv("DOCKER_SERVICE_CIDRS", "\"10.96.0.0/12\"")).To(Succeed())
Expect(os.Setenv("DOCKER_POD_CIDRS", "\"192.168.0.0/16\"")).To(Succeed())
createTestWorkloadCluster(ctx, mgmtInfo, workloadInfo)
})

AfterEach(func() {
fmt.Fprintf(GinkgoWriter, "Tearing down kind mgmt cluster\n")
mgmtCluster.Teardown(ctx)
mgmtInfo.mgmtCluster.Teardown(ctx)
fmt.Fprintf(GinkgoWriter, "Tearing down kind workload cluster\n")
if err := cluster.NewProvider().Delete(workloadClusterName, ""); err != nil {
if err := cluster.NewProvider().Delete(workloadInfo.workloadClusterName, ""); err != nil {
// Treat this as a non critical error
fmt.Fprintf(GinkgoWriter, "Deleting the kind cluster %q failed. You may need to remove this by hand.\n", workloadClusterName)
fmt.Fprintf(GinkgoWriter, "Deleting the kind cluster %q failed. You may need to remove this by hand.\n", workloadInfo.workloadClusterName)
}
os.RemoveAll(tmpDir)
})

Context("using specific core, control-plane, bootstrap, and capd provider version", func() {
It("should create a workload cluster", func() {
Eventually(func() ([]v1.Node, error) {
workloadClient, err := mgmtCluster.GetWorkloadClient(ctx, "default", workloadClusterName)
workloadClient, err := mgmtInfo.mgmtCluster.GetWorkloadClient(ctx, "default", workloadInfo.workloadClusterName)
if err != nil {
return nil, errors.Wrap(err, "failed to get workload client")
}
Expand Down
89 changes: 0 additions & 89 deletions cmd/clusterctl/test/e2e/constant.go
View file

This file was deleted.

91 changes: 57 additions & 34 deletions cmd/clusterctl/test/e2e/delete_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,76 +26,64 @@ import (
. "github.com/onsi/gomega"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/management/kind"
"k8s.io/apimachinery/pkg/api/meta"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/kind/pkg/cluster"

clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client"
infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha3"
)

var _ = Describe("clusterctl delete", func() {
var (
cfgFile string
mgmtCluster *kind.Cluster
mgmtClient client.Client
mgmtInfo testMgmtClusterInfo
deleteOptions clusterctlclient.DeleteOptions
cctlClient clusterctlclient.Client
)

BeforeEach(func() {
var err error
// Mgmt cluster info object
mgmtInfo = testMgmtClusterInfo{
clusterctlConfigFile: clusterctlConfigFile,
coreProvider: "cluster-api:v0.3.0",
bootstrapProviders: []string{"kubeadm-bootstrap:v0.3.0"},
controlPlaneProviders: []string{"kubeadm-control-plane:v0.3.0"},
infrastructureProviders: []string{"docker:v0.3.0"},
}
// Create the mgmt cluster and client
mgmtCluster, err = CreateKindCluster(kindConfigFile)
mgmtInfo.mgmtCluster, err = CreateKindCluster(kindConfigFile)
Expect(err).ToNot(HaveOccurred())
mgmtClient, err = mgmtCluster.GetClient()
mgmtInfo.mgmtClient, err = mgmtInfo.mgmtCluster.GetClient()
Expect(err).NotTo(HaveOccurred())
cctlClient, err = clusterctlclient.New(cfgFile)
Expect(err).ToNot(HaveOccurred())
initOptmgmtCluster := clusterctlclient.InitOptions{
Kubeconfig: mgmtCluster.KubeconfigPath,
CoreProvider: "cluster-api:v0.3.0",
BootstrapProviders: []string{"kubeadm-bootstrap:v0.3.0"},
ControlPlaneProviders: []string{"kubeadm-control-plane:v0.3.0"},
InfrastructureProviders: []string{"docker:v0.3.0"},
}
_, err = cctlClient.Init(initOptmgmtCluster)
Expect(err).ToNot(HaveOccurred())
framework.WaitForAPIServiceAvailable(ctx, mgmtCluster, "v1beta1.webhook.cert-manager.io")
CheckAndWaitCAPITestDeploymentsExist(mgmtClient)
// Create a capi Cluster and capd objects
err = mgmtCluster.Apply(ctx, []byte(testClusterYAML))
Expect(err).ToNot(HaveOccurred())

initTestMgmtCluster(ctx, mgmtInfo)

}, setupTimeout)

JustBeforeEach(func() {
err := cctlClient.Delete(deleteOptions)
c, err := clusterctlclient.New(mgmtInfo.clusterctlConfigFile)
Expect(err).ToNot(HaveOccurred())
err = c.Delete(deleteOptions)
Expect(err).ToNot(HaveOccurred())
})

AfterEach(func() {
fmt.Fprintf(GinkgoWriter, "Tearing down kind clusters\n")
mgmtCluster.Teardown(ctx)
fmt.Fprintf(GinkgoWriter, "Tearing down kind workload cluster\n")
if err := cluster.NewProvider().Delete("foo-cluster", ""); err != nil {
// Treat this as a non critical error
fmt.Fprintf(GinkgoWriter, "Deleting the kind cluster \"foo-cluster\" failed. You may need to remove this by hand.\n")
}
mgmtInfo.mgmtCluster.Teardown(ctx)
})

Context("deletes the infra provider", func() {
BeforeEach(func() {
deleteOptions = clusterctlclient.DeleteOptions{
Kubeconfig: mgmtCluster.KubeconfigPath,
Kubeconfig: mgmtInfo.mgmtCluster.KubeconfigPath,
Providers: []string{"docker"},
}
})
It("should delete of all infra provider components except the hosting namespace and the CRDs.", func() {
Eventually(
func() error {
err := mgmtClient.Get(ctx, client.ObjectKey{Namespace: "capd-system", Name: "capd-controller-manager"}, &appsv1.Deployment{})
err := mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Namespace: "capd-system", Name: "capd-controller-manager"}, &appsv1.Deployment{})
if err == nil || !apierrors.IsNotFound(err) {
// deployment still exists or some other error other than not found occured.
return fmt.Errorf("%v", err)
Expand All @@ -105,4 +93,39 @@ var _ = Describe("clusterctl delete", func() {
).ShouldNot(HaveOccurred())
})
})
Context("deletes everything", func() {
BeforeEach(func() {
deleteOptions = clusterctlclient.DeleteOptions{
Kubeconfig: mgmtInfo.mgmtCluster.KubeconfigPath,
ForceDeleteNamespace: true,
ForceDeleteCRD: true,
Providers: []string{},
}
})
It("should reset the management cluster to its original state", func() {
Eventually(
func() error {
// TODO: check all components are deleted.
err := mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Namespace: "capd-system", Name: "capd-controller-manager"}, &appsv1.Deployment{})
if err == nil || !apierrors.IsNotFound(err) {
return fmt.Errorf("%v", err)
}
// TODO: check namespace of all components are deleted.
err = mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Name: "capd-system"}, &corev1.Namespace{})
if err == nil || !apierrors.IsNotFound(err) {
return fmt.Errorf("%v", err)
}
// TODO: check that all CRDs are deleted.
err = mgmtInfo.mgmtClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "foo-cluster"}, &infrav1.DockerCluster{})
if err == nil {
return fmt.Errorf("%v", err)
}
if _, ok := err.(*meta.NoResourceMatchError); !ok {
return err
}
return nil
}, 3*time.Minute, 5*time.Second,
).ShouldNot(HaveOccurred())
})
})
})
13 changes: 10 additions & 3 deletions cmd/clusterctl/test/e2e/e2e_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ const (
)

var (
ctx context.Context
managerImage string
kindConfigFile string
ctx context.Context
managerImage string
kindConfigFile string
clusterctlConfigFile string
)

var _ = BeforeSuite(func() {
Expand All @@ -56,6 +57,12 @@ var _ = BeforeSuite(func() {
} else {
fmt.Fprintf(GinkgoWriter, "Using KIND_CONFIG_FILE: %v\n", kindConfigFile)
}
clusterctlConfigFile = os.Getenv("CLUSTERCTL_CONFIG")
if clusterctlConfigFile == "" {
fmt.Fprintf(GinkgoWriter, "CLUSTERCTL_CONFIG not found.\n")
} else {
fmt.Fprintf(GinkgoWriter, "Using CLUSTERCTL_CONFIG: %v\n", clusterctlConfigFile)
}
}, setupTimeout)

var _ = AfterSuite(func() {
Expand Down
Loading

0 comments on commit 5df11f0

Please sign in to comment.