diff --git a/test/framework/bootstrap/interfaces.go b/test/framework/bootstrap/interfaces.go new file mode 100644 index 000000000000..d621cad63f73 --- /dev/null +++ b/test/framework/bootstrap/interfaces.go @@ -0,0 +1,35 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import "context" + +// ClusterProvider defines the behavior of a type that is responsible for provisioning and managing a Kubernetes cluster. +type ClusterProvider interface { + // Create a Kubernetes cluster. + // Generally to be used in the BeforeSuite function to create a Kubernetes cluster to be shared between tests. + Create(context.Context) + + // GetKubeconfigPath returns the path to the kubeconfig file to be used to access the Kubernetes cluster. + GetKubeconfigPath() string + + // Dispose will completely clean up the provisioned cluster. + // This should be implemented as a synchronous function. + // Generally to be used in the AfterSuite function if a Kubernetes cluster is shared between tests. + // Should try to clean everything up and report any dangling artifacts that needs manual intervention. + Dispose(context.Context) +} diff --git a/test/framework/bootstrap/kind_provider.go b/test/framework/bootstrap/kind_provider.go new file mode 100644 index 000000000000..90c282d4530b --- /dev/null +++ b/test/framework/bootstrap/kind_provider.go @@ -0,0 +1,138 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "fmt" + "io/ioutil" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + kindv1 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" + kind "sigs.k8s.io/kind/pkg/cluster" +) + +// KindClusterOption is a NewKindClusterProvider option +type KindClusterOption interface { + apply(*kindClusterProvider) +} + +type kindClusterOptionAdapter func(*kindClusterProvider) + +func (adapter kindClusterOptionAdapter) apply(kindClusterProvider *kindClusterProvider) { + adapter(kindClusterProvider) +} + +// WithDockerSockMount implements a New Option that instruct the kindClusterProvider to mount /var/run/docker.sock into +// the new kind cluster. +func WithDockerSockMount() KindClusterOption { + return kindClusterOptionAdapter(func(k *kindClusterProvider) { + k.withDockerSock = true + }) +} + +// NewKindClusterProvider returns a ClusterProvider that can create a kind cluster. +func NewKindClusterProvider(name string, options ...KindClusterOption) *kindClusterProvider { + Expect(name).ToNot(BeEmpty(), "name is required for NewKindClusterProvider") + + clusterProvider := &kindClusterProvider{ + name: name, + } + for _, option := range options { + option.apply(clusterProvider) + } + return clusterProvider +} + +// kindClusterProvider implements a ClusterProvider that can create a kind cluster. +type kindClusterProvider struct { + name string + withDockerSock bool + kubeconfigPath string +} + +// Create a Kubernetes cluster using kind. +func (k *kindClusterProvider) Create(ctx context.Context) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Create") + + // Sets the kubeconfig path to a temp file. + // NB. the ClusterProvider is responsible for the cleanup of this file + f, err := ioutil.TempFile("", "e2e-kind") + Expect(err).ToNot(HaveOccurred(), "Failed to create kubeconfig file for the kind cluster %q", k.name) + k.kubeconfigPath = f.Name() + + // Creates the kind cluster + k.createKindCluster() +} + +// createKindCluster calls the kind library taking care of passing options for: +// - use a dedicated kubeconfig file (test should not alter the user environment) +// - if required, mount /var/run/docker.sock +func (k *kindClusterProvider) createKindCluster() { + kindCreateOptions := []kind.CreateOption{ + kind.CreateWithKubeconfigPath(k.kubeconfigPath), + } + if k.withDockerSock { + kindCreateOptions = append(kindCreateOptions, kind.CreateWithV1Alpha4Config(withDockerSockConfig())) + } + + err := kind.NewProvider().Create(k.name, kindCreateOptions...) + Expect(err).ToNot(HaveOccurred(), "Failed to create the kind cluster %q") +} + +// withDockerSockConfig returns a kind config for mounting /var/run/docker.sock into the kind node. +func withDockerSockConfig() *kindv1.Cluster { + cfg := &kindv1.Cluster{ + TypeMeta: kindv1.TypeMeta{ + APIVersion: "kind.x-k8s.io/v1alpha4", + Kind: "Cluster", + }, + } + kindv1.SetDefaultsCluster(cfg) + cfg.Nodes = []kindv1.Node{ + { + Role: kindv1.ControlPlaneRole, + ExtraMounts: []kindv1.Mount{ + { + HostPath: "/var/run/docker.sock", + ContainerPath: "/var/run/docker.sock", + }, + }, + }, + } + return cfg +} + +// GetKubeconfigPath returns the path to the kubeconfig file for the cluster. +func (k *kindClusterProvider) GetKubeconfigPath() string { + return k.kubeconfigPath +} + +// Dispose the kind cluster and its kubeconfig file. +func (k *kindClusterProvider) Dispose(ctx context.Context) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Dispose") + + if err := kind.NewProvider().Delete(k.name, k.kubeconfigPath); err != nil { + fmt.Fprintf(GinkgoWriter, "Deleting the kind cluster %q failed. You may need to remove this by hand.\n", k.name) + } + if err := os.Remove(k.kubeconfigPath); err != nil { + fmt.Fprintf(GinkgoWriter, "Deleting the kubeconfig file %q file. You may need to remove this by hand.\n", k.kubeconfigPath) + } +} diff --git a/test/framework/bootstrap/kind_util.go b/test/framework/bootstrap/kind_util.go new file mode 100644 index 000000000000..5a098da416f9 --- /dev/null +++ b/test/framework/bootstrap/kind_util.go @@ -0,0 +1,152 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/exec" + kind "sigs.k8s.io/kind/pkg/cluster" + kindnodes "sigs.k8s.io/kind/pkg/cluster/nodes" + kindnodesutils "sigs.k8s.io/kind/pkg/cluster/nodeutils" +) + +// CreateKindBootstrapClusterAndLoadImagesInput is the input for CreateKindBootstrapClusterAndLoadImages. +type CreateKindBootstrapClusterAndLoadImagesInput struct { + // Name of the cluster + Name string + + // RequiresDockerSock defines if the cluster requires the docker sock + RequiresDockerSock bool + + // Images to be loaded in the cluster (this is kind specific) + Images []framework.ContainerImage +} + +// CreateKindBootstrapClusterAndLoadImages returns a new Kubernetes cluster with pre-loaded images. +func CreateKindBootstrapClusterAndLoadImages(ctx context.Context, input CreateKindBootstrapClusterAndLoadImagesInput) ClusterProvider { + Expect(ctx).NotTo(BeNil(), "ctx is required for CreateKindBootstrapClusterAndLoadImages") + Expect(input.Name).ToNot(BeEmpty(), "Invalid argument. Name can't be empty when calling CreateKindBootstrapClusterAndLoadImages") + + fmt.Fprintf(GinkgoWriter, "Creating a kind cluster with name %q\n", input.Name) + + options := []KindClusterOption{} + if input.RequiresDockerSock { + options = append(options, WithDockerSockMount()) + } + clusterProvider := NewKindClusterProvider(input.Name, options...) + Expect(clusterProvider).ToNot(BeNil(), "Failed to create a kind cluster") + + clusterProvider.Create(ctx) + Expect(clusterProvider.GetKubeconfigPath()).To(BeAnExistingFile(), "The kubeconfig file for the kind cluster with name %q does not exists at %q as expected", input.Name, clusterProvider.GetKubeconfigPath()) + + LoadImagesToKindCluster(ctx, LoadImagesToKindClusterInput{ + Name: input.Name, + Images: input.Images, + }) + + return clusterProvider +} + +// LoadImagesToKindClusterInput is the input for LoadImagesToKindCluster. +type LoadImagesToKindClusterInput struct { + // Name of the cluster + Name string + + // Images to be loaded in the cluster (this is kind specific) + Images []framework.ContainerImage +} + +// LoadImagesToKindCluster provides a utility for loading images into a kind cluster. +func LoadImagesToKindCluster(ctx context.Context, input LoadImagesToKindClusterInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for LoadImagesToKindCluster") + Expect(input.Name).ToNot(BeEmpty(), "Invalid argument. Name can't be empty when calling LoadImagesToKindCluster") + + for _, image := range input.Images { + fmt.Fprintf(GinkgoWriter, "Loading image: %q\n", image.Name) + err := loadImage(ctx, input.Name, image.Name) + switch image.LoadBehavior { + case framework.MustLoadImage: + Expect(err).ToNot(HaveOccurred(), "Failed to load image %q into the kind cluster %q", image.Name, input.Name) + case framework.TryLoadImage: + if err != nil { + fmt.Fprintf(GinkgoWriter, "[WARNING] Unable to load image %q into the kind cluster %q: %v \n", image.Name, input.Name, err) + } + } + } +} + +// LoadImage will put a local image onto the kind node +func loadImage(ctx context.Context, cluster, image string) error { + // Save the image into a tar + dir, err := ioutil.TempDir("", "image-tar") + if err != nil { + return errors.Wrap(err, "failed to create tempdir") + } + defer os.RemoveAll(dir) + imageTarPath := filepath.Join(dir, "image.tar") + + err = save(ctx, image, imageTarPath) + if err != nil { + return err + } + + // Gets the nodes in the cluster + provider := kind.NewProvider() + nodeList, err := provider.ListInternalNodes(cluster) + if err != nil { + return err + } + + // Load the image on the selected nodes + for _, node := range nodeList { + if err := load(imageTarPath, node); err != nil { + return err + } + } + + return nil +} + +// copied from kind https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/cmd/kind/load/docker-image/docker-image.go#L168 +// save saves image to dest, as in `docker save` +func save(ctx context.Context, image, dest string) error { + _, _, err := exec.NewCommand( + exec.WithCommand("docker"), + exec.WithArgs("save", "-o", dest, image)).Run(ctx) + return err +} + +// copied from kind https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/cmd/kind/load/docker-image/docker-image.go#L158 +// loads an image tarball onto a node +func load(imageTarName string, node kindnodes.Node) error { + f, err := os.Open(imageTarName) + if err != nil { + return errors.Wrap(err, "failed to open image") + } + defer f.Close() + return kindnodesutils.LoadImageArchive(node, f) +} diff --git a/test/framework/cluster_proxy.go b/test/framework/cluster_proxy.go new file mode 100644 index 000000000000..1c278eda7a84 --- /dev/null +++ b/test/framework/cluster_proxy.go @@ -0,0 +1,252 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "fmt" + "io/ioutil" + "net/url" + "os" + goruntime "runtime" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/framework/exec" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ClusterProxy defines the behavior of a type that acts as an intermediary with an existing Kubernetes cluster. +// It should work with any Kubernetes cluster, no matter if the Cluster was created by a bootstrap.ClusterProvider, +// by Cluster API (a workload cluster or a self-hosted cluster) or else. +type ClusterProxy interface { + // GetName returns the name of the cluster. + GetName() string + + // GetKubeconfigPath returns the path to the kubeconfig file to be used to access the Kubernetes cluster. + GetKubeconfigPath() string + + // GetScheme returns the scheme defining the types hosted in the Kubernetes cluster. + // It is used when creating a controller-runtime client. + GetScheme() *runtime.Scheme + + // GetClient returns a controller-runtime client to the Kubernetes cluster. + GetClient() client.Client + + // GetClientSet returns a client-go client to the Kubernetes cluster. + GetClientSet() *kubernetes.Clientset + + // Apply to apply YAML to the Kubernetes cluster, `kubectl apply`. + Apply(context.Context, []byte) error + + // GetWorkloadCluster returns a proxy to a workload cluster defined in the Kubernetes cluster. + GetWorkloadCluster(ctx context.Context, namespace, name string) ClusterProxy + + // Dispose proxy's internal resources (the operation does not affects the Kubernetes cluster). + // This should be implemented as a synchronous function. + Dispose(context.Context) +} + +// clusterProxy provides a base implementation of the ClusterProxy interface. +type clusterProxy struct { + name string + kubeconfigPath string + scheme *runtime.Scheme + shouldCleanupKubeconfig bool +} + +// NewClusterProxy returns a clusterProxy given a KubeconfigPath and the scheme defining the types hosted in the cluster. +// If a kubeconfig file isn't provided, standard kubeconfig locations will be used (kubectl loading rules apply). +func NewClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme) ClusterProxy { + Expect(scheme).NotTo(BeNil(), "scheme is required for NewClusterProxy") + + if kubeconfigPath == "" { + kubeconfigPath = clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename() + } + return &clusterProxy{ + name: name, + kubeconfigPath: kubeconfigPath, + scheme: scheme, + shouldCleanupKubeconfig: false, + } +} + +// newFromAPIConfig returns a clusterProxy given a api.Config and the scheme defining the types hosted in the cluster. +func newFromAPIConfig(name string, config *api.Config, scheme *runtime.Scheme) ClusterProxy { + // NB. the ClusterProvider is responsible for the cleanup of this file + f, err := ioutil.TempFile("", "e2e-kubeconfig") + Expect(err).ToNot(HaveOccurred(), "Failed to create kubeconfig file for the kind cluster %q") + kubeconfigPath := f.Name() + + err = clientcmd.WriteToFile(*config, kubeconfigPath) + Expect(err).ToNot(HaveOccurred(), "Failed to write kubeconfig for the kind cluster to a file %q") + + return &clusterProxy{ + name: name, + kubeconfigPath: kubeconfigPath, + scheme: scheme, + shouldCleanupKubeconfig: true, + } +} + +// GetName returns the name of the cluster. +func (p *clusterProxy) GetName() string { + return p.name +} + +// GetKubeconfigPath returns the path to the kubeconfig file for the cluster. +func (p *clusterProxy) GetKubeconfigPath() string { + return p.kubeconfigPath +} + +// GetScheme returns the scheme defining the types hosted in the cluster. +func (p *clusterProxy) GetScheme() *runtime.Scheme { + return p.scheme +} + +// GetClient returns a controller-runtime client for the cluster. +func (p *clusterProxy) GetClient() client.Client { + config := p.getConfig() + + c, err := client.New(config, client.Options{Scheme: p.scheme}) + Expect(err).ToNot(HaveOccurred(), "Failed to get controller-runtime client") + + return c +} + +// GetClientSet returns a client-go client for the cluster. +func (p *clusterProxy) GetClientSet() *kubernetes.Clientset { + restConfig := p.getConfig() + + cs, err := kubernetes.NewForConfig(restConfig) + Expect(err).ToNot(HaveOccurred(), "Failed to get client-go client") + + return cs +} + +// Apply wraps `kubectl apply` and prints the output so we can see what gets applied to the cluster. +func (p *clusterProxy) Apply(ctx context.Context, resources []byte) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for Apply") + Expect(resources).NotTo(BeNil(), "resources is required for Apply") + + return exec.KubectlApply(ctx, p.kubeconfigPath, resources) +} + +func (p *clusterProxy) getConfig() *rest.Config { + config, err := clientcmd.LoadFromFile(p.kubeconfigPath) + Expect(err).ToNot(HaveOccurred(), "Failed to load Kubeconfig file from %q", p.kubeconfigPath) + + restConfig, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig() + Expect(err).ToNot(HaveOccurred(), "Failed to get ClientConfig from %q", p.kubeconfigPath) + + restConfig.UserAgent = "cluster-api-e2e" + return restConfig +} + +// GetWorkloadCluster returns ClusterProxy for the workload cluster. +func (p *clusterProxy) GetWorkloadCluster(ctx context.Context, namespace, name string) ClusterProxy { + Expect(ctx).NotTo(BeNil(), "ctx is required for GetWorkloadCluster") + Expect(namespace).NotTo(BeEmpty(), "namespace is required for GetWorkloadCluster") + Expect(name).NotTo(BeEmpty(), "name is required for GetWorkloadCluster") + + // gets the kubeconfig from the cluster + config := p.getKubeconfig(ctx, namespace, name) + + // if we are on mac and the cluster is a DockerCluster, it is required to fix the master address + // by using localhost:load-balancer-host-port instead of the address used in the docker network. + if goruntime.GOOS == "darwin" && p.isDockerCluster(ctx, namespace, name) { + p.fixConfig(ctx, name, config) + } + + return newFromAPIConfig(name, config, p.scheme) +} + +func (p *clusterProxy) getKubeconfig(ctx context.Context, namespace string, name string) *api.Config { + cl := p.GetClient() + + secret := &corev1.Secret{} + key := client.ObjectKey{ + Name: fmt.Sprintf("%s-kubeconfig", name), + Namespace: namespace, + } + Expect(cl.Get(ctx, key, secret)).To(Succeed(), "Failed to get %s", key) + Expect(secret.Data).To(HaveKey("value"), "Invalid secret %s", key) + + config, err := clientcmd.Load(secret.Data["value"]) + Expect(err).ToNot(HaveOccurred(), "Failed to convert %s into a kubeconfig file", key) + + return config +} + +func (p *clusterProxy) isDockerCluster(ctx context.Context, namespace string, name string) bool { + cl := p.GetClient() + + cluster := &clusterv1.Cluster{} + key := client.ObjectKey{ + Name: name, + Namespace: namespace, + } + Expect(cl.Get(ctx, key, cluster)).To(Succeed(), "Failed to get %s", key) + + return cluster.Spec.InfrastructureRef.Kind == "DockerCluster" +} + +func (p *clusterProxy) fixConfig(ctx context.Context, name string, config *api.Config) { + port, err := findLoadBalancerPort(ctx, name) + Expect(err).ToNot(HaveOccurred(), "Failed to get load balancer port") + + masterURL := &url.URL{ + Scheme: "https", + Host: "127.0.0.1:" + port, + } + currentCluster := config.Contexts[config.CurrentContext].Cluster + config.Clusters[currentCluster].Server = masterURL.String() +} + +func findLoadBalancerPort(ctx context.Context, name string) (string, error) { + loadBalancerName := name + "-lb" + portFormat := `{{index (index (index .NetworkSettings.Ports "6443/tcp") 0) "HostPort"}}` + getPathCmd := exec.NewCommand( + exec.WithCommand("docker"), + exec.WithArgs("inspect", loadBalancerName, "--format", portFormat), + ) + stdout, _, err := getPathCmd.Run(ctx) + if err != nil { + return "", err + } + return strings.TrimSpace(string(stdout)), nil +} + +// Dispose clusterProxy internal resources (the operation does not affects the Kubernetes cluster). +func (p *clusterProxy) Dispose(ctx context.Context) { + Expect(ctx).NotTo(BeNil(), "ctx is required for Dispose") + + if p.shouldCleanupKubeconfig { + if err := os.Remove(p.kubeconfigPath); err != nil { + fmt.Fprintf(GinkgoWriter, "Deleting the kubeconfig file %q file. You may need to remove this by hand.\n", p.kubeconfigPath) + } + } +} diff --git a/test/framework/clusterctl/management_cluster.go b/test/framework/clusterctl/management_cluster.go deleted file mode 100644 index 4c79bf08b92f..000000000000 --- a/test/framework/clusterctl/management_cluster.go +++ /dev/null @@ -1,194 +0,0 @@ -// +build e2e - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterctl - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clusterctlconfig "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" - "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/discovery" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Provides utilities for setting up a management cluster using clusterctl. - -// InitManagementClusterInput is the information required to initialize a new -// management cluster for e2e testing. -type InitManagementClusterInput struct { - // E2EConfig defining the configuration for the E2E test. - E2EConfig *E2EConfig - - // ClusterctlConfigPath is the path to a clusterctl config file that points to repositories to be used during "clusterctl init". - ClusterctlConfigPath string - - // LogsFolder defines a folder where to store clusterctl logs. - LogsFolder string - - // Scheme is used to initialize the scheme for the management cluster client. - Scheme *runtime.Scheme - - // NewManagementClusterFn should return a new management cluster. - NewManagementClusterFn func(name string, scheme *runtime.Scheme) (cluster framework.ManagementCluster, kubeConfigPath string, err error) -} - -// InitManagementCluster returns a new cluster initialized and the path to the kubeConfig file to be used to access it. -func InitManagementCluster(ctx context.Context, input *InitManagementClusterInput) (framework.ManagementCluster, string) { - // validate parameters and apply defaults - - Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling InitManagementCluster") - Expect(input.NewManagementClusterFn).ToNot(BeNil(), "Invalid argument. input.NewManagementClusterFn can't be nil when calling InitManagementCluster") - Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file") - - By(fmt.Sprintf("Creating the management cluster with name %s", input.E2EConfig.ManagementClusterName)) - - managementCluster, managementClusterKubeConfigPath, err := input.NewManagementClusterFn(input.E2EConfig.ManagementClusterName, input.Scheme) - Expect(err).ToNot(HaveOccurred(), "Failed to create the management cluster with name %s", input.E2EConfig.ManagementClusterName) - Expect(managementCluster).ToNot(BeNil(), "The management cluster with name %s should not be nil", input.E2EConfig.ManagementClusterName) - - // Load the images into the cluster. - if imageLoader, ok := managementCluster.(framework.ImageLoader); ok { - By("Loading images into the management cluster") - - for _, image := range input.E2EConfig.Images { - err := imageLoader.LoadImage(ctx, image.Name) - switch image.LoadBehavior { - case framework.MustLoadImage: - Expect(err).ToNot(HaveOccurred(), "Failed to load image %s into the kind cluster", image.Name) - case framework.TryLoadImage: - if err != nil { - fmt.Fprintf(GinkgoWriter, "[WARNING] Unable to load image %s into the kind cluster: %v \n", image.Name, err) - } - } - } - } - - By("Running clusterctl init") - - Init(ctx, InitInput{ - // pass reference to the management cluster hosting this test - KubeconfigPath: managementClusterKubeConfigPath, - // pass the clusterctl config file that points to the local provider repository created for this test, - ClusterctlConfigPath: input.ClusterctlConfigPath, - // setup the desired list of providers for a single-tenant management cluster - CoreProvider: clusterctlconfig.ClusterAPIProviderName, - BootstrapProviders: []string{clusterctlconfig.KubeadmBootstrapProviderName}, - ControlPlaneProviders: []string{clusterctlconfig.KubeadmControlPlaneProviderName}, - InfrastructureProviders: []string{input.E2EConfig.InfraProvider()}, - // setup output path for clusterctl logs - LogPath: input.LogsFolder, - }) - - By("Waiting for providers controllers to be running") - - client, err := managementCluster.GetClient() - Expect(err).NotTo(HaveOccurred()) - controllersDeployments := discovery.GetControllerDeployments(ctx, discovery.GetControllerDeploymentsInput{ - Lister: client, - }) - Expect(controllersDeployments).ToNot(BeNil()) - for _, deployment := range controllersDeployments { - framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ - Getter: client, - Deployment: deployment, - }, input.E2EConfig.IntervalsOrDefault("init-management-cluster/wait-controllers", "2m", "10s")...) - - // Start streaming logs from all controller providers - watchLogs(ctx, managementCluster, deployment.Namespace, deployment.Name, input.LogsFolder) - } - - return managementCluster, managementClusterKubeConfigPath -} - -// watchLogs streams logs for all containers for all pods belonging to a deployment. Each container's logs are streamed -// in a separate goroutine so they can all be streamed concurrently. This only causes a test failure if there are errors -// retrieving the deployment, its pods, or setting up a log file. If there is an error with the log streaming itself, -// that does not cause the test to fail. -func watchLogs(ctx context.Context, mgmt framework.ManagementCluster, namespace, deploymentName, logDir string) error { - c, err := mgmt.GetClient() - if err != nil { - return err - } - clientSet, err := mgmt.GetClientSet() - if err != nil { - return err - } - - deployment := &appsv1.Deployment{} - Expect(c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: deploymentName}, deployment)).To(Succeed()) - - selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector) - Expect(err).NotTo(HaveOccurred()) - - pods := &corev1.PodList{} - Expect(c.List(ctx, pods, client.InNamespace(namespace), client.MatchingLabels(selector))).To(Succeed()) - - for _, pod := range pods.Items { - for _, container := range deployment.Spec.Template.Spec.Containers { - // Watch each container's logs in a goroutine so we can stream them all concurrently. - go func(pod corev1.Pod, container corev1.Container) { - defer GinkgoRecover() - - logFile := path.Join(logDir, deploymentName, pod.Name, container.Name+".log") - fmt.Fprintf(GinkgoWriter, "Creating directory: %s\n", filepath.Dir(logFile)) - Expect(os.MkdirAll(filepath.Dir(logFile), 0755)).To(Succeed()) - - fmt.Fprintf(GinkgoWriter, "Creating file: %s\n", logFile) - f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - Expect(err).NotTo(HaveOccurred()) - defer f.Close() - - opts := &corev1.PodLogOptions{ - Container: container.Name, - Follow: true, - } - - podLogs, err := clientSet.CoreV1().Pods(namespace).GetLogs(pod.Name, opts).Stream() - if err != nil { - // Failing to stream logs should not cause the test to fail - fmt.Fprintf(GinkgoWriter, "Error starting logs stream for pod %s/%s, container %s: %v\n", namespace, pod.Name, container.Name, err) - return - } - defer podLogs.Close() - - out := bufio.NewWriter(f) - defer out.Flush() - _, err = out.ReadFrom(podLogs) - if err != nil && err != io.ErrUnexpectedEOF { - // Failing to stream logs should not cause the test to fail - fmt.Fprintf(GinkgoWriter, "Got error while streaming logs for pod %s/%s, container %s: %v\n", namespace, pod.Name, container.Name, err) - } - }(pod, container) - } - } - return nil -} diff --git a/test/framework/interfaces.go b/test/framework/interfaces.go index 4edcec8bc2e9..74c6c393e6e5 100644 --- a/test/framework/interfaces.go +++ b/test/framework/interfaces.go @@ -33,24 +33,28 @@ type ComponentGenerator interface { } // Applier is an interface around applying YAML to a cluster +// Deprecated. Please use ClusterProxy type Applier interface { // Apply allows us to apply YAML to the cluster, `kubectl apply` Apply(context.Context, []byte) error } // Waiter is an interface around waiting for something on a kubernetes cluster. +// Deprecated. Please use ClusterProxy type Waiter interface { // Wait allows us to wait for something in the cluster, `kubectl wait` Wait(context.Context, ...string) error } // ImageLoader is an interface around loading an image onto a cluster. +// Deprecated. Please use ClusterProxy type ImageLoader interface { // LoadImage will put a local image onto the cluster. LoadImage(context.Context, string) error } // ManagementCluster are all the features we need out of a kubernetes cluster to qualify as a management cluster. +// Deprecated. Please use ClusterProxy type ManagementCluster interface { Applier Waiter diff --git a/test/framework/management/kind/mgmt.go b/test/framework/management/kind/mgmt.go index 3126d21b8b44..7ca6e1c5d52e 100644 --- a/test/framework/management/kind/mgmt.go +++ b/test/framework/management/kind/mgmt.go @@ -44,6 +44,7 @@ import ( // Shells out to `kind`, `kubectl` // Cluster represents a Kubernetes cluster used as a management cluster backed by kind. +// Deprecated. Please use bootstrap.ClusterProvider and ClusterProxy type Cluster struct { Name string KubeconfigPath string @@ -55,11 +56,13 @@ type Cluster struct { } // NewCluster sets up a new kind cluster to be used as the management cluster. +// Deprecated. Please use bootstrap.ClusterProvider and ClusterProxy func NewCluster(ctx context.Context, name string, scheme *runtime.Scheme, images ...string) (*Cluster, error) { return create(ctx, name, "", scheme, images...) } // NewClusterWithConfig creates a kind cluster using a kind-config file. +// Deprecated. Please use bootstrap.ClusterProvider and ClusterProxy func NewClusterWithConfig(ctx context.Context, name, configFile string, scheme *runtime.Scheme, images ...string) (*Cluster, error) { return create(ctx, name, configFile, scheme, images...) } diff --git a/test/framework/management_cluster.go b/test/framework/management_cluster.go index 7d3f25420ebe..e31e2bf94973 100644 --- a/test/framework/management_cluster.go +++ b/test/framework/management_cluster.go @@ -17,8 +17,10 @@ limitations under the License. package framework import ( + "bufio" "context" "fmt" + "io" "os" "path" "path/filepath" @@ -77,6 +79,7 @@ func (c *InitManagementClusterInput) Defaults(ctx context.Context) { // InitManagementCluster returns a new cluster initialized as a CAPI management // cluster. +// Deprecated. Please use bootstrap.ClusterProvider and ClusterProxy func InitManagementCluster(ctx context.Context, input *InitManagementClusterInput) ManagementCluster { By("initializing the management cluster") Expect(input).ToNot(BeNil()) @@ -168,6 +171,75 @@ func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAv }, intervals...).Should(BeTrue(), "Deployment %s/%s failed to get status.Available = True condition", input.Deployment.GetNamespace(), input.Deployment.GetName()) } +// WatchControllerLogsInput is the input for WatchControllerLogs. +type WatchControllerLogsInput struct { + GetLister GetLister + ClientSet *kubernetes.Clientset + Deployment *appsv1.Deployment + LogPath string +} + +// WatchControllerLogs streams logs for all containers for all pods belonging to a deployment. Each container's logs are streamed +// in a separate goroutine so they can all be streamed concurrently. This only causes a test failure if there are errors +// retrieving the deployment, its pods, or setting up a log file. If there is an error with the log streaming itself, +// that does not cause the test to fail. +func WatchControllerLogs(ctx context.Context, input WatchControllerLogsInput) error { + Expect(ctx).NotTo(BeNil(), "ctx is required for WatchControllerLogs") + Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for WatchControllerLogs") + Expect(input.Deployment).NotTo(BeNil(), "input.Name is required for WatchControllerLogs") + + deployment := &appsv1.Deployment{} + key, err := client.ObjectKeyFromObject(input.Deployment) + Expect(err).NotTo(HaveOccurred(), "Failed to get key for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + Expect(input.GetLister.Get(ctx, key, deployment)).To(Succeed(), "Failed to get deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector) + Expect(err).NotTo(HaveOccurred(), "Failed to Pods selector for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + pods := &corev1.PodList{} + Expect(input.GetLister.List(ctx, pods, client.InNamespace(input.Deployment.Namespace), client.MatchingLabels(selector))).To(Succeed(), "Failed to list Pods for deployment %s/%s", input.Deployment.Namespace, input.Deployment.Name) + + for _, pod := range pods.Items { + for _, container := range deployment.Spec.Template.Spec.Containers { + fmt.Fprintf(GinkgoWriter, "Creating log watcher for controller %s/%s, pod %s, container %s\n", input.Deployment.Namespace, input.Deployment.Name, pod.Name, container.Name) + + // Watch each container's logs in a goroutine so we can stream them all concurrently. + go func(pod corev1.Pod, container corev1.Container) { + defer GinkgoRecover() + + logFile := path.Join(input.LogPath, input.Deployment.Name, pod.Name, container.Name+".log") + Expect(os.MkdirAll(filepath.Dir(logFile), 0755)).To(Succeed()) + + f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + Expect(err).NotTo(HaveOccurred()) + defer f.Close() + + opts := &corev1.PodLogOptions{ + Container: container.Name, + Follow: true, + } + + podLogs, err := input.ClientSet.CoreV1().Pods(input.Deployment.Namespace).GetLogs(pod.Name, opts).Stream() + if err != nil { + // Failing to stream logs should not cause the test to fail + fmt.Fprintf(GinkgoWriter, "Error starting logs stream for pod %s/%s, container %s: %v\n", input.Deployment.Namespace, pod.Name, container.Name, err) + return + } + defer podLogs.Close() + + out := bufio.NewWriter(f) + defer out.Flush() + _, err = out.ReadFrom(podLogs) + if err != nil && err != io.ErrUnexpectedEOF { + // Failing to stream logs should not cause the test to fail + fmt.Fprintf(GinkgoWriter, "Got error while streaming logs for pod %s/%s, container %s: %v\n", input.Deployment.Namespace, pod.Name, container.Name, err) + } + }(pod, container) + } + } + return nil +} + // CreateNamespaceInput is the input type for CreateNamespace. type CreateNamespaceInput struct { Creator Creator @@ -188,7 +260,7 @@ func CreateNamespace(ctx context.Context, input CreateNamespaceInput, intervals Name: input.Name, }, } - By(fmt.Sprintf("Creating namespace %s", input.Name)) + fmt.Fprintf(GinkgoWriter, "Creating namespace %s\n", input.Name) Eventually(func() error { return input.Creator.Create(context.TODO(), ns) }, intervals...).Should(Succeed()) @@ -212,7 +284,7 @@ func DeleteNamespace(ctx context.Context, input DeleteNamespaceInput, intervals Name: input.Name, }, } - By(fmt.Sprintf("Deleting namespace %s", input.Name)) + fmt.Fprintf(GinkgoWriter, "Deleting namespace %s\n", input.Name) Eventually(func() error { return input.Deleter.Delete(context.TODO(), ns) }, intervals...).Should(Succeed()) @@ -222,7 +294,7 @@ func DeleteNamespace(ctx context.Context, input DeleteNamespaceInput, intervals type WatchNamespaceEventsInput struct { ClientSet *kubernetes.Clientset Name string - LogPath string + LogFolder string } // WatchNamespaceEvents creates a watcher that streams namespace events into a file. @@ -233,7 +305,7 @@ type WatchNamespaceEventsInput struct { // framework.WatchNamespaceEvents(ctx, framework.WatchNamespaceEventsInput{ // ClientSet: clientSet, // Name: namespace.Name, -// LogPath: logPath, +// LogFolder: logFolder, // }) // }() // defer cancelWatches() @@ -242,8 +314,7 @@ func WatchNamespaceEvents(ctx context.Context, input WatchNamespaceEventsInput) Expect(input.ClientSet).NotTo(BeNil(), "input.ClientSet is required for WatchNamespaceEvents") Expect(input.Name).NotTo(BeEmpty(), "input.Name is required for WatchNamespaceEvents") - logFile := path.Join(input.LogPath, "resources", input.Name, "events.log") - fmt.Fprintf(GinkgoWriter, "Creating directory: %s\n", filepath.Dir(logFile)) + logFile := path.Join(input.LogFolder, "resources", input.Name, "events.log") Expect(os.MkdirAll(filepath.Dir(logFile), 0755)).To(Succeed()) f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)