From 2d60f27e9670d5096b7d81d46bfeb4a6621c79ae Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Tue, 11 Feb 2020 18:32:53 +0100 Subject: [PATCH] clusterctl read templates from different sources --- cmd/clusterctl/cmd/config_cluster.go | 92 ++++- cmd/clusterctl/pkg/client/client.go | 30 -- cmd/clusterctl/pkg/client/client_test.go | 4 + cmd/clusterctl/pkg/client/cluster/client.go | 9 +- cmd/clusterctl/pkg/client/cluster/template.go | 200 +++++++++ .../pkg/client/cluster/template_test.go | 379 ++++++++++++++++++ cmd/clusterctl/pkg/client/config.go | 174 +++++++- .../pkg/client/config/variables_client.go | 5 + cmd/clusterctl/pkg/client/config_test.go | 109 ++++- .../client/repository/repository_github.go | 4 +- .../repository/repository_github_test.go | 36 +- .../pkg/client/repository/template.go | 8 +- .../pkg/client/repository/template_client.go | 6 +- .../client/repository/template_client_test.go | 64 ++- .../pkg/client/repository/template_test.go | 22 +- .../pkg/internal/test/fake_github.go | 56 +++ 16 files changed, 1077 insertions(+), 121 deletions(-) create mode 100644 cmd/clusterctl/pkg/client/cluster/template.go create mode 100644 cmd/clusterctl/pkg/client/cluster/template_test.go create mode 100644 cmd/clusterctl/pkg/internal/test/fake_github.go diff --git a/cmd/clusterctl/cmd/config_cluster.go b/cmd/clusterctl/cmd/config_cluster.go index c012dc3bc879..77f673345639 100644 --- a/cmd/clusterctl/cmd/config_cluster.go +++ b/cmd/clusterctl/cmd/config_cluster.go @@ -17,6 +17,7 @@ limitations under the License. package cmd import ( + "fmt" "os" "github.com/pkg/errors" @@ -33,6 +34,13 @@ type configClusterOptions struct { kubernetesVersion string controlPlaneMachineCount int workerMachineCount int + + url string + configMapNamespace string + configMapName string + configMapDataKey string + + listVariables bool } var cc = &configClusterOptions{} @@ -65,50 +73,113 @@ var configClusterClusterCmd = &cobra.Command{ # Generates a yaml file for creating a Cluster API workload cluster with # custom number of nodes (if supported by provider's templates) - clusterctl config cluster my-cluster --control-plane-machine-count=3 --worker-machine-count=10`), + clusterctl config cluster my-cluster --control-plane-machine-count=3 --worker-machine-count=10 + + # Generates a yaml file for creating a Cluster API workload cluster using a template hosted on a ConfigMap + # instead of using the cluster templates hosted in the provider's repository. + clusterctl config cluster my-cluster --from-config-map MyTemplates + + # Generates a yaml file for creating a Cluster API workload cluster using a template hosted on specific URL + # instead of using the cluster templates hosted in the provider's repository. + clusterctl config cluster my-cluster --from https://github.com/foo-org/foo-repository/blob/master/cluster-template.yaml + + # Generates a yaml file for creating a Cluster API workload cluster using a template hosted on the local file system + clusterctl config cluster my-cluster --from ~/workspace/cluster-template.yaml`), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runGenerateCluster(args[0]) + return runGetClusterTemplate(args[0]) }, } func init() { configClusterClusterCmd.Flags().StringVarP(&cc.kubeconfig, "kubeconfig", "", "", "Path to the kubeconfig file to use for accessing the management cluster. If empty, default rules for kubeconfig discovery will be used") - configClusterClusterCmd.Flags().StringVarP(&cc.infrastructureProvider, "infrastructure", "i", "", "The infrastructure provider that should be used for creating the workload cluster") - - configClusterClusterCmd.Flags().StringVarP(&cc.flavor, "flavor", "f", "", "The template variant to be used for creating the workload cluster") + // flags for the template variables configClusterClusterCmd.Flags().StringVarP(&cc.targetNamespace, "target-namespace", "n", "", "The namespace where the objects describing the workload cluster should be deployed. If not specified, the current namespace will be used") configClusterClusterCmd.Flags().StringVarP(&cc.kubernetesVersion, "kubernetes-version", "", "", "The Kubernetes version to use for the workload cluster. By default (empty), the value from os env variables or the .cluster-api/clusterctl.yaml config file will be used") configClusterClusterCmd.Flags().IntVarP(&cc.controlPlaneMachineCount, "control-plane-machine-count", "", 1, "The number of control plane machines to be added to the workload cluster.") configClusterClusterCmd.Flags().IntVarP(&cc.workerMachineCount, "worker-machine-count", "", 0, "The number of worker machines to be added to the workload cluster.") + // flags for the repository source + configClusterClusterCmd.Flags().StringVarP(&cc.infrastructureProvider, "infrastructure", "i", "", "The infrastructure provider to read the workload cluster template from. By default (empty), the default infrastructure provider will be used if no other source is specified") + configClusterClusterCmd.Flags().StringVarP(&cc.flavor, "flavor", "f", "", "The workload cluster template variant to be used when reading from the infrastructure provider repository. By default (empty), the default cluster template will be used") + + // flags for the url source + configClusterClusterCmd.Flags().StringVarP(&cc.url, "from", "", "", "The URL to read the workload cluster template from. By default (empty), the infrastructure provider repository URL will be used") + + // flags for the config map source + configClusterClusterCmd.Flags().StringVarP(&cc.configMapName, "from-config-map", "", "", "The ConfigMap to read the workload cluster template from. This can be used as alternative to read from the provider repository or from an URL") + configClusterClusterCmd.Flags().StringVarP(&cc.configMapNamespace, "from-config-map-namespace", "", "", "The namespace where the ConfigMap exists. By default (empty), the current namespace will be used") + configClusterClusterCmd.Flags().StringVarP(&cc.configMapDataKey, "from-config-map-key", "", "", fmt.Sprintf("The ConfigMap.Data key where the workload cluster template is hosted. By default (empty), %q will be used", client.DefaultCustomTemplateConfigMapKey)) + + // other flags + configClusterClusterCmd.Flags().BoolVarP(&cc.listVariables, "list-variables", "", false, "Returns the list of variables expected by the template instead of the template yaml") + configCmd.AddCommand(configClusterClusterCmd) } -func runGenerateCluster(name string) error { +func runGetClusterTemplate(name string) error { c, err := client.New(cfgFile) if err != nil { return err } - options := client.GetClusterTemplateOptions{ + templateOptions := client.GetClusterTemplateOptions{ Kubeconfig: cc.kubeconfig, - InfrastructureProvider: cc.infrastructureProvider, - Flavor: cc.flavor, ClusterName: name, TargetNamespace: cc.targetNamespace, KubernetesVersion: cc.kubernetesVersion, ControlPlaneMachineCount: cc.controlPlaneMachineCount, WorkerMachineCount: cc.workerMachineCount, + ListVariablesOnly: cc.listVariables, } - template, err := c.GetClusterTemplate(options) + if cc.url != "" { + templateOptions.URLSource = &client.URLSourceOptions{ + URL: cc.url, + } + } + + if cc.configMapNamespace != "" || cc.configMapName != "" || cc.configMapDataKey != "" { + templateOptions.ConfigMapSource = &client.ConfigMapSourceOptions{ + Namespace: cc.configMapNamespace, + Name: cc.configMapName, + DataKey: cc.configMapDataKey, + } + } + + if cc.infrastructureProvider != "" || cc.flavor != "" { + templateOptions.ProviderRepositorySource = &client.ProviderRepositorySourceOptions{ + InfrastructureProvider: cc.infrastructureProvider, + Flavor: cc.flavor, + } + } + + template, err := c.GetClusterTemplate(templateOptions) if err != nil { return err } + if cc.listVariables { + return templateListVariablesOutput(template) + } + + return templateYAMLOutput(template) +} + +func templateListVariablesOutput(template client.Template) error { + if len(template.Variables()) > 0 { + fmt.Println("Variables:") + for _, v := range template.Variables() { + fmt.Printf(" - %s\n", v) + } + } + fmt.Println() + return nil +} + +func templateYAMLOutput(template client.Template) error { yaml, err := template.Yaml() if err != nil { return err @@ -118,6 +189,5 @@ func runGenerateCluster(name string) error { if _, err := os.Stdout.Write(yaml); err != nil { return errors.Wrap(err, "failed to write yaml to Stdout") } - return nil } diff --git a/cmd/clusterctl/pkg/client/client.go b/cmd/clusterctl/pkg/client/client.go index 26be6da5db7f..3471e8643765 100644 --- a/cmd/clusterctl/pkg/client/client.go +++ b/cmd/clusterctl/pkg/client/client.go @@ -55,36 +55,6 @@ type InitOptions struct { LogUsageInstructions bool } -// GetClusterTemplateOptions carries the options supported by GetClusterTemplate. -type GetClusterTemplateOptions struct { - // Kubeconfig file to use for accessing the management cluster. If empty, default rules for kubeconfig - // discovery will be used. - Kubeconfig string - - // InfrastructureProvider that should be used for creating the workload cluster. - InfrastructureProvider string - - // Flavor defines the template variant to be used for creating the workload cluster. - Flavor string - - // TargetNamespace where the objects describing the workload cluster should be deployed. If not specified, - // the current namespace will be used. - TargetNamespace string - - // ClusterName to be used for the workload cluster. - ClusterName string - - // KubernetesVersion to use for the workload cluster. By default (empty), the value from os env variables - // or the .cluster-api/clusterctl.yaml config file will be used. - KubernetesVersion string - - // ControlPlaneMachineCount defines the number of control plane machines to be added to the workload cluster. - ControlPlaneMachineCount int - - // WorkerMachineCount defines number of worker machines to be added to the workload cluster. - WorkerMachineCount int -} - // DeleteOptions carries the options supported by Delete. type DeleteOptions struct { // Kubeconfig file to use for accessing the management cluster. If empty, default rules for kubeconfig diff --git a/cmd/clusterctl/pkg/client/client_test.go b/cmd/clusterctl/pkg/client/client_test.go index 9146c2155c3f..8bf823c2423d 100644 --- a/cmd/clusterctl/pkg/client/client_test.go +++ b/cmd/clusterctl/pkg/client/client_test.go @@ -223,6 +223,10 @@ func (f *fakeClusterClient) ProviderUpgrader() cluster.ProviderUpgrader { return f.internalclient.ProviderUpgrader() } +func (f *fakeClusterClient) Template() cluster.TemplateClient { + return f.internalclient.Template() +} + func (f *fakeClusterClient) WithObjs(objs ...runtime.Object) *fakeClusterClient { f.fakeProxy.WithObjs(objs...) return f diff --git a/cmd/clusterctl/pkg/client/cluster/client.go b/cmd/clusterctl/pkg/client/cluster/client.go index 66bb300e794a..2ea1a5369010 100644 --- a/cmd/clusterctl/pkg/client/cluster/client.go +++ b/cmd/clusterctl/pkg/client/cluster/client.go @@ -66,6 +66,9 @@ type Client interface { // ProviderUpgrader returns a ProviderUpgrader that supports upgrading Cluster API providers. ProviderUpgrader() ProviderUpgrader + + // Template has methods to work with templates stored in the cluster. + Template() TemplateClient } // PollImmediateWaiter tries a condition func until it returns true, an error, or the timeout is reached. @@ -73,9 +76,9 @@ type PollImmediateWaiter func(interval, timeout time.Duration, condition wait.Co // clusterClient implements Client. type clusterClient struct { + configClient config.Client kubeconfig string proxy Proxy - configClient config.Client repositoryClientFactory RepositoryClientFactory pollImmediateWaiter PollImmediateWaiter } @@ -117,6 +120,10 @@ func (c *clusterClient) ProviderUpgrader() ProviderUpgrader { return newProviderUpgrader(c.configClient, c.repositoryClientFactory, c.ProviderInventory(), c.ProviderComponents()) } +func (c *clusterClient) Template() TemplateClient { + return newTemplateClient(c.proxy, c.configClient) +} + // Option is a configuration option supplied to New type Option func(*clusterClient) diff --git a/cmd/clusterctl/pkg/client/cluster/template.go b/cmd/clusterctl/pkg/client/cluster/template.go new file mode 100644 index 000000000000..16d7ff87e69e --- /dev/null +++ b/cmd/clusterctl/pkg/client/cluster/template.go @@ -0,0 +1,200 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "context" + "encoding/base64" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + + "github.com/google/go-github/github" + "github.com/pkg/errors" + "golang.org/x/oauth2" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/config" + "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/repository" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TemplateClient has methods to work with templates stored in the cluster/out of the provider repository. +type TemplateClient interface { + // GetFromConfigMap returns a workload cluster template from the given ConfigMap. + GetFromConfigMap(namespace, name, dataKey, targetNamespace string, listVariablesOnly bool) (repository.Template, error) + + // GetFromURL returns a workload cluster template from the given URL. + GetFromURL(templateURL, targetNamespace string, listVariablesOnly bool) (repository.Template, error) +} + +// templateClient implements TemplateClient. +type templateClient struct { + proxy Proxy + configClient config.Client + gitHubClientFactory func(configVariablesClient config.VariablesClient) (*github.Client, error) +} + +// ensure templateClient implements TemplateClient. +var _ TemplateClient = &templateClient{} + +// newTemplateClient returns a templateClient. +func newTemplateClient(proxy Proxy, configClient config.Client) *templateClient { + return &templateClient{ + proxy: proxy, + configClient: configClient, + gitHubClientFactory: getGitHubClient, + } +} + +func (t *templateClient) GetFromConfigMap(configMapNamespace, configMapName, configMapDataKey, targetNamespace string, listVariablesOnly bool) (repository.Template, error) { + if configMapNamespace == "" { + return nil, errors.New("invalid GetFromConfigMap operation: missing configMapNamespace value") + } + if configMapName == "" { + return nil, errors.New("invalid GetFromConfigMap operation: missing configMapName value") + } + + c, err := t.proxy.NewClient() + if err != nil { + return nil, err + } + + configMap := &corev1.ConfigMap{} + key := client.ObjectKey{ + Namespace: configMapNamespace, + Name: configMapName, + } + + if err := c.Get(ctx, key, configMap); err != nil { + return nil, errors.Wrapf(err, "error reading ConfigMap %s/%s", configMapNamespace, configMapName) + } + + data, ok := configMap.Data[configMapDataKey] + if !ok { + return nil, errors.Errorf("the ConfigMap %s/%s does not have the %q data key", configMapNamespace, configMapName, configMapDataKey) + } + + return repository.NewTemplate([]byte(data), t.configClient.Variables(), targetNamespace, listVariablesOnly) +} + +func (t *templateClient) GetFromURL(templateURL, targetNamespace string, listVariablesOnly bool) (repository.Template, error) { + if templateURL == "" { + return nil, errors.New("invalid GetFromURL operation: missing templateURL value") + } + + content, err := t.getURLContent(templateURL) + if err != nil { + return nil, errors.Wrapf(err, "invalid GetFromURL operation") + } + + return repository.NewTemplate(content, t.configClient.Variables(), targetNamespace, listVariablesOnly) +} + +func (t *templateClient) getURLContent(templateURL string) ([]byte, error) { + rURL, err := url.Parse(templateURL) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %q", templateURL) + } + + if rURL.Scheme == "https" && rURL.Host == "github.com" { + return t.getGitHubFileContent(rURL) + } + + if rURL.Scheme == "file" || rURL.Scheme == "" { + return t.getLocalFileContent(rURL) + } + + return nil, errors.Errorf("unable to read content from %q. Only reading from GitHub and local file system is supported", templateURL) +} + +func (t *templateClient) getLocalFileContent(rURL *url.URL) ([]byte, error) { + f, err := os.Stat(rURL.Path) + if err != nil { + return nil, errors.Errorf("failed to read file %q", rURL.Path) + } + if f.IsDir() { + return nil, errors.Errorf("invalid path: file %q is actually a directory", rURL.Path) + } + content, err := ioutil.ReadFile(rURL.Path) + if err != nil { + return nil, errors.Wrapf(err, "failed to read file %q", rURL.Path) + } + + return content, nil +} + +func (t *templateClient) getGitHubFileContent(rURL *url.URL) ([]byte, error) { + // Check if the path is in the expected format, + urlSplit := strings.Split(strings.TrimPrefix(rURL.Path, "/"), "/") + if len(urlSplit) < 5 { + return nil, errors.Errorf( + "invalid GitHub url %q: a GitHub url should be in the form https://github.com/{owner}/{repository}/blob/{branch}/{path-to-file}", rURL, + ) + } + + // Extract all the info from url split. + owner := urlSplit[0] + repository := urlSplit[1] + branch := urlSplit[3] + path := strings.Join(urlSplit[4:], "/") + + // gets the GitHub client + client, err := t.gitHubClientFactory(t.configClient.Variables()) + if err != nil { + return nil, err + } + + // gets the file from GiHub + fileContent, _, _, err := client.Repositories.GetContents(context.TODO(), owner, repository, path, &github.RepositoryContentGetOptions{Ref: branch}) + if err != nil { + return nil, handleGithubErr(err, "failed to get %q", rURL.Path) + } + if fileContent == nil { + return nil, errors.Errorf("%q does not return a valid file content", rURL.Path) + } + if fileContent.Encoding == nil || *fileContent.Encoding != "base64" { + return nil, errors.Errorf("invalid encoding detected for %q. Only base64 encoding supported", rURL.Path) + } + + content, err := base64.StdEncoding.DecodeString(*fileContent.Content) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode file %q", rURL.Path) + } + return content, nil +} + +func getGitHubClient(configVariablesClient config.VariablesClient) (*github.Client, error) { + var authenticatingHTTPClient *http.Client + if token, err := configVariablesClient.Get(config.GitHubTokenVariable); err == nil { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + authenticatingHTTPClient = oauth2.NewClient(context.TODO(), ts) + } + + return github.NewClient(authenticatingHTTPClient), nil +} + +// handleGithubErr wraps error messages +func handleGithubErr(err error, message string, args ...interface{}) error { + if _, ok := err.(*github.RateLimitError); ok { + return errors.New("rate limit for github api has been reached. Please wait one hour or get a personal API tokens a assign it to the GITHUB_TOKEN environment variable") + } + return errors.Wrapf(err, message, args...) +} diff --git a/cmd/clusterctl/pkg/client/cluster/template_test.go b/cmd/clusterctl/pkg/client/cluster/template_test.go new file mode 100644 index 000000000000..2d3a3af0ca60 --- /dev/null +++ b/cmd/clusterctl/pkg/client/cluster/template_test.go @@ -0,0 +1,379 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/google/go-github/github" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/config" + "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/repository" + "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/internal/test" +) + +var template = "apiVersion: cluster.x-k8s.io/v1alpha3\n" + + "kind: Cluster\n" + + "---\n" + + "apiVersion: cluster.x-k8s.io/v1alpha3\n" + + "kind: Machine" + +func Test_templateClient_GetFromConfigMap(t *testing.T) { + configClient, err := config.New("", config.InjectReader(test.NewFakeReader())) + if err != nil { + t.Fatal(err) + } + + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "my-template", + }, + Data: map[string]string{ + "prod": template, + }, + } + + type fields struct { + proxy Proxy + configClient config.Client + } + type args struct { + configMapNamespace string + configMapName string + configMapDataKey string + targetNamespace string + listVariablesOnly bool + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "Return template", + fields: fields{ + proxy: test.NewFakeProxy().WithObjs(configMap), + configClient: configClient, + }, + args: args{ + configMapNamespace: "ns1", + configMapName: "my-template", + configMapDataKey: "prod", + targetNamespace: "", + listVariablesOnly: false, + }, + want: template, + wantErr: false, + }, + { + name: "Config map does not exists", + fields: fields{ + proxy: test.NewFakeProxy().WithObjs(configMap), + configClient: configClient, + }, + args: args{ + configMapNamespace: "ns1", + configMapName: "something-else", + configMapDataKey: "prod", + targetNamespace: "", + listVariablesOnly: false, + }, + want: "", + wantErr: true, + }, + { + name: "Config map key does not exists", + fields: fields{ + proxy: test.NewFakeProxy().WithObjs(configMap), + configClient: configClient, + }, + args: args{ + configMapNamespace: "ns1", + configMapName: "my-template", + configMapDataKey: "something-else", + targetNamespace: "", + listVariablesOnly: false, + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tc := &templateClient{ + proxy: tt.fields.proxy, + configClient: tt.fields.configClient, + } + got, err := tc.GetFromConfigMap(tt.args.configMapNamespace, tt.args.configMapName, tt.args.configMapDataKey, tt.args.targetNamespace, tt.args.listVariablesOnly) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + + wantTemplate, err := repository.NewTemplate([]byte(tt.want), configClient.Variables(), tt.args.targetNamespace, tt.args.listVariablesOnly) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, wantTemplate) { + t.Errorf("got = %v, want %v", got, wantTemplate) + } + }) + } +} + +func Test_templateClient_getGitHubFileContent(t *testing.T) { + client, mux, teardown := test.NewFakeGitHub() + defer teardown() + + configClient, err := config.New("", config.InjectReader(test.NewFakeReader())) + if err != nil { + t.Fatal(err) + } + + mux.HandleFunc("/repos/kubernetes-sigs/cluster-api/contents/config/default/cluster-template.yaml", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{ + "type": "file", + "encoding": "base64", + "content": "`+base64.StdEncoding.EncodeToString([]byte(template))+`", + "sha": "f5f369044773ff9c6383c087466d12adb6fa0828", + "size": 12, + "name": "cluster-template.yaml", + "path": "config/default/cluster-template.yaml" + }`) + }) + + type args struct { + rURL *url.URL + } + tests := []struct { + name string + args args + want []byte + wantErr bool + }{ + { + name: "Return custom template", + args: args{ + rURL: mustParseURL("https://github.com/kubernetes-sigs/cluster-api/blob/master/config/default/cluster-template.yaml"), + }, + want: []byte(template), + wantErr: false, + }, + { + name: "Wrong url", + args: args{ + rURL: mustParseURL("https://github.com/kubernetes-sigs/cluster-api/blob/master/config/default/something-else.yaml"), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &templateClient{ + configClient: configClient, + gitHubClientFactory: func(configVariablesClient config.VariablesClient) (*github.Client, error) { + return client, nil + }, + } + got, err := c.getGitHubFileContent(tt.args.rURL) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_templateClient_getLocalFileContent(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "cc") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + path := filepath.Join(tmpDir, "cluster-template.yaml") + if err := ioutil.WriteFile(path, []byte(template), 0644); err != nil { + t.Fatalf("err: %s", err) + } + + type args struct { + rURL *url.URL + } + tests := []struct { + name string + args args + want []byte + wantErr bool + }{ + { + name: "Return custom template", + args: args{ + rURL: mustParseURL(path), + }, + want: []byte(template), + wantErr: false, + }, + { + name: "Wrong path", + args: args{ + rURL: mustParseURL(filepath.Join(tmpDir, "something-else.yaml")), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &templateClient{} + got, err := c.getLocalFileContent(tt.args.rURL) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_templateClient_GetFromURL(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "cc") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + configClient, err := config.New("", config.InjectReader(test.NewFakeReader())) + if err != nil { + t.Fatal(err) + } + + client, mux, teardown := test.NewFakeGitHub() + defer teardown() + + mux.HandleFunc("/repos/kubernetes-sigs/cluster-api/contents/config/default/cluster-template.yaml", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{ + "type": "file", + "encoding": "base64", + "content": "`+base64.StdEncoding.EncodeToString([]byte(template))+`", + "sha": "f5f369044773ff9c6383c087466d12adb6fa0828", + "size": 12, + "name": "cluster-template.yaml", + "path": "config/default/cluster-template.yaml" + }`) + }) + + path := filepath.Join(tmpDir, "cluster-template.yaml") + if err := ioutil.WriteFile(path, []byte(template), 0644); err != nil { + t.Fatalf("err: %s", err) + } + + type args struct { + templateURL string + targetNamespace string + listVariablesOnly bool + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Get from local file system", + args: args{ + templateURL: path, + targetNamespace: "", + listVariablesOnly: false, + }, + want: template, + wantErr: false, + }, + { + name: "Get from GitHub", + args: args{ + templateURL: "https://github.com/kubernetes-sigs/cluster-api/blob/master/config/default/cluster-template.yaml", + targetNamespace: "", + listVariablesOnly: false, + }, + want: template, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &templateClient{ + configClient: configClient, + gitHubClientFactory: func(configVariablesClient config.VariablesClient) (*github.Client, error) { + return client, nil + }, + } + got, err := c.GetFromURL(tt.args.templateURL, tt.args.targetNamespace, tt.args.listVariablesOnly) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + + wantTemplate, err := repository.NewTemplate([]byte(tt.want), configClient.Variables(), tt.args.targetNamespace, tt.args.listVariablesOnly) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, wantTemplate) { + t.Errorf("got = %v, want %v", got, wantTemplate) + } + }) + } +} + +func mustParseURL(rawURL string) *url.URL { + rURL, err := url.Parse(rawURL) + if err != nil { + panic(err) + } + return rURL +} diff --git a/cmd/clusterctl/pkg/client/config.go b/cmd/clusterctl/pkg/client/config.go index bbc7c3c3479f..d1193794eed9 100644 --- a/cmd/clusterctl/pkg/client/config.go +++ b/cmd/clusterctl/pkg/client/config.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/version" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" + "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/cluster" ) func (c *clusterctlClient) GetProvidersConfig() ([]Provider, error) { @@ -47,19 +48,150 @@ func (c *clusterctlClient) GetProviderComponents(provider, targetNameSpace, watc return components, nil } +// GetClusterTemplateOptions carries the options supported by GetClusterTemplate. +type GetClusterTemplateOptions struct { + // Kubeconfig file to use for accessing the management cluster. If empty, default rules for kubeconfig + // discovery will be used. + Kubeconfig string + + // ProviderRepositorySource to be used for reading the workload cluster template from a provider repository; + // only one template source can be used at time; if not other source will be set, a ProviderRepositorySource + // will be generated inferring values from the cluster. + ProviderRepositorySource *ProviderRepositorySourceOptions + + // URLSource to be used for reading the workload cluster template; only one template source can be used at time. + URLSource *URLSourceOptions + + // ConfigMapSource to be used for reading the workload cluster template; only one template source can be used at time. + ConfigMapSource *ConfigMapSourceOptions + + // TargetNamespace where the objects describing the workload cluster should be deployed. If not specified, + // the current namespace will be used. + TargetNamespace string + + // ClusterName to be used for the workload cluster. + ClusterName string + + // KubernetesVersion to use for the workload cluster. By default (empty), the value from os env variables + // or the .cluster-api/clusterctl.yaml config file will be used. + KubernetesVersion string + + // ControlPlaneMachineCount defines the number of control plane machines to be added to the workload cluster. + ControlPlaneMachineCount int + + // WorkerMachineCount defines number of worker machines to be added to the workload cluster. + WorkerMachineCount int + + // listVariablesOnly sets the GetClusterTemplate method to return the list of variables expected by the template + // without executing any further processing. + ListVariablesOnly bool +} + +// numSources return the number of template sources currently set on a GetClusterTemplateOptions. +func (o *GetClusterTemplateOptions) numSources() int { + numSources := 0 + if o.ProviderRepositorySource != nil { + numSources++ + } + if o.ConfigMapSource != nil { + numSources++ + } + if o.URLSource != nil { + numSources++ + } + return numSources +} + +// ProviderRepositorySourceOptions defines the options to be used when reading a workload cluster template +// from a provider repository. +type ProviderRepositorySourceOptions struct { + // InfrastructureProvider to read the workload cluster template from. By default (empty), the default + // infrastructure provider will be used if no other sources are specified. + InfrastructureProvider string + + // Flavor defines The workload cluster template variant to be used when reading from the infrastructure + // provider repository. By default (empty), the default cluster template will be used. + Flavor string +} + +// URLSourceOptions defines the options to be used when reading a workload cluster template from an URL. +type URLSourceOptions struct { + // URL to read the workload cluster template from. + URL string +} + +// DefaultCustomTemplateConfigMapKey where the workload cluster template is hosted. +const DefaultCustomTemplateConfigMapKey = "template" + +// ConfigMapSourceOptions defines the options to be used when reading a workload cluster template from a ConfigMap. +type ConfigMapSourceOptions struct { + // Namespace where the ConfigMap exists. By default (empty), the current namespace will be used. + Namespace string + + // Name to read the workload cluster template from. + Name string + + // DataKey where the workload cluster template is hosted. By default (empty), the + // DefaultCustomTemplateConfigMapKey will be used. + DataKey string +} + func (c *clusterctlClient) GetClusterTemplate(options GetClusterTemplateOptions) (Template, error) { + // Checks that no more than on source is set + numsSource := options.numSources() + if numsSource > 1 { + return nil, errors.New("invalid cluster template source: only one template can be used at time") + } + + // If no source is set, defaults to using an empty ProviderRepositorySource so values will be + // inferred from the cluster inventory. + if numsSource == 0 { + options.ProviderRepositorySource = &ProviderRepositorySourceOptions{} + } + // Gets the client for the current management cluster cluster, err := c.clusterClientFactory(options.Kubeconfig) if err != nil { return nil, err } + // If the option specifying the targetNamespace is empty, try to detect it. + if options.TargetNamespace == "" { + currentNamespace, err := cluster.Proxy().CurrentNamespace() + if err != nil { + return nil, err + } + options.TargetNamespace = currentNamespace + } + + // Inject some of the templateOptions into the configClient so they can be consumed as a variables from the template. + if err := c.templateOptionsToVariables(options); err != nil { + return nil, err + } + + // Gets the workload cluster template from the selected source + if options.ProviderRepositorySource != nil { + return c.getTemplateFromRepository(cluster, *options.ProviderRepositorySource, options.TargetNamespace, options.ListVariablesOnly) + } + if options.ConfigMapSource != nil { + return c.getTemplateFromConfigMap(cluster, *options.ConfigMapSource, options.TargetNamespace, options.ListVariablesOnly) + } + if options.URLSource != nil { + return c.getTemplateFromURL(cluster, *options.URLSource, options.TargetNamespace, options.ListVariablesOnly) + } + + return nil, errors.New("unable to read custom template. Please specify a template source") +} + +// getTemplateFromRepository returns a workload cluster template from a provider repository. +func (c *clusterctlClient) getTemplateFromRepository(cluster cluster.Client, source ProviderRepositorySourceOptions, targetNamespace string, listVariablesOnly bool) (Template, error) { + // ensure the custom resource definitions required by clusterctl are in place if err := cluster.ProviderInventory().EnsureCustomResourceDefinitions(); err != nil { return nil, err } // If the option specifying the name of the infrastructure provider to get templates from is empty, try to detect it. - provider := options.InfrastructureProvider + provider := source.InfrastructureProvider if provider == "" { defaultProviderName, err := cluster.ProviderInventory().GetDefaultProviderName(clusterctlv1.InfrastructureProviderType) if err != nil { @@ -91,20 +223,6 @@ func (c *clusterctlClient) GetClusterTemplate(options GetClusterTemplateOptions) version = defaultProviderVersion } - // If the option specifying the targetNamespace is empty, try to detect it. - if options.TargetNamespace == "" { - currentNamespace, err := cluster.Proxy().CurrentNamespace() - if err != nil { - return nil, err - } - options.TargetNamespace = currentNamespace - } - - // Inject some of the templateOptions into the configClient so they can be consumed as a variables from the template. - if err := c.templateOptionsToVariables(options); err != nil { - return nil, err - } - // Get the template from the template repository. providerConfig, err := c.configClient.Providers().Get(name) if err != nil { @@ -116,13 +234,37 @@ func (c *clusterctlClient) GetClusterTemplate(options GetClusterTemplateOptions) return nil, err } - template, err := repo.Templates(version).Get(options.Flavor, options.TargetNamespace) + template, err := repo.Templates(version).Get(source.Flavor, targetNamespace, listVariablesOnly) if err != nil { return nil, err } return template, nil } +// getTemplateFromConfigMap returns a workload cluster template from a ConfigMap. +func (c *clusterctlClient) getTemplateFromConfigMap(cluster cluster.Client, source ConfigMapSourceOptions, targetNamespace string, listVariablesOnly bool) (Template, error) { + // If the option specifying the configMapNamespace is empty, default it to the current namespace. + if source.Namespace == "" { + currentNamespace, err := cluster.Proxy().CurrentNamespace() + if err != nil { + return nil, err + } + source.Namespace = currentNamespace + } + + // If the option specifying the configMapDataKey is empty, default it. + if source.DataKey == "" { + source.DataKey = DefaultCustomTemplateConfigMapKey + } + + return cluster.Template().GetFromConfigMap(source.Namespace, source.Name, source.DataKey, targetNamespace, listVariablesOnly) +} + +// getTemplateFromURL returns a workload cluster template from an URL. +func (c *clusterctlClient) getTemplateFromURL(cluster cluster.Client, source URLSourceOptions, targetNamespace string, listVariablesOnly bool) (Template, error) { + return cluster.Template().GetFromURL(source.URL, targetNamespace, listVariablesOnly) +} + // templateOptionsToVariables injects some of the templateOptions to the configClient so they can be consumed as a variables from the template. func (c *clusterctlClient) templateOptionsToVariables(options GetClusterTemplateOptions) error { diff --git a/cmd/clusterctl/pkg/client/config/variables_client.go b/cmd/clusterctl/pkg/client/config/variables_client.go index 3343041dc4aa..6b511ee768a2 100644 --- a/cmd/clusterctl/pkg/client/config/variables_client.go +++ b/cmd/clusterctl/pkg/client/config/variables_client.go @@ -18,6 +18,11 @@ package config import "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/internal/test" +const ( + // GitHubTokenVariable defines a variable hosting the GitHub access token + GitHubTokenVariable = "github-token" +) + // VariablesClient has methods to work with environment variables and with variables defined in the clusterctl configuration file. type VariablesClient interface { // Get returns a variable value. If the variable is not defined an error is returned. diff --git a/cmd/clusterctl/pkg/client/config_test.go b/cmd/clusterctl/pkg/client/config_test.go index 4b33e2252242..132b2edef7b3 100644 --- a/cmd/clusterctl/pkg/client/config_test.go +++ b/cmd/clusterctl/pkg/client/config_test.go @@ -18,9 +18,14 @@ package client import ( "fmt" + "io/ioutil" + "os" + "path/filepath" "reflect" "testing" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" "sigs.k8s.io/cluster-api/cmd/clusterctl/pkg/client/config" ) @@ -304,16 +309,46 @@ func Test_clusterctlClient_templateOptionsToVariables(t *testing.T) { } func Test_clusterctlClient_GetClusterTemplate(t *testing.T) { + rawTemplate := templateYAML("ns3", "${ CLUSTER_NAME }") + + // Template on a file + tmpDir, err := ioutil.TempDir("", "cc") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + path := filepath.Join(tmpDir, "cluster-template.yaml") + if err := ioutil.WriteFile(path, rawTemplate, 0644); err != nil { + t.Fatalf("err: %s", err) + } + + // Template on a repository & in a ConfigMap + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "my-template", + }, + Data: map[string]string{ + "prod": string(rawTemplate), + }, + } + config1 := newFakeConfig(). WithProvider(infraProviderConfig) repository1 := newFakeRepository(infraProviderConfig, config1.Variables()). WithPaths("root", "components"). WithDefaultVersion("v3.0.0"). - WithFile("v3.0.0", "cluster-template.yaml", templateYAML("ns3", "${ CLUSTER_NAME }")) + WithFile("v3.0.0", "cluster-template.yaml", rawTemplate) cluster1 := newFakeCluster("kubeconfig", config1). - WithProviderInventory(infraProviderConfig.Name(), infraProviderConfig.Type(), "v3.0.0", "foo", "bar") + WithProviderInventory(infraProviderConfig.Name(), infraProviderConfig.Type(), "v3.0.0", "foo", "bar"). + WithObjs(configMap) client := newFakeClient(config1). WithCluster(cluster1). @@ -336,12 +371,14 @@ func Test_clusterctlClient_GetClusterTemplate(t *testing.T) { wantErr bool }{ { - name: "pass", + name: "repository source - pass", args: args{ options: GetClusterTemplateOptions{ - Kubeconfig: "kubeconfig", - InfrastructureProvider: "infra:v3.0.0", - Flavor: "", + Kubeconfig: "kubeconfig", + ProviderRepositorySource: &ProviderRepositorySourceOptions{ + InfrastructureProvider: "infra:v3.0.0", + Flavor: "", + }, ClusterName: "test", TargetNamespace: "ns1", ControlPlaneMachineCount: 1, @@ -354,12 +391,14 @@ func Test_clusterctlClient_GetClusterTemplate(t *testing.T) { }, }, { - name: "detects provider name/version if missing", + name: "repository source - detects provider name/version if missing", args: args{ options: GetClusterTemplateOptions{ - Kubeconfig: "kubeconfig", - InfrastructureProvider: "", // empty triggers auto-detection of the provider name/version - Flavor: "", + Kubeconfig: "kubeconfig", + ProviderRepositorySource: &ProviderRepositorySourceOptions{ + InfrastructureProvider: "", // empty triggers auto-detection of the provider name/version + Flavor: "", + }, ClusterName: "test", TargetNamespace: "ns1", ControlPlaneMachineCount: 1, @@ -372,12 +411,14 @@ func Test_clusterctlClient_GetClusterTemplate(t *testing.T) { }, }, { - name: "use current namespace if targetNamespace is missing", + name: "repository source - use current namespace if targetNamespace is missing", args: args{ options: GetClusterTemplateOptions{ - Kubeconfig: "kubeconfig", - InfrastructureProvider: "infra:v3.0.0", - Flavor: "", + Kubeconfig: "kubeconfig", + ProviderRepositorySource: &ProviderRepositorySourceOptions{ + InfrastructureProvider: "infra:v3.0.0", + Flavor: "", + }, ClusterName: "test", TargetNamespace: "", // empty triggers usage of the current namespace ControlPlaneMachineCount: 1, @@ -389,6 +430,46 @@ func Test_clusterctlClient_GetClusterTemplate(t *testing.T) { yaml: templateYAML("default", "test"), // original template modified with target namespace and variable replacement }, }, + { + name: "URL source - pass", + args: args{ + options: GetClusterTemplateOptions{ + Kubeconfig: "kubeconfig", + URLSource: &URLSourceOptions{ + URL: path, + }, + ClusterName: "test", + TargetNamespace: "ns1", + ControlPlaneMachineCount: 1, + }, + }, + want: templateValues{ + variables: []string{"CLUSTER_NAME"}, // variable detected + targetNamespace: "ns1", + yaml: templateYAML("ns1", "test"), // original template modified with target namespace and variable replacement + }, + }, + { + name: "ConfigMap source - pass", + args: args{ + options: GetClusterTemplateOptions{ + Kubeconfig: "kubeconfig", + ConfigMapSource: &ConfigMapSourceOptions{ + Namespace: "ns1", + Name: "my-template", + DataKey: "prod", + }, + ClusterName: "test", + TargetNamespace: "ns1", + ControlPlaneMachineCount: 1, + }, + }, + want: templateValues{ + variables: []string{"CLUSTER_NAME"}, // variable detected + targetNamespace: "ns1", + yaml: templateYAML("ns1", "test"), // original template modified with target namespace and variable replacement + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/clusterctl/pkg/client/repository/repository_github.go b/cmd/clusterctl/pkg/client/repository/repository_github.go index 76bc57642d55..792bc5b5527d 100644 --- a/cmd/clusterctl/pkg/client/repository/repository_github.go +++ b/cmd/clusterctl/pkg/client/repository/repository_github.go @@ -34,7 +34,6 @@ import ( const ( httpsScheme = "https" githubDomain = "github.com" - githubTokeVariable = "github-token" githubReleaseRepository = "releases" githubLatestReleaseLabel = "latest" ) @@ -145,8 +144,7 @@ func newGitHubRepository(providerConfig config.Provider, configVariablesClient c componentsPath: componentsPath, } - token, err := configVariablesClient.Get(githubTokeVariable) - if err == nil { + if token, err := configVariablesClient.Get(config.GitHubTokenVariable); err == nil { repo.setClientToken(token) } diff --git a/cmd/clusterctl/pkg/client/repository/repository_github_test.go b/cmd/clusterctl/pkg/client/repository/repository_github_test.go index 63a9a6fd67ff..98919a0a467f 100644 --- a/cmd/clusterctl/pkg/client/repository/repository_github_test.go +++ b/cmd/clusterctl/pkg/client/repository/repository_github_test.go @@ -19,8 +19,6 @@ package repository import ( "fmt" "net/http" - "net/http/httptest" - "net/url" "reflect" "testing" @@ -37,7 +35,7 @@ import ( //TODO: test getComponentsPath func Test_gitHubRepository_getVersions(t *testing.T) { - client, mux, teardown := setup() + client, mux, teardown := test.NewFakeGitHub() defer teardown() // setup an handler for returning 5 fake releases @@ -96,7 +94,7 @@ func Test_gitHubRepository_getVersions(t *testing.T) { } func Test_gitHubRepository_getLatestRelease(t *testing.T) { - client, mux, teardown := setup() + client, mux, teardown := test.NewFakeGitHub() defer teardown() // setup an handler for returning 4 fake releases @@ -168,7 +166,7 @@ func Test_gitHubRepository_getLatestRelease(t *testing.T) { } func Test_gitHubRepository_getReleaseByTag(t *testing.T) { - client, mux, teardown := setup() + client, mux, teardown := test.NewFakeGitHub() defer teardown() providerConfig := config.NewProvider("test", "https://github.com/o/r/releases/v0.4.1/path", clusterctlv1.CoreProviderType) @@ -240,12 +238,12 @@ func Test_gitHubRepository_getReleaseByTag(t *testing.T) { } func Test_gitHubRepository_downloadFilesFromRelease(t *testing.T) { - client, mux, teardown := setup() + client, mux, teardown := test.NewFakeGitHub() defer teardown() providerConfig := config.NewProvider("test", "https://github.com/o/r/releases/v0.4.1/file.yaml", clusterctlv1.CoreProviderType) //tree/master/path not relevant for the test - // setup an handler for returning a fake release asset + // test.NewFakeGitHub an handler for returning a fake release asset mux.HandleFunc("/repos/o/r/releases/assets/1", func(w http.ResponseWriter, r *http.Request) { testMethod(t, r, "GET") w.Header().Set("Content-Type", "application/octet-stream") @@ -343,30 +341,6 @@ func Test_gitHubRepository_downloadFilesFromRelease(t *testing.T) { } } -const baseURLPath = "/api-v3" - -// setup sets up a test HTTP server along with a github.Client that is -// configured to talk to that test server. Tests should register handlers on -// mux which provide mock responses for the API method being tested. -func setup() (client *github.Client, mux *http.ServeMux, teardown func()) { - // mux is the HTTP request multiplexer used with the test server. - mux = http.NewServeMux() - - apiHandler := http.NewServeMux() - apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux)) - - // server is a test HTTP server used to provide mock API responses. - server := httptest.NewServer(apiHandler) - - // client is the GitHub client being tested and is configured to use test server. - client = github.NewClient(nil) - url, _ := url.Parse(server.URL + baseURLPath + "/") - client.BaseURL = url - client.UploadURL = url - - return client, mux, server.Close -} - func testMethod(t *testing.T, r *http.Request, want string) { if got := r.Method; got != want { t.Errorf("Request method: %v, want %v", got, want) diff --git a/cmd/clusterctl/pkg/client/repository/template.go b/cmd/clusterctl/pkg/client/repository/template.go index b5f0560c60ab..5d42980b99ab 100644 --- a/cmd/clusterctl/pkg/client/repository/template.go +++ b/cmd/clusterctl/pkg/client/repository/template.go @@ -70,9 +70,15 @@ func (t *template) Yaml() ([]byte, error) { } // NewTemplate returns a new objects embedding a cluster template YAML file. -func NewTemplate(rawYaml []byte, configVariablesClient config.VariablesClient, targetNamespace string) (*template, error) { +func NewTemplate(rawYaml []byte, configVariablesClient config.VariablesClient, targetNamespace string, listVariablesOnly bool) (*template, error) { // Inspect variables and replace with values from the configuration. variables := inspectVariables(rawYaml) + if listVariablesOnly { + return &template{ + variables: variables, + targetNamespace: targetNamespace, + }, nil + } yaml, err := replaceVariables(rawYaml, variables, configVariablesClient) if err != nil { diff --git a/cmd/clusterctl/pkg/client/repository/template_client.go b/cmd/clusterctl/pkg/client/repository/template_client.go index 450de42ee8c8..19ad2d9ded76 100644 --- a/cmd/clusterctl/pkg/client/repository/template_client.go +++ b/cmd/clusterctl/pkg/client/repository/template_client.go @@ -39,7 +39,7 @@ type TemplateOptions struct { // TemplateClient has methods to work with cluster templates hosted on a provider repository. // Templates are yaml files to be used for creating a guest cluster. type TemplateClient interface { - Get(flavor, targetNamespace string) (Template, error) + Get(flavor, targetNamespace string, listVariablesOnly bool) (Template, error) } // templateClient implements TemplateClient. @@ -66,7 +66,7 @@ func newTemplateClient(provider config.Provider, version string, repository Repo // Get return the template for the flavor specified. // In case the template does not exists, an error is returned. // Get assumes the following naming convention for templates: cluster-template[-].yaml -func (c *templateClient) Get(flavor, targetNamespace string) (Template, error) { +func (c *templateClient) Get(flavor, targetNamespace string, listVariablesOnly bool) (Template, error) { log := logf.Log if targetNamespace == "" { @@ -100,5 +100,5 @@ func (c *templateClient) Get(flavor, targetNamespace string) (Template, error) { log.V(1).Info("Using", "Override", name, "Provider", c.provider.Name(), "Version", version) } - return NewTemplate(rawYaml, c.configVariablesClient, targetNamespace) + return NewTemplate(rawYaml, c.configVariablesClient, targetNamespace, listVariablesOnly) } diff --git a/cmd/clusterctl/pkg/client/repository/template_client_test.go b/cmd/clusterctl/pkg/client/repository/template_client_test.go index cec190facf7f..328a1c8a4efd 100644 --- a/cmd/clusterctl/pkg/client/repository/template_client_test.go +++ b/cmd/clusterctl/pkg/client/repository/template_client_test.go @@ -37,8 +37,9 @@ func Test_templates_Get(t *testing.T) { configVariablesClient config.VariablesClient } type args struct { - flavor string - targetNamespace string + flavor string + targetNamespace string + listVariablesOnly bool } type want struct { variables []string @@ -63,8 +64,9 @@ func Test_templates_Get(t *testing.T) { configVariablesClient: test.NewFakeVariableClient().WithVar(variableName, variableValue), }, args: args{ - flavor: "", - targetNamespace: "ns1", + flavor: "", + targetNamespace: "ns1", + listVariablesOnly: false, }, want: want{ variables: []string{variableName}, @@ -84,8 +86,9 @@ func Test_templates_Get(t *testing.T) { configVariablesClient: test.NewFakeVariableClient().WithVar(variableName, variableValue), }, args: args{ - flavor: "prod", - targetNamespace: "ns1", + flavor: "prod", + targetNamespace: "ns1", + listVariablesOnly: false, }, want: want{ variables: []string{variableName}, @@ -104,16 +107,57 @@ func Test_templates_Get(t *testing.T) { configVariablesClient: test.NewFakeVariableClient().WithVar(variableName, variableValue), }, args: args{ - flavor: "", - targetNamespace: "ns1", + flavor: "", + targetNamespace: "ns1", + listVariablesOnly: false, + }, + wantErr: true, + }, + { + name: "fails if variables does not exists", + fields: fields{ + version: "v1.0", + provider: p1, + repository: test.NewFakeRepository(). + WithPaths("root", ""). + WithDefaultVersion("v1.0"). + WithFile("v1.0", "cluster-template.yaml", templateMapYaml), + configVariablesClient: test.NewFakeVariableClient(), + }, + args: args{ + flavor: "", + targetNamespace: "ns1", + listVariablesOnly: false, }, wantErr: true, }, + { + name: "pass if variables does not exists but listVariablesOnly flag is set", + fields: fields{ + version: "v1.0", + provider: p1, + repository: test.NewFakeRepository(). + WithPaths("root", ""). + WithDefaultVersion("v1.0"). + WithFile("v1.0", "cluster-template.yaml", templateMapYaml), + configVariablesClient: test.NewFakeVariableClient(), + }, + args: args{ + flavor: "", + targetNamespace: "ns1", + listVariablesOnly: true, + }, + want: want{ + variables: []string{variableName}, + targetNamespace: "ns1", + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { f := newTemplateClient(tt.fields.provider, tt.fields.version, tt.fields.repository, tt.fields.configVariablesClient) - got, err := f.Get(tt.args.flavor, tt.args.targetNamespace) + got, err := f.Get(tt.args.flavor, tt.args.targetNamespace, tt.args.listVariablesOnly) if (err != nil) != tt.wantErr { t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) } @@ -135,7 +179,7 @@ func Test_templates_Get(t *testing.T) { t.Fatalf("got.Yaml error = %v", err) } - if !bytes.Contains(yaml, []byte(fmt.Sprintf("variable: %s", variableValue))) { + if !tt.args.listVariablesOnly && !bytes.Contains(yaml, []byte(fmt.Sprintf("variable: %s", variableValue))) { t.Error("got.Yaml without variable substitution") } diff --git a/cmd/clusterctl/pkg/client/repository/template_test.go b/cmd/clusterctl/pkg/client/repository/template_test.go index 9c2fb4b884f8..ac0a622c5a39 100644 --- a/cmd/clusterctl/pkg/client/repository/template_test.go +++ b/cmd/clusterctl/pkg/client/repository/template_test.go @@ -40,6 +40,7 @@ func Test_newTemplate(t *testing.T) { rawYaml []byte configVariablesClient config.VariablesClient targetNamespace string + listVariablesOnly bool } type want struct { variables []string @@ -57,6 +58,21 @@ func Test_newTemplate(t *testing.T) { rawYaml: templateMapYaml, configVariablesClient: test.NewFakeVariableClient().WithVar(variableName, variableValue), targetNamespace: "ns1", + listVariablesOnly: false, + }, + want: want{ + variables: []string{variableName}, + targetNamespace: "ns1", + }, + wantErr: false, + }, + { + name: "List variable only", + args: args{ + rawYaml: templateMapYaml, + configVariablesClient: test.NewFakeVariableClient(), + targetNamespace: "ns1", + listVariablesOnly: true, }, want: want{ variables: []string{variableName}, @@ -67,7 +83,7 @@ func Test_newTemplate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewTemplate(tt.args.rawYaml, tt.args.configVariablesClient, tt.args.targetNamespace) + got, err := NewTemplate(tt.args.rawYaml, tt.args.configVariablesClient, tt.args.targetNamespace, tt.args.listVariablesOnly) if (err != nil) != tt.wantErr { t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) } @@ -83,6 +99,10 @@ func Test_newTemplate(t *testing.T) { t.Errorf("got.TargetNamespace() = %v, want = %v ", got.TargetNamespace(), tt.want.targetNamespace) } + if tt.args.listVariablesOnly { + return + } + // check variable replaced in components yaml, err := got.Yaml() if err != nil { diff --git a/cmd/clusterctl/pkg/internal/test/fake_github.go b/cmd/clusterctl/pkg/internal/test/fake_github.go new file mode 100644 index 000000000000..bb9f45a654d4 --- /dev/null +++ b/cmd/clusterctl/pkg/internal/test/fake_github.go @@ -0,0 +1,56 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-github/github" +) + +const baseURLPath = "/api-v3" + +// NewFakeGitHub sets up a test HTTP server along with a github.Client that is +// configured to talk to that test server. Tests should register handlers on +// mux which provide mock responses for the API method being tested. +func NewFakeGitHub() (client *github.Client, mux *http.ServeMux, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + apiHandler := http.NewServeMux() + apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux)) + + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + // client is the GitHub client being tested and is configured to use test server. + client = github.NewClient(nil) + url, _ := url.Parse(server.URL + baseURLPath + "/") + client.BaseURL = url + client.UploadURL = url + + return client, mux, server.Close +} + +func TestMethod(t *testing.T, r *http.Request, want string) { + if got := r.Method; got != want { + t.Errorf("Request method: %v, want %v", got, want) + } +}