diff --git a/cmd/BUILD b/cmd/BUILD deleted file mode 100644 index 2da1a0ac88dc6..0000000000000 --- a/cmd/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/clicheck:all-srcs", - "//cmd/cloud-controller-manager:all-srcs", - "//cmd/dependencycheck:all-srcs", - "//cmd/gendocs:all-srcs", - "//cmd/genkubedocs:all-srcs", - "//cmd/genman:all-srcs", - "//cmd/genswaggertypedocs:all-srcs", - "//cmd/genutils:all-srcs", - "//cmd/genyaml:all-srcs", - "//cmd/importverifier:all-srcs", - "//cmd/kube-apiserver:all-srcs", - "//cmd/kube-controller-manager:all-srcs", - "//cmd/kube-proxy:all-srcs", - "//cmd/kube-scheduler:all-srcs", - "//cmd/kubeadm:all-srcs", - "//cmd/kubectl:all-srcs", - "//cmd/kubectl-convert:all-srcs", - "//cmd/kubelet:all-srcs", - "//cmd/kubemark:all-srcs", - "//cmd/linkcheck:all-srcs", - "//cmd/preferredimports:all-srcs", - "//cmd/verifydependencies:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/OWNERS b/cmd/OWNERS deleted file mode 100644 index d81537c055bd8..0000000000000 --- a/cmd/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - dchen1107 - - lavalamp - - mikedanese - - thockin - - dims -approvers: - - dchen1107 - - lavalamp - - mikedanese - - thockin diff --git a/cmd/clicheck/BUILD b/cmd/clicheck/BUILD deleted file mode 100644 index a8bff329b020b..0000000000000 --- a/cmd/clicheck/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) - -go_binary( - name = "clicheck", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["check_cli_conventions.go"], - importpath = "k8s.io/kubernetes/cmd/clicheck", - deps = [ - "//staging/src/k8s.io/kubectl/pkg/cmd:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/cmd/util/sanity:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/clicheck/OWNERS b/cmd/clicheck/OWNERS deleted file mode 100644 index d3e63eb0d8a5d..0000000000000 --- a/cmd/clicheck/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- sig-cli-maintainers -reviewers: -- sig-cli diff --git a/cmd/clicheck/check_cli_conventions.go b/cmd/clicheck/check_cli_conventions.go deleted file mode 100644 index e24d6930a12ef..0000000000000 --- a/cmd/clicheck/check_cli_conventions.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "io/ioutil" - "os" - - "k8s.io/kubectl/pkg/cmd" - cmdsanity "k8s.io/kubectl/pkg/cmd/util/sanity" -) - -func main() { - var errorCount int - - kubectl := cmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) - errors := cmdsanity.RunCmdChecks(kubectl, cmdsanity.AllCmdChecks, []string{}) - for _, err := range errors { - errorCount++ - fmt.Fprintf(os.Stderr, " %d. %s\n", errorCount, err) - } - - errors = cmdsanity.RunGlobalChecks(cmdsanity.AllGlobalChecks) - for _, err := range errors { - errorCount++ - fmt.Fprintf(os.Stderr, " %d. %s\n", errorCount, err) - } - - if errorCount > 0 { - fmt.Fprintf(os.Stdout, "Found %d errors.\n", errorCount) - os.Exit(1) - } - - fmt.Fprintln(os.Stdout, "Congrats, CLI looks good!") -} diff --git a/cmd/cloud-controller-manager/.import-restrictions b/cmd/cloud-controller-manager/.import-restrictions deleted file mode 100644 index 4515e2f7be521..0000000000000 --- a/cmd/cloud-controller-manager/.import-restrictions +++ /dev/null @@ -1,37 +0,0 @@ -rules: - - selectorRegexp: k8s[.]io/kubernetes - allowedPrefixes: - - k8s.io/kubernetes/pkg/api/legacyscheme - - k8s.io/kubernetes/pkg/api/service - - k8s.io/kubernetes/pkg/api/v1/pod - - k8s.io/kubernetes/pkg/apis/apps - - k8s.io/kubernetes/pkg/apis/autoscaling - - k8s.io/kubernetes/pkg/apis/core - - k8s.io/kubernetes/pkg/apis/core/helper - - k8s.io/kubernetes/pkg/apis/core/install - - k8s.io/kubernetes/pkg/apis/core/pods - - k8s.io/kubernetes/pkg/apis/core/v1 - - k8s.io/kubernetes/pkg/apis/core/v1/helper - - k8s.io/kubernetes/pkg/apis/core/validation - - k8s.io/kubernetes/pkg/apis/scheduling - - k8s.io/kubernetes/pkg/capabilities - - k8s.io/kubernetes/pkg/cluster/ports - - k8s.io/kubernetes/pkg/controller - - k8s.io/kubernetes/pkg/controller/nodeipam - - k8s.io/kubernetes/pkg/controller/nodeipam/config - - k8s.io/kubernetes/pkg/controller/nodeipam/ipam - - k8s.io/kubernetes/pkg/controller/nodeipam/ipam/cidrset - - k8s.io/kubernetes/pkg/controller/nodeipam/ipam/sync - - k8s.io/kubernetes/pkg/controller/nodeipam/ipam/test - - k8s.io/kubernetes/pkg/controller/testutil - - k8s.io/kubernetes/pkg/controller/util/node - - k8s.io/kubernetes/pkg/features - - k8s.io/kubernetes/pkg/fieldpath - - k8s.io/kubernetes/pkg/kubelet/types - - k8s.io/kubernetes/pkg/kubelet/util/format - - k8s.io/kubernetes/pkg/security/apparmor - - k8s.io/kubernetes/pkg/securitycontext - - k8s.io/kubernetes/pkg/util/hash - - k8s.io/kubernetes/pkg/util/node - - k8s.io/kubernetes/pkg/util/parsers - - k8s.io/kubernetes/pkg/util/taints \ No newline at end of file diff --git a/cmd/cloud-controller-manager/BUILD b/cmd/cloud-controller-manager/BUILD deleted file mode 100644 index e2cdabf15e6f1..0000000000000 --- a/cmd/cloud-controller-manager/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "cloud-controller-manager", - embed = [":go_default_library"], - pure = "on", - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = [ - "main.go", - "nodeipamcontroller.go", - "providers.go", - ], - importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager", - deps = [ - "//pkg/controller/nodeipam:go_default_library", - "//pkg/controller/nodeipam/config:go_default_library", - "//pkg/controller/nodeipam/ipam:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/cloud-provider:go_default_library", - "//staging/src/k8s.io/cloud-provider/app:go_default_library", - "//staging/src/k8s.io/cloud-provider/app/config:go_default_library", - "//staging/src/k8s.io/cloud-provider/options:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - "//staging/src/k8s.io/controller-manager/app:go_default_library", - "//staging/src/k8s.io/controller-manager/pkg/features:go_default_library", - "//staging/src/k8s.io/legacy-cloud-providers/aws:go_default_library", - "//staging/src/k8s.io/legacy-cloud-providers/azure:go_default_library", - "//staging/src/k8s.io/legacy-cloud-providers/gce:go_default_library", - "//staging/src/k8s.io/legacy-cloud-providers/openstack:go_default_library", - "//staging/src/k8s.io/legacy-cloud-providers/vsphere:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/cloud-controller-manager/OWNERS b/cmd/cloud-controller-manager/OWNERS deleted file mode 100644 index 50e888e5b0ee0..0000000000000 --- a/cmd/cloud-controller-manager/OWNERS +++ /dev/null @@ -1,20 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- thockin -- luxas -- andrewsykim -- cheftako -- sttts -reviewers: -- thockin -- luxas -- andrewsykim -- cheftako -- stewart-yu -- sttts -emeritus_approvers: -- wlan0 -labels: -- sig/api-machinery -- sig/cloud-provider diff --git a/cmd/cloud-controller-manager/main.go b/cmd/cloud-controller-manager/main.go deleted file mode 100644 index 62843f55a2b6d..0000000000000 --- a/cmd/cloud-controller-manager/main.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The external controller manager is responsible for running controller loops that -// are cloud provider dependent. It uses the API to listen to new events on resources. - -// This file should be written by each cloud provider. -// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go -// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go -// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go -// The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use. - -package main - -import ( - "fmt" - "math/rand" - "net/http" - "os" - "time" - - "github.com/spf13/pflag" - cloudprovider "k8s.io/cloud-provider" - "k8s.io/cloud-provider/app" - cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - "k8s.io/cloud-provider/options" - "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/klog/v2" - nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" - // For existing cloud providers, the option to import legacy providers is still available. - // e.g. _"k8s.io/legacy-cloud-providers/" -) - -const ( - // cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case. - cloudProviderName = "SampleCloudProviderName" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - // cloudProviderConfigFile shows an sample of parse config file from flag option - var flagset *pflag.FlagSet = pflag.NewFlagSet("flagSet", pflag.ContinueOnError) - var cloudProviderConfigFile *string = flagset.String("cloud-provider-configfile", "", "This is the sample input for cloud provider config file") - pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true - _ = pflag.CommandLine.Parse(os.Args[1:]) - - // this is an example of allow-listing specific controller loops - controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"} - - s, err := options.NewCloudControllerManagerOptions() - if err != nil { - klog.Fatalf("unable to initialize command options: %v", err) - } - c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, *cloudProviderConfigFile) - if err != nil { - klog.Fatalf("Cloud provider could not be initialized: %v", err) - } - if cloud == nil { - klog.Fatalf("cloud provider is nil") - } - - if !cloud.HasClusterID() { - if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { - klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") - } else { - klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") - } - } - - // Initialize the cloud provider with a reference to the clientBuilder - cloud.Initialize(c.ClientBuilder, make(chan struct{})) - // Set the informer on the user cloud object - if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { - informerUserCloud.SetInformers(c.SharedInformers) - } - - controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud) - - // Here is an example to remove the controller which is not needed. - // e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need. - //delete(controllerInitializers, "cloud-node-lifecycle") - - // Here is an example to add an controller(NodeIpamController) which will be used by cloud provider - // generate nodeipamconfig. Here is an sample code. Please pass the right parameter in your code. - // If you do not need additional controller, please ignore. - nodeipamconfig := nodeipamconfig.NodeIPAMControllerConfiguration{ - ServiceCIDR: "sample", - SecondaryServiceCIDR: "sample", - NodeCIDRMaskSize: 11, - NodeCIDRMaskSizeIPv4: 11, - NodeCIDRMaskSizeIPv6: 111, - } - controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeipamconfig, cloud) - - command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers) - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - // Here is an sample - pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc) - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - // the flags could be set before execute - command.Flags().VisitAll(func(flag *pflag.Flag) { - if flag.Name == "cloud-provider" { - flag.Value.Set("SampleCloudProviderFlagValue") - return - } - }) - if err := command.Execute(); err != nil { - os.Exit(1) - } -} - -func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) { - return startNodeIpamController(ccmconfig, nodeipamconfig, ctx, cloud) - } -} diff --git a/cmd/cloud-controller-manager/nodeipamcontroller.go b/cmd/cloud-controller-manager/nodeipamcontroller.go deleted file mode 100644 index c47cbd3bc98d7..0000000000000 --- a/cmd/cloud-controller-manager/nodeipamcontroller.go +++ /dev/null @@ -1,210 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file holds the code related with the sample nodeipamcontroller -// which demonstrates how cloud providers add external controllers to cloud-controller-manager - -package main - -import ( - "errors" - "fmt" - "net" - "net/http" - "strings" - - utilfeature "k8s.io/apiserver/pkg/util/feature" - cloudprovider "k8s.io/cloud-provider" - cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/controller-manager/pkg/features" - "k8s.io/klog/v2" - nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam" - nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" - "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" - netutils "k8s.io/utils/net" -) - -const ( - // defaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr - defaultNodeMaskCIDRIPv4 = 24 - // defaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr - defaultNodeMaskCIDRIPv6 = 64 -) - -func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) { - var serviceCIDR *net.IPNet - var secondaryServiceCIDR *net.IPNet - - // should we start nodeIPAM - if !ccmconfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs { - return nil, false, nil - } - - // failure: bad cidrs in config - clusterCIDRs, dualStack, err := processCIDRs(ccmconfig.ComponentConfig.KubeCloudShared.ClusterCIDR) - if err != nil { - return nil, false, err - } - - // failure: more than one cidr and dual stack is not enabled - if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs)) - } - - // failure: more than one cidr but they are not configured as dual stack - if len(clusterCIDRs) > 1 && !dualStack { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs)) - } - - // failure: more than cidrs is not allowed even with dual stack - if len(clusterCIDRs) > 2 { - return nil, false, fmt.Errorf("len of clusters is:%v > more than max allowed of 2", len(clusterCIDRs)) - } - - // service cidr processing - if len(strings.TrimSpace(nodeipamconfig.ServiceCIDR)) != 0 { - _, serviceCIDR, err = net.ParseCIDR(nodeipamconfig.ServiceCIDR) - if err != nil { - klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.ServiceCIDR, err) - } - } - - if len(strings.TrimSpace(nodeipamconfig.SecondaryServiceCIDR)) != 0 { - _, secondaryServiceCIDR, err = net.ParseCIDR(nodeipamconfig.SecondaryServiceCIDR) - if err != nil { - klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.SecondaryServiceCIDR, err) - } - } - - // the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided - if serviceCIDR != nil && secondaryServiceCIDR != nil { - // should have dual stack flag enabled - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - return nil, false, fmt.Errorf("secondary service cidr is provided and IPv6DualStack feature is not enabled") - } - - // should be dual stack (from different IPFamilies) - dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR}) - if err != nil { - return nil, false, fmt.Errorf("failed to perform dualstack check on serviceCIDR and secondaryServiceCIDR error:%v", err) - } - if !dualstackServiceCIDR { - return nil, false, fmt.Errorf("serviceCIDR and secondaryServiceCIDR are not dualstack (from different IPfamiles)") - } - } - - var nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6 int - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - // only --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 supported with dual stack clusters. - // --node-cidr-mask-size flag is incompatible with dual stack clusters. - nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(nodeipamconfig) - } else { - // only --node-cidr-mask-size supported with single stack clusters. - // --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 flags are incompatible with dual stack clusters. - nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(nodeipamconfig) - } - - if err != nil { - return nil, false, err - } - - // get list of node cidr mask sizes - nodeCIDRMaskSizes := getNodeCIDRMaskSizes(clusterCIDRs, nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6) - - nodeIpamController, err := nodeipamcontroller.NewNodeIpamController( - ctx.InformerFactory.Core().V1().Nodes(), - cloud, - ctx.ClientBuilder.ClientOrDie("node-controller"), - clusterCIDRs, - serviceCIDR, - secondaryServiceCIDR, - nodeCIDRMaskSizes, - ipam.CIDRAllocatorType(ccmconfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType), - ) - if err != nil { - return nil, true, err - } - go nodeIpamController.Run(ctx.Stop) - return nil, true, nil -} - -// processCIDRs is a helper function that works on a comma separated cidrs and returns -// a list of typed cidrs -// a flag if cidrs represents a dual stack -// error if failed to parse any of the cidrs -func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) { - cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",") - - cidrs, err := netutils.ParseCIDRs(cidrsSplit) - if err != nil { - return nil, false, err - } - - // if cidrs has an error then the previous call will fail - // safe to ignore error checking on next call - dualstack, _ := netutils.IsDualStackCIDRs(cidrs) - - return cidrs, dualstack, nil -} - -// setNodeCIDRMaskSizes returns the IPv4 and IPv6 node cidr mask sizes. -// If --node-cidr-mask-size not set, then it will return default IPv4 and IPv6 cidr mask sizes. -func setNodeCIDRMaskSizes(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) { - ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6 - // NodeCIDRMaskSizeIPv4 and NodeCIDRMaskSizeIPv6 can be used only for dual-stack clusters - if cfg.NodeCIDRMaskSizeIPv4 != 0 || cfg.NodeCIDRMaskSizeIPv6 != 0 { - return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 are not allowed with non dual-stack clusters") - } - if cfg.NodeCIDRMaskSize != 0 { - ipv4Mask = int(cfg.NodeCIDRMaskSize) - ipv6Mask = int(cfg.NodeCIDRMaskSize) - } - return ipv4Mask, ipv6Mask, nil -} - -// setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided -// for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided, -// then it will return default IPv4 and IPv6 cidr mask sizes. -func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) { - ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6 - // NodeCIDRMaskSize can be used only for single stack clusters - if cfg.NodeCIDRMaskSize != 0 { - return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size is not allowed with dual-stack clusters") - } - if cfg.NodeCIDRMaskSizeIPv4 != 0 { - ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4) - } - if cfg.NodeCIDRMaskSizeIPv6 != 0 { - ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6) - } - return ipv4Mask, ipv6Mask, nil -} - -// getNodeCIDRMaskSizes is a helper function that helps the generate the node cidr mask -// sizes slice based on the cluster cidr slice -func getNodeCIDRMaskSizes(clusterCIDRs []*net.IPNet, maskSizeIPv4, maskSizeIPv6 int) []int { - nodeMaskCIDRs := make([]int, len(clusterCIDRs)) - - for idx, clusterCIDR := range clusterCIDRs { - if netutils.IsIPv6CIDR(clusterCIDR) { - nodeMaskCIDRs[idx] = maskSizeIPv6 - } else { - nodeMaskCIDRs[idx] = maskSizeIPv4 - } - } - return nodeMaskCIDRs -} diff --git a/cmd/cloud-controller-manager/providers.go b/cmd/cloud-controller-manager/providers.go deleted file mode 100644 index 91a74e758d372..0000000000000 --- a/cmd/cloud-controller-manager/providers.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build !providerless - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The external controller manager is responsible for running controller loops that -// are cloud provider dependent. It uses the API to listen to new events on resources. - -package main - -import ( - // NOTE: Importing all in-tree cloud-providers is not required when - // implementing an out-of-tree cloud-provider. - _ "k8s.io/legacy-cloud-providers/aws" - _ "k8s.io/legacy-cloud-providers/azure" - _ "k8s.io/legacy-cloud-providers/gce" - _ "k8s.io/legacy-cloud-providers/openstack" - _ "k8s.io/legacy-cloud-providers/vsphere" -) diff --git a/cmd/dependencycheck/BUILD b/cmd/dependencycheck/BUILD deleted file mode 100644 index c80b7cf0dba42..0000000000000 --- a/cmd/dependencycheck/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "go_default_library", - srcs = ["dependencycheck.go"], - importpath = "k8s.io/kubernetes/cmd/dependencycheck", - visibility = ["//visibility:private"], -) - -go_binary( - name = "vendorcycle", - embed = [":go_default_library"], - visibility = ["//visibility:public"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/dependencycheck/OWNERS b/cmd/dependencycheck/OWNERS deleted file mode 100644 index 22de439b4edf5..0000000000000 --- a/cmd/dependencycheck/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - hasheddan -approvers: - - bentheelder - - hasheddan - - liggitt diff --git a/cmd/dependencycheck/dependencycheck.go b/cmd/dependencycheck/dependencycheck.go deleted file mode 100644 index 693d87b55dece..0000000000000 --- a/cmd/dependencycheck/dependencycheck.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Checks for restricted dependencies in go packages. Does not check transitive -// dependencies implicitly, so they must be supplied in dependencies file if -// they are to be evaluated. -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "log" - "regexp" -) - -var ( - exclude = flag.String("exclude", "", "skip packages regex pattern (e.g. '^k8s.io/kubernetes/')") - restrict = flag.String("restrict", "", "restricted dependencies regex pattern (e.g. '^k8s.io/(apimachinery|client-go)/')") -) - -type goPackage struct { - Name string - ImportPath string - Imports []string - TestImports []string - XTestImports []string -} - -func main() { - flag.Parse() - - args := flag.Args() - - if len(args) != 1 { - log.Fatalf("usage: dependencycheck (e.g. 'go list -mod=vendor -test -deps -json ./vendor/...')") - } - if *restrict == "" { - log.Fatalf("Must specify restricted regex pattern") - } - depsPattern, err := regexp.Compile(*restrict) - if err != nil { - log.Fatalf("Error compiling restricted dependencies regex: %v", err) - } - var excludePattern *regexp.Regexp - if *exclude != "" { - excludePattern, err = regexp.Compile(*exclude) - if err != nil { - log.Fatalf("Error compiling excluded package regex: %v", err) - } - } - b, err := ioutil.ReadFile(args[0]) - if err != nil { - log.Fatalf("Error reading dependencies file: %v", err) - } - - packages := []goPackage{} - decoder := json.NewDecoder(bytes.NewBuffer(b)) - for { - pkg := goPackage{} - if err := decoder.Decode(&pkg); err != nil { - if err == io.EOF { - break - } - log.Fatalf("Error unmarshaling dependencies file: %v", err) - } - packages = append(packages, pkg) - } - - violations := map[string][]string{} - for _, p := range packages { - if excludePattern != nil && excludePattern.MatchString(p.ImportPath) { - continue - } - importViolations := []string{} - allImports := []string{} - allImports = append(allImports, p.Imports...) - allImports = append(allImports, p.TestImports...) - allImports = append(allImports, p.XTestImports...) - for _, i := range allImports { - if depsPattern.MatchString(i) { - importViolations = append(importViolations, i) - } - } - if len(importViolations) > 0 { - violations[p.ImportPath] = importViolations - } - } - - if len(violations) > 0 { - for k, v := range violations { - fmt.Printf("Found dependency violations in package %s:\n", k) - for _, a := range v { - fmt.Println("--> " + a) - } - } - log.Fatal("Found restricted dependency violations in packages") - } -} diff --git a/cmd/gendocs/BUILD b/cmd/gendocs/BUILD deleted file mode 100644 index 5988ad15ecee7..0000000000000 --- a/cmd/gendocs/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_binary( - name = "gendocs", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["gen_kubectl_docs.go"], - importpath = "k8s.io/kubernetes/cmd/gendocs", - deps = [ - "//cmd/genutils:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/cmd:go_default_library", - "//vendor/github.com/spf13/cobra/doc:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/gendocs/gen_kubectl_docs.go b/cmd/gendocs/gen_kubectl_docs.go deleted file mode 100644 index fe0bcff123ba9..0000000000000 --- a/cmd/gendocs/gen_kubectl_docs.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "io/ioutil" - "os" - - "github.com/spf13/cobra/doc" - "k8s.io/kubectl/pkg/cmd" - "k8s.io/kubernetes/cmd/genutils" -) - -func main() { - // use os.Args instead of "flags" because "flags" will mess up the man pages! - path := "docs/" - if len(os.Args) == 2 { - path = os.Args[1] - } else if len(os.Args) > 2 { - fmt.Fprintf(os.Stderr, "usage: %s [output directory]\n", os.Args[0]) - os.Exit(1) - } - - outDir, err := genutils.OutDir(path) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err) - os.Exit(1) - } - - // Set environment variables used by kubectl so the output is consistent, - // regardless of where we run. - os.Setenv("HOME", "/home/username") - // TODO os.Stdin should really be something like ioutil.Discard, but a Reader - kubectl := cmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) - doc.GenMarkdownTree(kubectl, outDir) -} diff --git a/cmd/genkubedocs/BUILD b/cmd/genkubedocs/BUILD deleted file mode 100644 index 2c3dab43b249c..0000000000000 --- a/cmd/genkubedocs/BUILD +++ /dev/null @@ -1,56 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_binary( - name = "genkubedocs", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = [ - "gen_kube_docs.go", - "postprocessing.go", - ], - importpath = "k8s.io/kubernetes/cmd/genkubedocs", - deps = [ - "//cmd/genutils:go_default_library", - "//cmd/kube-apiserver/app:go_default_library", - "//cmd/kube-controller-manager/app:go_default_library", - "//cmd/kube-proxy/app:go_default_library", - "//cmd/kube-scheduler/app:go_default_library", - "//cmd/kubeadm/app/cmd:go_default_library", - "//cmd/kubelet/app:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/cobra/doc:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["postprocessing_test.go"], - embed = [":go_default_library"], -) diff --git a/cmd/genkubedocs/gen_kube_docs.go b/cmd/genkubedocs/gen_kube_docs.go deleted file mode 100644 index 29d68f0f43b90..0000000000000 --- a/cmd/genkubedocs/gen_kube_docs.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - - "github.com/spf13/cobra/doc" - "github.com/spf13/pflag" - "k8s.io/kubernetes/cmd/genutils" - apiservapp "k8s.io/kubernetes/cmd/kube-apiserver/app" - cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app" - proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" - schapp "k8s.io/kubernetes/cmd/kube-scheduler/app" - kubeadmapp "k8s.io/kubernetes/cmd/kubeadm/app/cmd" - kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" -) - -func main() { - // use os.Args instead of "flags" because "flags" will mess up the man pages! - path := "" - module := "" - if len(os.Args) == 3 { - path = os.Args[1] - module = os.Args[2] - } else { - fmt.Fprintf(os.Stderr, "usage: %s [output directory] [module] \n", os.Args[0]) - os.Exit(1) - } - - outDir, err := genutils.OutDir(path) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err) - os.Exit(1) - } - - switch module { - case "kube-apiserver": - // generate docs for kube-apiserver - apiserver := apiservapp.NewAPIServerCommand() - doc.GenMarkdownTree(apiserver, outDir) - case "kube-controller-manager": - // generate docs for kube-controller-manager - controllermanager := cmapp.NewControllerManagerCommand() - doc.GenMarkdownTree(controllermanager, outDir) - case "kube-proxy": - // generate docs for kube-proxy - proxy := proxyapp.NewProxyCommand() - doc.GenMarkdownTree(proxy, outDir) - case "kube-scheduler": - // generate docs for kube-scheduler - scheduler := schapp.NewSchedulerCommand() - doc.GenMarkdownTree(scheduler, outDir) - case "kubelet": - // generate docs for kubelet - kubelet := kubeletapp.NewKubeletCommand() - doc.GenMarkdownTree(kubelet, outDir) - case "kubeadm": - // resets global flags created by kubelet or other commands e.g. - // --azure-container-registry-config from pkg/credentialprovider/azure - // --version pkg/version/verflag - pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) - - // generate docs for kubeadm - kubeadm := kubeadmapp.NewKubeadmCommand(os.Stdin, os.Stdout, os.Stderr) - doc.GenMarkdownTree(kubeadm, outDir) - - // cleanup generated code for usage as include in the website - MarkdownPostProcessing(kubeadm, outDir, cleanupForInclude) - default: - fmt.Fprintf(os.Stderr, "Module %s is not supported", module) - os.Exit(1) - } -} diff --git a/cmd/genkubedocs/postprocessing.go b/cmd/genkubedocs/postprocessing.go deleted file mode 100644 index c51e1443bc38d..0000000000000 --- a/cmd/genkubedocs/postprocessing.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "io/ioutil" - "path/filepath" - "strings" - - "github.com/spf13/cobra" -) - -// MarkdownPostProcessing goes though the generated files -func MarkdownPostProcessing(cmd *cobra.Command, dir string, processor func(string) string) error { - for _, c := range cmd.Commands() { - if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { - continue - } - if err := MarkdownPostProcessing(c, dir, processor); err != nil { - return err - } - } - - basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".md" - filename := filepath.Join(dir, basename) - - markdownBytes, err := ioutil.ReadFile(filename) - if err != nil { - return err - } - - processedMarkDown := processor(string(markdownBytes)) - - return ioutil.WriteFile(filename, []byte(processedMarkDown), 0644) -} - -// cleanupForInclude parts of markdown that will make difficult to use it as include in the website: -// - The title of the document (this allow more flexibility for include, e.g. include in tabs) -// - The sections see also, that assumes file will be used as a main page -func cleanupForInclude(md string) string { - lines := strings.Split(md, "\n") - - cleanMd := "" - for i, line := range lines { - if i == 0 { - continue - } - if line == "### SEE ALSO" { - break - } - - cleanMd += line - if i < len(lines)-1 { - cleanMd += "\n" - } - } - - return cleanMd -} diff --git a/cmd/genkubedocs/postprocessing_test.go b/cmd/genkubedocs/postprocessing_test.go deleted file mode 100644 index 66ef3c76fdb06..0000000000000 --- a/cmd/genkubedocs/postprocessing_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestCleanupForInclude(t *testing.T) { - - var tests = []struct { - markdown, expectedMarkdown string - }{ - { // first line is removed - // Nb. first line is the title of the document, and by removing it you get - // more flexibility for include, e.g. include in tabs - markdown: "line 1\n" + - "line 2\n" + - "line 3", - expectedMarkdown: "line 2\n" + - "line 3", - }, - { // everything after ###SEE ALSO is removed - // Nb. see also, that assumes file will be used as a main page (does not apply to includes) - markdown: "line 1\n" + - "line 2\n" + - "### SEE ALSO\n" + - "line 3", - expectedMarkdown: "line 2\n", - }, - } - for _, rt := range tests { - actual := cleanupForInclude(rt.markdown) - if actual != rt.expectedMarkdown { - t.Errorf( - "failed cleanupForInclude:\n\texpected: %s\n\t actual: %s", - rt.expectedMarkdown, - actual, - ) - } - } - -} diff --git a/cmd/genman/BUILD b/cmd/genman/BUILD deleted file mode 100644 index 2e372b5aae7f8..0000000000000 --- a/cmd/genman/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_binary( - name = "genman", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["gen_kube_man.go"], - importpath = "k8s.io/kubernetes/cmd/genman", - deps = [ - "//cmd/genutils:go_default_library", - "//cmd/kube-apiserver/app:go_default_library", - "//cmd/kube-controller-manager/app:go_default_library", - "//cmd/kube-proxy/app:go_default_library", - "//cmd/kube-scheduler/app:go_default_library", - "//cmd/kubeadm/app/cmd:go_default_library", - "//cmd/kubelet/app:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/cmd:go_default_library", - "//vendor/github.com/cpuguy83/go-md2man/v2/md2man:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/genman/gen_kube_man.go b/cmd/genman/gen_kube_man.go deleted file mode 100644 index 9a95e76217a75..0000000000000 --- a/cmd/genman/gen_kube_man.go +++ /dev/null @@ -1,222 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "strings" - - mangen "github.com/cpuguy83/go-md2man/v2/md2man" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - kubectlcmd "k8s.io/kubectl/pkg/cmd" - "k8s.io/kubernetes/cmd/genutils" - apiservapp "k8s.io/kubernetes/cmd/kube-apiserver/app" - cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app" - proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" - schapp "k8s.io/kubernetes/cmd/kube-scheduler/app" - kubeadmapp "k8s.io/kubernetes/cmd/kubeadm/app/cmd" - kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" -) - -func main() { - // use os.Args instead of "flags" because "flags" will mess up the man pages! - path := "docs/man/man1" - module := "" - if len(os.Args) == 3 { - path = os.Args[1] - module = os.Args[2] - } else { - fmt.Fprintf(os.Stderr, "usage: %s [output directory] [module] \n", os.Args[0]) - os.Exit(1) - } - - outDir, err := genutils.OutDir(path) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err) - os.Exit(1) - } - - // Set environment variables used by command so the output is consistent, - // regardless of where we run. - os.Setenv("HOME", "/home/username") - - switch module { - case "kube-apiserver": - // generate manpage for kube-apiserver - apiserver := apiservapp.NewAPIServerCommand() - genMarkdown(apiserver, "", outDir) - for _, c := range apiserver.Commands() { - genMarkdown(c, "kube-apiserver", outDir) - } - case "kube-controller-manager": - // generate manpage for kube-controller-manager - controllermanager := cmapp.NewControllerManagerCommand() - genMarkdown(controllermanager, "", outDir) - for _, c := range controllermanager.Commands() { - genMarkdown(c, "kube-controller-manager", outDir) - } - case "kube-proxy": - // generate manpage for kube-proxy - proxy := proxyapp.NewProxyCommand() - genMarkdown(proxy, "", outDir) - for _, c := range proxy.Commands() { - genMarkdown(c, "kube-proxy", outDir) - } - case "kube-scheduler": - // generate manpage for kube-scheduler - scheduler := schapp.NewSchedulerCommand() - genMarkdown(scheduler, "", outDir) - for _, c := range scheduler.Commands() { - genMarkdown(c, "kube-scheduler", outDir) - } - case "kubelet": - // generate manpage for kubelet - kubelet := kubeletapp.NewKubeletCommand() - genMarkdown(kubelet, "", outDir) - for _, c := range kubelet.Commands() { - genMarkdown(c, "kubelet", outDir) - } - case "kubectl": - // generate manpage for kubectl - // TODO os.Stdin should really be something like ioutil.Discard, but a Reader - kubectl := kubectlcmd.NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) - genMarkdown(kubectl, "", outDir) - for _, c := range kubectl.Commands() { - genMarkdown(c, "kubectl", outDir) - } - case "kubeadm": - // generate manpage for kubelet - kubeadm := kubeadmapp.NewKubeadmCommand(os.Stdin, os.Stdout, os.Stderr) - genMarkdown(kubeadm, "", outDir) - for _, c := range kubeadm.Commands() { - genMarkdown(c, "kubeadm", outDir) - } - default: - fmt.Fprintf(os.Stderr, "Module %s is not supported", module) - os.Exit(1) - } -} - -func preamble(out *bytes.Buffer, name, short, long string) { - out.WriteString(`% KUBERNETES(1) kubernetes User Manuals -% Eric Paris -% Jan 2015 -# NAME -`) - fmt.Fprintf(out, "%s \\- %s\n\n", name, short) - fmt.Fprintf(out, "# SYNOPSIS\n") - fmt.Fprintf(out, "**%s** [OPTIONS]\n\n", name) - fmt.Fprintf(out, "# DESCRIPTION\n") - fmt.Fprintf(out, "%s\n\n", long) -} - -func printFlags(out *bytes.Buffer, flags *pflag.FlagSet) { - flags.VisitAll(func(flag *pflag.Flag) { - format := "**--%s**=%s\n\t%s\n\n" - if flag.Value.Type() == "string" { - // put quotes on the value - format = "**--%s**=%q\n\t%s\n\n" - } - - // Todo, when we mark a shorthand is deprecated, but specify an empty message. - // The flag.ShorthandDeprecated is empty as the shorthand is deprecated. - // Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok. - if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 { - format = "**-%s**, " + format - fmt.Fprintf(out, format, flag.Shorthand, flag.Name, flag.DefValue, flag.Usage) - } else { - fmt.Fprintf(out, format, flag.Name, flag.DefValue, flag.Usage) - } - }) -} - -func printOptions(out *bytes.Buffer, command *cobra.Command) { - flags := command.NonInheritedFlags() - if flags.HasFlags() { - fmt.Fprintf(out, "# OPTIONS\n") - printFlags(out, flags) - fmt.Fprintf(out, "\n") - } - flags = command.InheritedFlags() - if flags.HasFlags() { - fmt.Fprintf(out, "# OPTIONS INHERITED FROM PARENT COMMANDS\n") - printFlags(out, flags) - fmt.Fprintf(out, "\n") - } -} - -func genMarkdown(command *cobra.Command, parent, docsDir string) { - dparent := strings.Replace(parent, " ", "-", -1) - name := command.Name() - dname := name - if len(parent) > 0 { - dname = dparent + "-" + name - name = parent + " " + name - } - - out := new(bytes.Buffer) - short := command.Short - long := command.Long - if len(long) == 0 { - long = short - } - - preamble(out, name, short, long) - printOptions(out, command) - - if len(command.Example) > 0 { - fmt.Fprintf(out, "# EXAMPLE\n") - fmt.Fprintf(out, "```\n%s\n```\n", command.Example) - } - - if len(command.Commands()) > 0 || len(parent) > 0 { - fmt.Fprintf(out, "# SEE ALSO\n") - if len(parent) > 0 { - fmt.Fprintf(out, "**%s(1)**, ", dparent) - } - for _, c := range command.Commands() { - fmt.Fprintf(out, "**%s-%s(1)**, ", dname, c.Name()) - genMarkdown(c, name, docsDir) - } - fmt.Fprintf(out, "\n") - } - - out.WriteString(` -# HISTORY -January 2015, Originally compiled by Eric Paris (eparis at redhat dot com) based on the kubernetes source material, but hopefully they have been automatically generated since! -`) - - final := mangen.Render(out.Bytes()) - - filename := docsDir + dname + ".1" - outFile, err := os.Create(filename) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - defer outFile.Close() - _, err = outFile.Write(final) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - -} diff --git a/cmd/genswaggertypedocs/BUILD b/cmd/genswaggertypedocs/BUILD deleted file mode 100644 index 4630c3f075963..0000000000000 --- a/cmd/genswaggertypedocs/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_binary( - name = "genswaggertypedocs", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["swagger_type_docs.go"], - importpath = "k8s.io/kubernetes/cmd/genswaggertypedocs", - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/genswaggertypedocs/swagger_type_docs.go b/cmd/genswaggertypedocs/swagger_type_docs.go deleted file mode 100644 index 699173c861454..0000000000000 --- a/cmd/genswaggertypedocs/swagger_type_docs.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "io" - "os" - - kruntime "k8s.io/apimachinery/pkg/runtime" - - flag "github.com/spf13/pflag" - "k8s.io/klog/v2" -) - -var ( - functionDest = flag.StringP("func-dest", "f", "-", "Output for swagger functions; '-' means stdout (default)") - typeSrc = flag.StringP("type-src", "s", "", "From where we are going to read the types") - verify = flag.BoolP("verify", "v", false, "Verifies if the given type-src file has documentation for every type") -) - -func main() { - flag.Parse() - - if *typeSrc == "" { - klog.Fatalf("Please define -s flag as it is the source file") - } - - var funcOut io.Writer - if *functionDest == "-" { - funcOut = os.Stdout - } else { - file, err := os.Create(*functionDest) - if err != nil { - klog.Fatalf("Couldn't open %v: %v", *functionDest, err) - } - defer file.Close() - funcOut = file - } - - docsForTypes := kruntime.ParseDocumentationFrom(*typeSrc) - - if *verify { - rc, err := kruntime.VerifySwaggerDocsExist(docsForTypes, funcOut) - if err != nil { - fmt.Fprintf(os.Stderr, "Error in verification process: %s\n", err) - } - os.Exit(rc) - } - - if len(docsForTypes) > 0 { - if err := kruntime.WriteSwaggerDocFunc(docsForTypes, funcOut); err != nil { - fmt.Fprintf(os.Stderr, "Error when writing swagger documentation functions: %s\n", err) - os.Exit(-1) - } - } -} diff --git a/cmd/genutils/BUILD b/cmd/genutils/BUILD deleted file mode 100644 index 19d92efce5989..0000000000000 --- a/cmd/genutils/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["genutils.go"], - importpath = "k8s.io/kubernetes/cmd/genutils", -) - -go_test( - name = "go_default_test", - srcs = ["genutils_test.go"], - embed = [":go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/genutils/genutils.go b/cmd/genutils/genutils.go deleted file mode 100644 index c94b25db547bf..0000000000000 --- a/cmd/genutils/genutils.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package genutils - -import ( - "fmt" - "os" - "path/filepath" -) - -// OutDir creates the absolute path name from path and checks path exists. -// Returns absolute path including trailing '/' or error if path does not exist. -func OutDir(path string) (string, error) { - outDir, err := filepath.Abs(path) - if err != nil { - return "", err - } - - stat, err := os.Stat(outDir) - if err != nil { - return "", err - } - - if !stat.IsDir() { - return "", fmt.Errorf("output directory %s is not a directory", outDir) - } - outDir = outDir + "/" - return outDir, nil -} diff --git a/cmd/genutils/genutils_test.go b/cmd/genutils/genutils_test.go deleted file mode 100644 index 6cc63ee9968cc..0000000000000 --- a/cmd/genutils/genutils_test.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package genutils - -import ( - "testing" -) - -func TestValidDir(t *testing.T) { - _, err := OutDir("./") - if err != nil { - t.Fatal(err) - } -} - -func TestInvalidDir(t *testing.T) { - _, err := OutDir("./nondir") - if err == nil { - t.Fatal("expected an error") - } -} - -func TestNotDir(t *testing.T) { - _, err := OutDir("./genutils_test.go") - if err == nil { - t.Fatal("expected an error") - } -} diff --git a/cmd/genyaml/BUILD b/cmd/genyaml/BUILD deleted file mode 100644 index d9313554a3224..0000000000000 --- a/cmd/genyaml/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_binary( - name = "genyaml", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["gen_kubectl_yaml.go"], - importpath = "k8s.io/kubernetes/cmd/genyaml", - deps = [ - "//cmd/genutils:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/cmd:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/gopkg.in/yaml.v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/genyaml/gen_kubectl_yaml.go b/cmd/genyaml/gen_kubectl_yaml.go deleted file mode 100644 index 4f3c2159cb1a8..0000000000000 --- a/cmd/genyaml/gen_kubectl_yaml.go +++ /dev/null @@ -1,168 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "gopkg.in/yaml.v2" - "k8s.io/kubectl/pkg/cmd" - "k8s.io/kubernetes/cmd/genutils" -) - -type cmdOption struct { - Name string - Shorthand string `yaml:",omitempty"` - DefaultValue string `yaml:"default_value,omitempty"` - Usage string `yaml:",omitempty"` -} - -type cmdDoc struct { - Name string - Synopsis string `yaml:",omitempty"` - Description string `yaml:",omitempty"` - Options []cmdOption `yaml:",omitempty"` - InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"` - Example string `yaml:",omitempty"` - SeeAlso []string `yaml:"see_also,omitempty"` -} - -func main() { - path := "docs/yaml/kubectl" - if len(os.Args) == 2 { - path = os.Args[1] - } else if len(os.Args) > 2 { - fmt.Fprintf(os.Stderr, "usage: %s [output directory]\n", os.Args[0]) - os.Exit(1) - } - - outDir, err := genutils.OutDir(path) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err) - os.Exit(1) - } - - // Set environment variables used by kubectl so the output is consistent, - // regardless of where we run. - os.Setenv("HOME", "/home/username") - kubectl := cmd.NewKubectlCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard) - genYaml(kubectl, "", outDir) - for _, c := range kubectl.Commands() { - genYaml(c, "kubectl", outDir) - } -} - -// Temporary workaround for yaml lib generating incorrect yaml with long strings -// that do not contain \n. -func forceMultiLine(s string) string { - if len(s) > 60 && !strings.Contains(s, "\n") { - s = s + "\n" - } - return s -} - -func genFlagResult(flags *pflag.FlagSet) []cmdOption { - result := []cmdOption{} - - flags.VisitAll(func(flag *pflag.Flag) { - // Todo, when we mark a shorthand is deprecated, but specify an empty message. - // The flag.ShorthandDeprecated is empty as the shorthand is deprecated. - // Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok. - if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 { - opt := cmdOption{ - flag.Name, - flag.Shorthand, - flag.DefValue, - forceMultiLine(flag.Usage), - } - result = append(result, opt) - } else { - opt := cmdOption{ - Name: flag.Name, - DefaultValue: forceMultiLine(flag.DefValue), - Usage: forceMultiLine(flag.Usage), - } - result = append(result, opt) - } - }) - - return result -} - -func genYaml(command *cobra.Command, parent, docsDir string) { - doc := cmdDoc{} - - doc.Name = command.Name() - doc.Synopsis = forceMultiLine(command.Short) - doc.Description = forceMultiLine(command.Long) - - flags := command.NonInheritedFlags() - if flags.HasFlags() { - doc.Options = genFlagResult(flags) - } - flags = command.InheritedFlags() - if flags.HasFlags() { - doc.InheritedOptions = genFlagResult(flags) - } - - if len(command.Example) > 0 { - doc.Example = command.Example - } - - if len(command.Commands()) > 0 || len(parent) > 0 { - result := []string{} - if len(parent) > 0 { - result = append(result, parent) - } - for _, c := range command.Commands() { - result = append(result, c.Name()) - } - doc.SeeAlso = result - } - - final, err := yaml.Marshal(&doc) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - var filename string - - if parent == "" { - filename = docsDir + doc.Name + ".yaml" - } else { - filename = docsDir + parent + "_" + doc.Name + ".yaml" - } - - outFile, err := os.Create(filename) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - defer outFile.Close() - _, err = outFile.Write(final) - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/cmd/importverifier/BUILD b/cmd/importverifier/BUILD deleted file mode 100644 index 1d4d8e2effc91..0000000000000 --- a/cmd/importverifier/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) - -go_binary( - name = "importverifier", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["importverifier.go"], - importpath = "k8s.io/kubernetes/cmd/importverifier", - deps = ["//vendor/gopkg.in/yaml.v2:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/importverifier/OWNERS b/cmd/importverifier/OWNERS deleted file mode 100644 index 759919f3ac1a2..0000000000000 --- a/cmd/importverifier/OWNERS +++ /dev/null @@ -1,10 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - stevekuznetsov - - deads2k - - sttts -approvers: - - stevekuznetsov - - deads2k - - sttts diff --git a/cmd/importverifier/importverifier.go b/cmd/importverifier/importverifier.go deleted file mode 100644 index c73dad68c2a5f..0000000000000 --- a/cmd/importverifier/importverifier.go +++ /dev/null @@ -1,278 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "gopkg.in/yaml.v2" -) - -// Package is a subset of cmd/go.Package -type Package struct { - Dir string `yaml:",omitempty"` // directory containing package sources - ImportPath string `yaml:",omitempty"` // import path of package in dir - Imports []string `yaml:",omitempty"` // import paths used by this package - TestImports []string `yaml:",omitempty"` // imports from TestGoFiles - XTestImports []string `yaml:",omitempty"` // imports from XTestGoFiles -} - -// ImportRestriction describes a set of allowable import -// trees for a tree of source code -type ImportRestriction struct { - // BaseDir is the root of the package tree that is - // restricted by this configuration, given as a - // relative path from the root of the repository - BaseDir string `yaml:"baseImportPath"` - // IgnoredSubTrees are roots of sub-trees of the - // BaseDir for which we do not want to enforce - // any import restrictions whatsoever, given as - // relative paths from the root of the repository - IgnoredSubTrees []string `yaml:"ignoredSubTrees,omitempty"` - // AllowedImports are roots of package trees that - // are allowed to be imported from the BaseDir, - // given as paths that would be used in a Go - // import statement - AllowedImports []string `yaml:"allowedImports"` - // ExcludeTests will skip checking test dependencies. - ExcludeTests bool `yaml:"excludeTests"` -} - -// ForbiddenImportsFor determines all of the forbidden -// imports for a package given the import restrictions -func (i *ImportRestriction) ForbiddenImportsFor(pkg Package) ([]string, error) { - if restricted, err := i.isRestrictedDir(pkg.Dir); err != nil { - return []string{}, err - } else if !restricted { - return []string{}, nil - } - - return i.forbiddenImportsFor(pkg), nil -} - -// isRestrictedDir determines if the source directory has -// any restrictions placed on it by this configuration. -// A path will be restricted if: -// - it falls under the base import path -// - it does not fall under any of the ignored sub-trees -func (i *ImportRestriction) isRestrictedDir(dir string) (bool, error) { - if under, err := isPathUnder(i.BaseDir, dir); err != nil { - return false, err - } else if !under { - return false, nil - } - - for _, ignored := range i.IgnoredSubTrees { - if under, err := isPathUnder(ignored, dir); err != nil { - return false, err - } else if under { - return false, nil - } - } - - return true, nil -} - -// isPathUnder determines if path is under base -func isPathUnder(base, path string) (bool, error) { - absBase, err := filepath.Abs(base) - if err != nil { - return false, err - } - absPath, err := filepath.Abs(path) - if err != nil { - return false, err - } - - relPath, err := filepath.Rel(absBase, absPath) - if err != nil { - return false, err - } - - // if path is below base, the relative path - // from base to path will not start with `../` - return !strings.HasPrefix(relPath, ".."), nil -} - -// forbiddenImportsFor determines all of the forbidden -// imports for a package given the import restrictions -// and returns a deduplicated list of them -func (i *ImportRestriction) forbiddenImportsFor(pkg Package) []string { - forbiddenImportSet := map[string]struct{}{} - imports := pkg.Imports - if !i.ExcludeTests { - imports = append(imports, append(pkg.TestImports, pkg.XTestImports...)...) - } - for _, imp := range imports { - path := extractVendorPath(imp) - if i.isForbidden(path) { - forbiddenImportSet[path] = struct{}{} - } - } - - var forbiddenImports []string - for imp := range forbiddenImportSet { - forbiddenImports = append(forbiddenImports, imp) - } - return forbiddenImports -} - -// extractVendorPath removes a vendor prefix if one exists -func extractVendorPath(path string) string { - vendorPath := "/vendor/" - if !strings.Contains(path, vendorPath) { - return path - } - - return path[strings.Index(path, vendorPath)+len(vendorPath):] -} - -// isForbidden determines if an import is forbidden, -// which is true when the import is: -// - of a package under the rootPackage -// - is not of the base import path or a sub-package of it -// - is not of an allowed path or a sub-package of one -func (i *ImportRestriction) isForbidden(imp string) bool { - importsBelowRoot := strings.HasPrefix(imp, rootPackage) - importsBelowBase := strings.HasPrefix(imp, i.BaseDir) - importsAllowed := false - for _, allowed := range i.AllowedImports { - exactlyImportsAllowed := imp == allowed - importsBelowAllowed := strings.HasPrefix(imp, fmt.Sprintf("%s/", allowed)) - importsAllowed = importsAllowed || (importsBelowAllowed || exactlyImportsAllowed) - } - - return importsBelowRoot && !importsBelowBase && !importsAllowed -} - -var rootPackage string - -func main() { - if len(os.Args) != 3 { - log.Fatalf("Usage: %s ROOT RESTRICTIONS.yaml", os.Args[0]) - } - - rootPackage = os.Args[1] - configFile := os.Args[2] - importRestrictions, err := loadImportRestrictions(configFile) - if err != nil { - log.Fatalf("Failed to load import restrictions: %v", err) - } - - foundForbiddenImports := false - for _, restriction := range importRestrictions { - log.Printf("Inspecting imports under %s...\n", restriction.BaseDir) - packages, err := resolvePackageTree(restriction.BaseDir) - if err != nil { - log.Fatalf("Failed to resolve package tree: %v", err) - } else if len(packages) == 0 { - log.Fatalf("Found no packages under tree %s", restriction.BaseDir) - } - - log.Printf("- validating imports for %d packages in the tree", len(packages)) - restrictionViolated := false - for _, pkg := range packages { - if forbidden, err := restriction.ForbiddenImportsFor(pkg); err != nil { - log.Fatalf("-- failed to validate imports: %v", err) - } else if len(forbidden) != 0 { - logForbiddenPackages(pkg.ImportPath, forbidden) - restrictionViolated = true - } - } - if restrictionViolated { - foundForbiddenImports = true - log.Println("- FAIL") - } else { - log.Println("- OK") - } - } - - if foundForbiddenImports { - os.Exit(1) - } -} - -func loadImportRestrictions(configFile string) ([]ImportRestriction, error) { - config, err := ioutil.ReadFile(configFile) - if err != nil { - return nil, fmt.Errorf("failed to load configuration from %s: %v", configFile, err) - } - - var importRestrictions []ImportRestriction - if err := yaml.Unmarshal(config, &importRestrictions); err != nil { - return nil, fmt.Errorf("failed to unmarshal from %s: %v", configFile, err) - } - - return importRestrictions, nil -} - -func resolvePackageTree(treeBase string) ([]Package, error) { - cmd := "go" - args := []string{"list", "-json", fmt.Sprintf("%s...", treeBase)} - stdout, err := exec.Command(cmd, args...).Output() - if err != nil { - var message string - if ee, ok := err.(*exec.ExitError); ok { - message = fmt.Sprintf("%v\n%v", ee, string(ee.Stderr)) - } else { - message = fmt.Sprintf("%v", err) - } - return nil, fmt.Errorf("failed to run `%s %s`: %v", cmd, strings.Join(args, " "), message) - } - - packages, err := decodePackages(bytes.NewReader(stdout)) - if err != nil { - return nil, fmt.Errorf("failed to decode packages: %v", err) - } - - return packages, nil -} - -func decodePackages(r io.Reader) ([]Package, error) { - // `go list -json` concatenates package definitions - // instead of emitting a single valid JSON, so we - // need to stream the output to decode it into the - // data we are looking for instead of just using a - // simple JSON decoder on stdout - var packages []Package - decoder := json.NewDecoder(r) - for decoder.More() { - var pkg Package - if err := decoder.Decode(&pkg); err != nil { - return nil, fmt.Errorf("invalid package: %v", err) - } - packages = append(packages, pkg) - } - - return packages, nil -} - -func logForbiddenPackages(base string, forbidden []string) { - log.Printf("-- found forbidden imports for %s:\n", base) - for _, forbiddenPackage := range forbidden { - log.Printf("--- %s\n", forbiddenPackage) - } -} diff --git a/cmd/kube-apiserver/BUILD b/cmd/kube-apiserver/BUILD deleted file mode 100644 index 428ccd126e818..0000000000000 --- a/cmd/kube-apiserver/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kube-apiserver", - embed = [":go_default_library"], - pure = "on", - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["apiserver.go"], - importpath = "k8s.io/kubernetes/cmd/kube-apiserver", - deps = [ - "//cmd/kube-apiserver/app:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-apiserver/app:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kube-apiserver/OWNERS b/cmd/kube-apiserver/OWNERS deleted file mode 100644 index 88bff5fda7a34..0000000000000 --- a/cmd/kube-apiserver/OWNERS +++ /dev/null @@ -1,30 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- caesarxuchao -- deads2k -- lavalamp -- liggitt -- mml -- nikhiljindal -- smarterclayton -- sttts -reviewers: -- lavalamp -- cheftako -- smarterclayton -- wojtek-t -- deads2k -- derekwaynecarr -- caesarxuchao -- mikedanese -- liggitt -- nikhiljindal -- ncdc -- sttts -- hzxuzhonghu -- logicalhan -- yue9944882 -labels: -- sig/api-machinery -- area/apiserver diff --git a/cmd/kube-apiserver/apiserver.go b/cmd/kube-apiserver/apiserver.go deleted file mode 100644 index 97ac595d836eb..0000000000000 --- a/cmd/kube-apiserver/apiserver.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// apiserver is the main api server and master for the cluster. -// it is responsible for serving the cluster management API. -package main - -import ( - "math/rand" - "os" - "time" - - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/kubernetes/cmd/kube-apiserver/app" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := app.NewAPIServerCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD deleted file mode 100644 index bf45bdb09a3fe..0000000000000 --- a/cmd/kube-apiserver/app/BUILD +++ /dev/null @@ -1,102 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "aggregator.go", - "apiextensions.go", - "server.go", - ], - importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kube-apiserver/app/options:go_default_library", - "//pkg/api/legacyscheme:go_default_library", - "//pkg/capabilities:go_default_library", - "//pkg/controlplane:go_default_library", - "//pkg/controlplane/controller/crdregistration:go_default_library", - "//pkg/controlplane/reconcilers:go_default_library", - "//pkg/controlplane/tunneler:go_default_library", - "//pkg/features:go_default_library", - "//pkg/generated/openapi:go_default_library", - "//pkg/kubeapiserver:go_default_library", - "//pkg/kubeapiserver/admission:go_default_library", - "//pkg/kubeapiserver/authenticator:go_default_library", - "//pkg/kubeapiserver/authorizer/modes:go_default_library", - "//pkg/registry/rbac/rest:go_default_library", - "//pkg/serviceaccount:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/egressselector:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/flowcontrol:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//staging/src/k8s.io/cloud-provider:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/workqueue:go_default_library", - "//staging/src/k8s.io/component-base/term:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/apiserver:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["server_test.go"], - embed = [":go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-apiserver/app/options:all-srcs", - "//cmd/kube-apiserver/app/testing:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go deleted file mode 100644 index 4289ce48d8fe6..0000000000000 --- a/cmd/kube-apiserver/app/aggregator.go +++ /dev/null @@ -1,323 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to create a Kubernetes -// APIServer by binding together the API, master and APIServer infrastructure. -// It can be configured and called directly or via the hyperkube framework. -package app - -import ( - "fmt" - "net/http" - "strings" - "sync" - - "k8s.io/klog/v2" - - apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/features" - genericfeatures "k8s.io/apiserver/pkg/features" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/healthz" - genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/util/feature" - utilfeature "k8s.io/apiserver/pkg/util/feature" - kubeexternalinformers "k8s.io/client-go/informers" - "k8s.io/client-go/tools/cache" - v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper" - "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1" - informers "k8s.io/kube-aggregator/pkg/client/informers/externalversions/apiregistration/v1" - "k8s.io/kube-aggregator/pkg/controllers/autoregister" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/pkg/controlplane/controller/crdregistration" -) - -func createAggregatorConfig( - kubeAPIServerConfig genericapiserver.Config, - commandOptions *options.ServerRunOptions, - externalInformers kubeexternalinformers.SharedInformerFactory, - serviceResolver aggregatorapiserver.ServiceResolver, - proxyTransport *http.Transport, - pluginInitializers []admission.PluginInitializer, -) (*aggregatorapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator - genericConfig := kubeAPIServerConfig - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) && - utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) { - // Add StorageVersionPrecondition handler to aggregator-apiserver. - // The handler will block write requests to built-in resources until the - // target resources' storage versions are up-to-date. - genericConfig.BuildHandlerChainFunc = genericapiserver.BuildHandlerChainWithStorageVersionPrecondition - } - - // override genericConfig.AdmissionControl with kube-aggregator's scheme, - // because aggregator apiserver should use its own scheme to convert its own resources. - err := commandOptions.Admission.ApplyTo( - &genericConfig, - externalInformers, - genericConfig.LoopbackClientConfig, - feature.DefaultFeatureGate, - pluginInitializers...) - if err != nil { - return nil, err - } - - // copy the etcd options so we don't mutate originals. - etcdOptions := *commandOptions.Etcd - etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) - etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) - genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} - - // override MergedResourceConfig with aggregator defaults and registry - if err := commandOptions.APIEnablement.ApplyTo( - &genericConfig, - aggregatorapiserver.DefaultAPIResourceConfigSource(), - aggregatorscheme.Scheme); err != nil { - return nil, err - } - - aggregatorConfig := &aggregatorapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: externalInformers, - }, - ExtraConfig: aggregatorapiserver.ExtraConfig{ - ProxyClientCertFile: commandOptions.ProxyClientCertFile, - ProxyClientKeyFile: commandOptions.ProxyClientKeyFile, - ServiceResolver: serviceResolver, - ProxyTransport: proxyTransport, - }, - } - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - aggregatorConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return aggregatorConfig, nil -} - -func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget, apiExtensionInformers apiextensionsinformers.SharedInformerFactory) (*aggregatorapiserver.APIAggregator, error) { - aggregatorServer, err := aggregatorConfig.Complete().NewWithDelegate(delegateAPIServer) - if err != nil { - return nil, err - } - - // create controllers for auto-registration - apiRegistrationClient, err := apiregistrationclient.NewForConfig(aggregatorConfig.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - autoRegistrationController := autoregister.NewAutoRegisterController(aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), apiRegistrationClient) - apiServices := apiServicesToRegister(delegateAPIServer, autoRegistrationController) - crdRegistrationController := crdregistration.NewCRDRegistrationController( - apiExtensionInformers.Apiextensions().V1().CustomResourceDefinitions(), - autoRegistrationController) - - err = aggregatorServer.GenericAPIServer.AddPostStartHook("kube-apiserver-autoregistration", func(context genericapiserver.PostStartHookContext) error { - go crdRegistrationController.Run(5, context.StopCh) - go func() { - // let the CRD controller process the initial set of CRDs before starting the autoregistration controller. - // this prevents the autoregistration controller's initial sync from deleting APIServices for CRDs that still exist. - // we only need to do this if CRDs are enabled on this server. We can't use discovery because we are the source for discovery. - if aggregatorConfig.GenericConfig.MergedResourceConfig.AnyVersionForGroupEnabled("apiextensions.k8s.io") { - crdRegistrationController.WaitForInitialSync() - } - autoRegistrationController.Run(5, context.StopCh) - }() - return nil - }) - if err != nil { - return nil, err - } - - err = aggregatorServer.GenericAPIServer.AddBootSequenceHealthChecks( - makeAPIServiceAvailableHealthCheck( - "autoregister-completion", - apiServices, - aggregatorServer.APIRegistrationInformers.Apiregistration().V1().APIServices(), - ), - ) - if err != nil { - return nil, err - } - - return aggregatorServer, nil -} - -func makeAPIService(gv schema.GroupVersion) *v1.APIService { - apiServicePriority, ok := apiVersionPriorities[gv] - if !ok { - // if we aren't found, then we shouldn't register ourselves because it could result in a CRD group version - // being permanently stuck in the APIServices list. - klog.Infof("Skipping APIService creation for %v", gv) - return nil - } - return &v1.APIService{ - ObjectMeta: metav1.ObjectMeta{Name: gv.Version + "." + gv.Group}, - Spec: v1.APIServiceSpec{ - Group: gv.Group, - Version: gv.Version, - GroupPriorityMinimum: apiServicePriority.group, - VersionPriority: apiServicePriority.version, - }, - } -} - -// makeAPIServiceAvailableHealthCheck returns a healthz check that returns healthy -// once all of the specified services have been observed to be available at least once. -func makeAPIServiceAvailableHealthCheck(name string, apiServices []*v1.APIService, apiServiceInformer informers.APIServiceInformer) healthz.HealthChecker { - // Track the auto-registered API services that have not been observed to be available yet - pendingServiceNamesLock := &sync.RWMutex{} - pendingServiceNames := sets.NewString() - for _, service := range apiServices { - pendingServiceNames.Insert(service.Name) - } - - // When an APIService in the list is seen as available, remove it from the pending list - handleAPIServiceChange := func(service *v1.APIService) { - pendingServiceNamesLock.Lock() - defer pendingServiceNamesLock.Unlock() - if !pendingServiceNames.Has(service.Name) { - return - } - if v1helper.IsAPIServiceConditionTrue(service, v1.Available) { - pendingServiceNames.Delete(service.Name) - } - } - - // Watch add/update events for APIServices - apiServiceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { handleAPIServiceChange(obj.(*v1.APIService)) }, - UpdateFunc: func(old, new interface{}) { handleAPIServiceChange(new.(*v1.APIService)) }, - }) - - // Don't return healthy until the pending list is empty - return healthz.NamedCheck(name, func(r *http.Request) error { - pendingServiceNamesLock.RLock() - defer pendingServiceNamesLock.RUnlock() - if pendingServiceNames.Len() > 0 { - return fmt.Errorf("missing APIService: %v", pendingServiceNames.List()) - } - return nil - }) -} - -// priority defines group priority that is used in discovery. This controls -// group position in the kubectl output. -type priority struct { - // group indicates the order of the group relative to other groups. - group int32 - // version indicates the relative order of the version inside of its group. - version int32 -} - -// The proper way to resolve this letting the aggregator know the desired group and version-within-group order of the underlying servers -// is to refactor the genericapiserver.DelegationTarget to include a list of priorities based on which APIs were installed. -// This requires the APIGroupInfo struct to evolve and include the concept of priorities and to avoid mistakes, the core storage map there needs to be updated. -// That ripples out every bit as far as you'd expect, so for 1.7 we'll include the list here instead of being built up during storage. -var apiVersionPriorities = map[schema.GroupVersion]priority{ - {Group: "", Version: "v1"}: {group: 18000, version: 1}, - // to my knowledge, nothing below here collides - {Group: "apps", Version: "v1"}: {group: 17800, version: 15}, - {Group: "events.k8s.io", Version: "v1"}: {group: 17750, version: 15}, - {Group: "events.k8s.io", Version: "v1beta1"}: {group: 17750, version: 5}, - {Group: "authentication.k8s.io", Version: "v1"}: {group: 17700, version: 15}, - {Group: "authentication.k8s.io", Version: "v1beta1"}: {group: 17700, version: 9}, - {Group: "authorization.k8s.io", Version: "v1"}: {group: 17600, version: 15}, - {Group: "authorization.k8s.io", Version: "v1beta1"}: {group: 17600, version: 9}, - {Group: "autoscaling", Version: "v1"}: {group: 17500, version: 15}, - {Group: "autoscaling", Version: "v2beta1"}: {group: 17500, version: 9}, - {Group: "autoscaling", Version: "v2beta2"}: {group: 17500, version: 1}, - {Group: "batch", Version: "v1"}: {group: 17400, version: 15}, - {Group: "batch", Version: "v1beta1"}: {group: 17400, version: 9}, - {Group: "batch", Version: "v2alpha1"}: {group: 17400, version: 9}, - {Group: "certificates.k8s.io", Version: "v1"}: {group: 17300, version: 15}, - {Group: "certificates.k8s.io", Version: "v1beta1"}: {group: 17300, version: 9}, - {Group: "networking.k8s.io", Version: "v1"}: {group: 17200, version: 15}, - {Group: "networking.k8s.io", Version: "v1beta1"}: {group: 17200, version: 9}, - {Group: "extensions", Version: "v1beta1"}: {group: 17150, version: 1}, // prioritize below networking.k8s.io, which contains the GA version of Ingress, the only resource remaining in extensions/v1beta1 - {Group: "policy", Version: "v1beta1"}: {group: 17100, version: 9}, - {Group: "rbac.authorization.k8s.io", Version: "v1"}: {group: 17000, version: 15}, - {Group: "rbac.authorization.k8s.io", Version: "v1beta1"}: {group: 17000, version: 12}, - {Group: "rbac.authorization.k8s.io", Version: "v1alpha1"}: {group: 17000, version: 9}, - {Group: "storage.k8s.io", Version: "v1"}: {group: 16800, version: 15}, - {Group: "storage.k8s.io", Version: "v1beta1"}: {group: 16800, version: 9}, - {Group: "storage.k8s.io", Version: "v1alpha1"}: {group: 16800, version: 1}, - {Group: "apiextensions.k8s.io", Version: "v1"}: {group: 16700, version: 15}, - {Group: "apiextensions.k8s.io", Version: "v1beta1"}: {group: 16700, version: 9}, - {Group: "admissionregistration.k8s.io", Version: "v1"}: {group: 16700, version: 15}, - {Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {group: 16700, version: 12}, - {Group: "scheduling.k8s.io", Version: "v1"}: {group: 16600, version: 15}, - {Group: "scheduling.k8s.io", Version: "v1beta1"}: {group: 16600, version: 12}, - {Group: "scheduling.k8s.io", Version: "v1alpha1"}: {group: 16600, version: 9}, - {Group: "coordination.k8s.io", Version: "v1"}: {group: 16500, version: 15}, - {Group: "coordination.k8s.io", Version: "v1beta1"}: {group: 16500, version: 9}, - {Group: "node.k8s.io", Version: "v1"}: {group: 16300, version: 15}, - {Group: "node.k8s.io", Version: "v1alpha1"}: {group: 16300, version: 1}, - {Group: "node.k8s.io", Version: "v1beta1"}: {group: 16300, version: 9}, - {Group: "discovery.k8s.io", Version: "v1beta1"}: {group: 16200, version: 12}, - {Group: "discovery.k8s.io", Version: "v1alpha1"}: {group: 16200, version: 9}, - {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta1"}: {group: 16100, version: 12}, - {Group: "flowcontrol.apiserver.k8s.io", Version: "v1alpha1"}: {group: 16100, version: 9}, - {Group: "internal.apiserver.k8s.io", Version: "v1alpha1"}: {group: 16000, version: 9}, - // Append a new group to the end of the list if unsure. - // You can use min(existing group)-100 as the initial value for a group. - // Version can be set to 9 (to have space around) for a new group. -} - -func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*v1.APIService { - apiServices := []*v1.APIService{} - - for _, curr := range delegateAPIServer.ListedPaths() { - if curr == "/api/v1" { - apiService := makeAPIService(schema.GroupVersion{Group: "", Version: "v1"}) - registration.AddAPIServiceToSyncOnStart(apiService) - apiServices = append(apiServices, apiService) - continue - } - - if !strings.HasPrefix(curr, "/apis/") { - continue - } - // this comes back in a list that looks like /apis/rbac.authorization.k8s.io/v1alpha1 - tokens := strings.Split(curr, "/") - if len(tokens) != 4 { - continue - } - - apiService := makeAPIService(schema.GroupVersion{Group: tokens[2], Version: tokens[3]}) - if apiService == nil { - continue - } - registration.AddAPIServiceToSyncOnStart(apiService) - apiServices = append(apiServices, apiService) - } - - return apiServices -} diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go deleted file mode 100644 index 438e8588c2b0e..0000000000000 --- a/cmd/kube-apiserver/app/apiextensions.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to create a Kubernetes -// APIServer by binding together the API, master and APIServer infrastructure. -// It can be configured and called directly or via the hyperkube framework. -package app - -import ( - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/features" - genericapiserver "k8s.io/apiserver/pkg/server" - genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/util/feature" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/apiserver/pkg/util/webhook" - kubeexternalinformers "k8s.io/client-go/informers" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" -) - -func createAPIExtensionsConfig( - kubeAPIServerConfig genericapiserver.Config, - externalInformers kubeexternalinformers.SharedInformerFactory, - pluginInitializers []admission.PluginInitializer, - commandOptions *options.ServerRunOptions, - masterCount int, - serviceResolver webhook.ServiceResolver, - authResolverWrapper webhook.AuthenticationInfoResolverWrapper, -) (*apiextensionsapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions - genericConfig := kubeAPIServerConfig - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - - // override genericConfig.AdmissionControl with apiextensions' scheme, - // because apiextentions apiserver should use its own scheme to convert resources. - err := commandOptions.Admission.ApplyTo( - &genericConfig, - externalInformers, - genericConfig.LoopbackClientConfig, - feature.DefaultFeatureGate, - pluginInitializers...) - if err != nil { - return nil, err - } - - // copy the etcd options so we don't mutate originals. - etcdOptions := *commandOptions.Etcd - etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) - etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) - genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions} - - // override MergedResourceConfig with apiextensions defaults and registry - if err := commandOptions.APIEnablement.ApplyTo( - &genericConfig, - apiextensionsapiserver.DefaultAPIResourceConfigSource(), - apiextensionsapiserver.Scheme); err != nil { - return nil, err - } - - apiextensionsConfig := &apiextensionsapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: externalInformers, - }, - ExtraConfig: apiextensionsapiserver.ExtraConfig{ - CRDRESTOptionsGetter: apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions), - MasterCount: masterCount, - AuthResolverWrapper: authResolverWrapper, - ServiceResolver: serviceResolver, - }, - } - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - apiextensionsConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return apiextensionsConfig, nil -} - -func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) { - return apiextensionsConfig.Complete().New(delegateAPIServer) -} diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD deleted file mode 100644 index 5d7dc156efff0..0000000000000 --- a/cmd/kube-apiserver/app/options/BUILD +++ /dev/null @@ -1,88 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "globalflags.go", - "globalflags_providers.go", - "options.go", - "validation.go", - ], - importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/options", - deps = [ - "//pkg/api/legacyscheme:go_default_library", - "//pkg/apis/core:go_default_library", - "//pkg/cloudprovider/providers:go_default_library", - "//pkg/cluster/ports:go_default_library", - "//pkg/controlplane/reconcilers:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubeapiserver/options:go_default_library", - "//pkg/kubelet/client:go_default_library", - "//pkg/serviceaccount:go_default_library", - "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//staging/src/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "globalflags_test.go", - "options_test.go", - "validation_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//pkg/apis/core:go_default_library", - "//pkg/controlplane/reconcilers:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubeapiserver/options:go_default_library", - "//pkg/kubelet/client:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library", - "//staging/src/k8s.io/apiserver/plugin/pkg/audit/truncate:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", - "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", - "//vendor/github.com/google/go-cmp/cmp/cmpopts:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kube-apiserver/app/options/globalflags.go b/cmd/kube-apiserver/app/options/globalflags.go deleted file mode 100644 index 05c5e14ef4d56..0000000000000 --- a/cmd/kube-apiserver/app/options/globalflags.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - "k8s.io/component-base/cli/globalflag" - - // ensure libs have a chance to globally register their flags - _ "k8s.io/apiserver/pkg/admission" - _ "k8s.io/kubernetes/pkg/cloudprovider/providers" -) - -// AddCustomGlobalFlags explicitly registers flags that internal packages register -// against the global flagsets from "flag". We do this in order to prevent -// unwanted flags from leaking into the kube-apiserver's flagset. -func AddCustomGlobalFlags(fs *pflag.FlagSet) { - // Lookup flags in global flag set and re-register the values with our flagset. - - // Adds flags from k8s.io/kubernetes/pkg/cloudprovider/providers. - registerLegacyGlobalFlags(fs) - - // Adds flags from k8s.io/apiserver/pkg/admission. - globalflag.Register(fs, "default-not-ready-toleration-seconds") - globalflag.Register(fs, "default-unreachable-toleration-seconds") -} diff --git a/cmd/kube-apiserver/app/options/globalflags_providerless.go b/cmd/kube-apiserver/app/options/globalflags_providerless.go deleted file mode 100644 index de00215c414f5..0000000000000 --- a/cmd/kube-apiserver/app/options/globalflags_providerless.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" -) - -func registerLegacyGlobalFlags(fs *pflag.FlagSet) { - // no-op when no legacy providers are compiled in -} diff --git a/cmd/kube-apiserver/app/options/globalflags_providers.go b/cmd/kube-apiserver/app/options/globalflags_providers.go deleted file mode 100644 index 61107cf38076b..0000000000000 --- a/cmd/kube-apiserver/app/options/globalflags_providers.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - "k8s.io/component-base/cli/globalflag" -) - -func registerLegacyGlobalFlags(fs *pflag.FlagSet) { - globalflag.Register(fs, "cloud-provider-gce-lb-src-cidrs") - globalflag.Register(fs, "cloud-provider-gce-l7lb-src-cidrs") - fs.MarkDeprecated("cloud-provider-gce-lb-src-cidrs", "This flag will be removed once the GCE Cloud Provider is removed from kube-apiserver") -} diff --git a/cmd/kube-apiserver/app/options/globalflags_test.go b/cmd/kube-apiserver/app/options/globalflags_test.go deleted file mode 100644 index 7510efc74e4fb..0000000000000 --- a/cmd/kube-apiserver/app/options/globalflags_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "flag" - "reflect" - "sort" - "strings" - "testing" - - "github.com/spf13/pflag" - - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/cli/globalflag" -) - -func TestAddCustomGlobalFlags(t *testing.T) { - namedFlagSets := &cliflag.NamedFlagSets{} - - // Note that we will register all flags (including klog flags) into the same - // flag set. This allows us to test against all global flags from - // flags.CommandLine. - nfs := namedFlagSets.FlagSet("test") - globalflag.AddGlobalFlags(nfs, "test-cmd") - AddCustomGlobalFlags(nfs) - - actualFlag := []string{} - nfs.VisitAll(func(flag *pflag.Flag) { - actualFlag = append(actualFlag, flag.Name) - }) - - // Get all flags from flags.CommandLine, except flag `test.*`. - wantedFlag := []string{"help"} - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.VisitAll(func(flag *pflag.Flag) { - if !strings.Contains(flag.Name, "test.") { - wantedFlag = append(wantedFlag, flag.Name) - } - }) - sort.Strings(wantedFlag) - - if !reflect.DeepEqual(wantedFlag, actualFlag) { - t.Errorf("[Default]: expected %+v, got %+v", wantedFlag, actualFlag) - } -} diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go deleted file mode 100644 index 0d05859d2a76b..0000000000000 --- a/cmd/kube-apiserver/app/options/options.go +++ /dev/null @@ -1,287 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package options contains flags and options for initializing an apiserver -package options - -import ( - "net" - "strings" - "time" - - "github.com/spf13/pflag" - utilnet "k8s.io/apimachinery/pkg/util/net" - genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/storage/storagebackend" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/cluster/ports" - "k8s.io/kubernetes/pkg/controlplane/reconcilers" - _ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates - kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" - kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" - "k8s.io/kubernetes/pkg/serviceaccount" -) - -// InsecurePortFlags are dummy flags, they are kept only for compatibility and will be removed in v1.24. -// TODO: remove these flags in v1.24. -var InsecurePortFlags = []string{"insecure-port", "port"} - -// ServerRunOptions runs a kubernetes api server. -type ServerRunOptions struct { - GenericServerRunOptions *genericoptions.ServerRunOptions - Etcd *genericoptions.EtcdOptions - SecureServing *genericoptions.SecureServingOptionsWithLoopback - Audit *genericoptions.AuditOptions - Features *genericoptions.FeatureOptions - Admission *kubeoptions.AdmissionOptions - Authentication *kubeoptions.BuiltInAuthenticationOptions - Authorization *kubeoptions.BuiltInAuthorizationOptions - CloudProvider *kubeoptions.CloudProviderOptions - APIEnablement *genericoptions.APIEnablementOptions - EgressSelector *genericoptions.EgressSelectorOptions - Metrics *metrics.Options - Logs *logs.Options - - AllowPrivileged bool - EnableLogsHandler bool - EventTTL time.Duration - KubeletConfig kubeletclient.KubeletClientConfig - KubernetesServiceNodePort int - MaxConnectionBytesPerSec int64 - // ServiceClusterIPRange is mapped to input provided by user - ServiceClusterIPRanges string - // PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results - // of parsing ServiceClusterIPRange into actual values - PrimaryServiceClusterIPRange net.IPNet - SecondaryServiceClusterIPRange net.IPNet - - ServiceNodePortRange utilnet.PortRange - SSHKeyfile string - SSHUser string - - ProxyClientCertFile string - ProxyClientKeyFile string - - EnableAggregatorRouting bool - - MasterCount int - EndpointReconcilerType string - - IdentityLeaseDurationSeconds int - IdentityLeaseRenewIntervalSeconds int - - ServiceAccountSigningKeyFile string - ServiceAccountIssuer serviceaccount.TokenGenerator - ServiceAccountTokenMaxExpiration time.Duration - - ShowHiddenMetricsForVersion string -} - -// NewServerRunOptions creates a new ServerRunOptions object with default parameters -func NewServerRunOptions() *ServerRunOptions { - s := ServerRunOptions{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(), - Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), - SecureServing: kubeoptions.NewSecureServingOptions(), - Audit: genericoptions.NewAuditOptions(), - Features: genericoptions.NewFeatureOptions(), - Admission: kubeoptions.NewAdmissionOptions(), - Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(), - Authorization: kubeoptions.NewBuiltInAuthorizationOptions(), - CloudProvider: kubeoptions.NewCloudProviderOptions(), - APIEnablement: genericoptions.NewAPIEnablementOptions(), - EgressSelector: genericoptions.NewEgressSelectorOptions(), - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), - - EnableLogsHandler: true, - EventTTL: 1 * time.Hour, - MasterCount: 1, - EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), - IdentityLeaseDurationSeconds: 3600, - IdentityLeaseRenewIntervalSeconds: 10, - KubeletConfig: kubeletclient.KubeletClientConfig{ - Port: ports.KubeletPort, - ReadOnlyPort: ports.KubeletReadOnlyPort, - PreferredAddressTypes: []string{ - // --override-hostname - string(api.NodeHostName), - - // internal, preferring DNS if reported - string(api.NodeInternalDNS), - string(api.NodeInternalIP), - - // external, preferring DNS if reported - string(api.NodeExternalDNS), - string(api.NodeExternalIP), - }, - HTTPTimeout: time.Duration(5) * time.Second, - }, - ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, - } - - // Overwrite the default for storage data format. - s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" - - return &s -} - -// TODO: remove these insecure flags in v1.24 -func addDummyInsecureFlags(fs *pflag.FlagSet) { - var ( - bindAddr = net.IPv4(127, 0, 0, 1) - bindPort int - ) - - for _, name := range []string{"insecure-bind-address", "address"} { - fs.IPVar(&bindAddr, name, bindAddr, ""+ - "The IP address on which to serve the insecure port (set to 0.0.0.0 or :: for listening in all interfaces and IP families).") - fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.") - } - - for _, name := range InsecurePortFlags { - fs.IntVar(&bindPort, name, bindPort, ""+ - "The port on which to serve unsecured, unauthenticated access.") - fs.MarkDeprecated(name, "This flag has no effect now and will be removed in v1.24.") - } -} - -// Flags returns flags for a specific APIServer by section name -func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { - // Add the generic flags. - s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic")) - s.Etcd.AddFlags(fss.FlagSet("etcd")) - s.SecureServing.AddFlags(fss.FlagSet("secure serving")) - addDummyInsecureFlags(fss.FlagSet("insecure serving")) - s.Audit.AddFlags(fss.FlagSet("auditing")) - s.Features.AddFlags(fss.FlagSet("features")) - s.Authentication.AddFlags(fss.FlagSet("authentication")) - s.Authorization.AddFlags(fss.FlagSet("authorization")) - s.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) - s.APIEnablement.AddFlags(fss.FlagSet("API enablement")) - s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) - s.Admission.AddFlags(fss.FlagSet("admission")) - s.Metrics.AddFlags(fss.FlagSet("metrics")) - s.Logs.AddFlags(fss.FlagSet("logs")) - - // Note: the weird ""+ in below lines seems to be the only way to get gofmt to - // arrange these text blocks sensibly. Grrr. - fs := fss.FlagSet("misc") - fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, - "Amount of time to retain events.") - - fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, - "If true, allow privileged containers. [default=false]") - - fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler, - "If true, install a /logs handler for the apiserver logs.") - fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19") - - // Deprecated in release 1.9 - fs.StringVar(&s.SSHUser, "ssh-user", s.SSHUser, - "If non-empty, use secure SSH proxy to the nodes, using this user name") - fs.MarkDeprecated("ssh-user", "This flag will be removed in a future version.") - - // Deprecated in release 1.9 - fs.StringVar(&s.SSHKeyfile, "ssh-keyfile", s.SSHKeyfile, - "If non-empty, use secure SSH proxy to the nodes, using this user keyfile") - fs.MarkDeprecated("ssh-keyfile", "This flag will be removed in a future version.") - - fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+ - "If non-zero, throttle each user connection to this number of bytes/sec. "+ - "Currently only applies to long-running requests.") - - fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, - "The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)") - - fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", string(s.EndpointReconcilerType), - "Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+")") - - fs.IntVar(&s.IdentityLeaseDurationSeconds, "identity-lease-duration-seconds", s.IdentityLeaseDurationSeconds, - "The duration of kube-apiserver lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.)") - - fs.IntVar(&s.IdentityLeaseRenewIntervalSeconds, "identity-lease-renew-interval-seconds", s.IdentityLeaseRenewIntervalSeconds, - "The interval of kube-apiserver renewing its lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.)") - - // See #14282 for details on how to test/try this option out. - // TODO: remove this comment once this option is tested in CI. - fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+ - "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+ - "of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+ - "service will be of type ClusterIP.") - - // TODO (khenidak) change documentation as we move IPv6DualStack feature from ALPHA to BETA - fs.StringVar(&s.ServiceClusterIPRanges, "service-cluster-ip-range", s.ServiceClusterIPRanges, ""+ - "A CIDR notation IP range from which to assign service cluster IPs. This must not "+ - "overlap with any IP ranges assigned to nodes or pods.") - - fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+ - "A port range to reserve for services with NodePort visibility. "+ - "Example: '30000-32767'. Inclusive at both ends of the range.") - - // Kubelet related flags: - kubeletHTTPS := true - fs.BoolVar(&kubeletHTTPS, "kubelet-https", kubeletHTTPS, "Use https for kubelet connections.") - fs.MarkDeprecated("kubelet-https", "API Server connections to kubelets always use https. This flag will be removed in 1.22.") - - fs.StringSliceVar(&s.KubeletConfig.PreferredAddressTypes, "kubelet-preferred-address-types", s.KubeletConfig.PreferredAddressTypes, - "List of the preferred NodeAddressTypes to use for kubelet connections.") - - fs.UintVar(&s.KubeletConfig.Port, "kubelet-port", s.KubeletConfig.Port, - "DEPRECATED: kubelet port.") - fs.MarkDeprecated("kubelet-port", "kubelet-port is deprecated and will be removed.") - - fs.UintVar(&s.KubeletConfig.ReadOnlyPort, "kubelet-read-only-port", s.KubeletConfig.ReadOnlyPort, - "DEPRECATED: kubelet read only port.") - fs.MarkDeprecated("kubelet-read-only-port", "kubelet-read-only-port is deprecated and will be removed.") - - fs.DurationVar(&s.KubeletConfig.HTTPTimeout, "kubelet-timeout", s.KubeletConfig.HTTPTimeout, - "Timeout for kubelet operations.") - - fs.StringVar(&s.KubeletConfig.CertFile, "kubelet-client-certificate", s.KubeletConfig.CertFile, - "Path to a client cert file for TLS.") - - fs.StringVar(&s.KubeletConfig.KeyFile, "kubelet-client-key", s.KubeletConfig.KeyFile, - "Path to a client key file for TLS.") - - fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile, - "Path to a cert file for the certificate authority.") - - fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+ - "Client certificate used to prove the identity of the aggregator or kube-apiserver "+ - "when it must call out during a request. This includes proxying requests to a user "+ - "api-server and calling out to webhook admission plugins. It is expected that this "+ - "cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+ - "That CA is published in the 'extension-apiserver-authentication' configmap in "+ - "the kube-system namespace. Components receiving calls from kube-aggregator should "+ - "use that CA to perform their half of the mutual TLS verification.") - fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+ - "Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+ - "when it must call out during a request. This includes proxying requests to a user "+ - "api-server and calling out to webhook admission plugins.") - - fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting, - "Turns on aggregator routing requests to endpoints IP rather than cluster IP.") - - fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+ - "Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.") - - return fss -} diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go deleted file mode 100644 index baa1eafcab335..0000000000000 --- a/cmd/kube-apiserver/app/options/options_test.go +++ /dev/null @@ -1,316 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "net" - "reflect" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/spf13/pflag" - "k8s.io/component-base/logs" - - "k8s.io/apiserver/pkg/admission" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/storage/storagebackend" - auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" - audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" - restclient "k8s.io/client-go/rest" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/metrics" - kapi "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/controlplane/reconcilers" - kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" - kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" -) - -func TestAddFlags(t *testing.T) { - fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) - s := NewServerRunOptions() - for _, f := range s.Flags().FlagSets { - fs.AddFlagSet(f) - } - - args := []string{ - "--enable-admission-plugins=AlwaysDeny", - "--admission-control-config-file=/admission-control-config", - "--advertise-address=192.168.10.10", - "--allow-privileged=false", - "--anonymous-auth=false", - "--apiserver-count=5", - "--audit-log-maxage=11", - "--audit-log-maxbackup=12", - "--audit-log-maxsize=13", - "--audit-log-path=/var/log", - "--audit-log-mode=blocking", - "--audit-log-batch-buffer-size=46", - "--audit-log-batch-max-size=47", - "--audit-log-batch-max-wait=48s", - "--audit-log-batch-throttle-enable=true", - "--audit-log-batch-throttle-qps=49.5", - "--audit-log-batch-throttle-burst=50", - "--audit-log-truncate-enabled=true", - "--audit-log-truncate-max-batch-size=45", - "--audit-log-truncate-max-event-size=44", - "--audit-log-version=audit.k8s.io/v1alpha1", - "--audit-policy-file=/policy", - "--audit-webhook-config-file=/webhook-config", - "--audit-webhook-mode=blocking", - "--audit-webhook-batch-buffer-size=42", - "--audit-webhook-batch-max-size=43", - "--audit-webhook-batch-max-wait=1s", - "--audit-webhook-batch-throttle-enable=false", - "--audit-webhook-batch-throttle-qps=43.5", - "--audit-webhook-batch-throttle-burst=44", - "--audit-webhook-truncate-enabled=true", - "--audit-webhook-truncate-max-batch-size=43", - "--audit-webhook-truncate-max-event-size=42", - "--audit-webhook-initial-backoff=2s", - "--audit-webhook-version=audit.k8s.io/v1alpha1", - "--authentication-token-webhook-cache-ttl=3m", - "--authentication-token-webhook-config-file=/token-webhook-config", - "--authorization-mode=AlwaysDeny,RBAC", - "--authorization-policy-file=/policy", - "--authorization-webhook-cache-authorized-ttl=3m", - "--authorization-webhook-cache-unauthorized-ttl=1m", - "--authorization-webhook-config-file=/webhook-config", - "--bind-address=192.168.10.20", - "--client-ca-file=/client-ca", - "--cloud-config=/cloud-config", - "--cloud-provider=azure", - "--cors-allowed-origins=10.10.10.100,10.10.10.200", - "--contention-profiling=true", - "--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml", - "--enable-aggregator-routing=true", - "--enable-priority-and-fairness=false", - "--enable-logs-handler=false", - "--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType), - "--etcd-keyfile=/var/run/kubernetes/etcd.key", - "--etcd-certfile=/var/run/kubernetes/etcdce.crt", - "--etcd-cafile=/var/run/kubernetes/etcdca.crt", - "--http2-max-streams-per-connection=42", - "--kubelet-read-only-port=10255", - "--kubelet-timeout=5s", - "--kubelet-client-certificate=/var/run/kubernetes/ceserver.crt", - "--kubelet-client-key=/var/run/kubernetes/server.key", - "--kubelet-certificate-authority=/var/run/kubernetes/caserver.crt", - "--proxy-client-cert-file=/var/run/kubernetes/proxy.crt", - "--proxy-client-key-file=/var/run/kubernetes/proxy.key", - "--request-timeout=2m", - "--storage-backend=etcd3", - "--service-cluster-ip-range=192.168.128.0/17", - } - fs.Parse(args) - - // This is a snapshot of expected options parsed by args. - expected := &ServerRunOptions{ - ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, - ServiceClusterIPRanges: (&net.IPNet{IP: net.ParseIP("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(), - MasterCount: 5, - EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), - AllowPrivileged: false, - GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ - AdvertiseAddress: net.ParseIP("192.168.10.10"), - CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, - MaxRequestsInFlight: 400, - MaxMutatingRequestsInFlight: 200, - RequestTimeout: time.Duration(2) * time.Minute, - MinRequestTimeout: 1800, - JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), - MaxRequestBodyBytes: int64(3 * 1024 * 1024), - }, - Admission: &kubeoptions.AdmissionOptions{ - GenericAdmission: &apiserveroptions.AdmissionOptions{ - RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder, - DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins, - EnablePlugins: []string{"AlwaysDeny"}, - ConfigFile: "/admission-control-config", - Plugins: s.Admission.GenericAdmission.Plugins, - Decorators: s.Admission.GenericAdmission.Decorators, - }, - }, - Etcd: &apiserveroptions.EtcdOptions{ - StorageConfig: storagebackend.Config{ - Type: "etcd3", - Transport: storagebackend.TransportConfig{ - ServerList: nil, - KeyFile: "/var/run/kubernetes/etcd.key", - TrustedCAFile: "/var/run/kubernetes/etcdca.crt", - CertFile: "/var/run/kubernetes/etcdce.crt", - }, - Paging: true, - Prefix: "/registry", - CompactionInterval: storagebackend.DefaultCompactInterval, - CountMetricPollPeriod: time.Minute, - DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, - HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, - LeaseReuseDurationSeconds: storagebackend.DefaultLeaseReuseDurationSeconds, - }, - DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", - DeleteCollectionWorkers: 1, - EnableGarbageCollection: true, - EnableWatchCache: true, - DefaultWatchCacheSize: 100, - }, - SecureServing: (&apiserveroptions.SecureServingOptions{ - BindAddress: net.ParseIP("192.168.10.20"), - BindPort: 6443, - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/var/run/kubernetes", - PairName: "apiserver", - }, - HTTP2MaxStreamsPerConnection: 42, - Required: true, - }).WithLoopback(), - EventTTL: 1 * time.Hour, - KubeletConfig: kubeletclient.KubeletClientConfig{ - Port: 10250, - ReadOnlyPort: 10255, - PreferredAddressTypes: []string{ - string(kapi.NodeHostName), - string(kapi.NodeInternalDNS), - string(kapi.NodeInternalIP), - string(kapi.NodeExternalDNS), - string(kapi.NodeExternalIP), - }, - HTTPTimeout: time.Duration(5) * time.Second, - TLSClientConfig: restclient.TLSClientConfig{ - CertFile: "/var/run/kubernetes/ceserver.crt", - KeyFile: "/var/run/kubernetes/server.key", - CAFile: "/var/run/kubernetes/caserver.crt", - }, - }, - Audit: &apiserveroptions.AuditOptions{ - LogOptions: apiserveroptions.AuditLogOptions{ - Path: "/var/log", - MaxAge: 11, - MaxBackups: 12, - MaxSize: 13, - Format: "json", - BatchOptions: apiserveroptions.AuditBatchOptions{ - Mode: "blocking", - BatchConfig: auditbuffered.BatchConfig{ - BufferSize: 46, - MaxBatchSize: 47, - MaxBatchWait: 48 * time.Second, - ThrottleEnable: true, - ThrottleQPS: 49.5, - ThrottleBurst: 50, - }, - }, - TruncateOptions: apiserveroptions.AuditTruncateOptions{ - Enabled: true, - TruncateConfig: audittruncate.Config{ - MaxBatchSize: 45, - MaxEventSize: 44, - }, - }, - GroupVersionString: "audit.k8s.io/v1alpha1", - }, - WebhookOptions: apiserveroptions.AuditWebhookOptions{ - ConfigFile: "/webhook-config", - BatchOptions: apiserveroptions.AuditBatchOptions{ - Mode: "blocking", - BatchConfig: auditbuffered.BatchConfig{ - BufferSize: 42, - MaxBatchSize: 43, - MaxBatchWait: 1 * time.Second, - ThrottleEnable: false, - ThrottleQPS: 43.5, - ThrottleBurst: 44, - AsyncDelegate: true, - }, - }, - TruncateOptions: apiserveroptions.AuditTruncateOptions{ - Enabled: true, - TruncateConfig: audittruncate.Config{ - MaxBatchSize: 43, - MaxEventSize: 42, - }, - }, - InitialBackoff: 2 * time.Second, - GroupVersionString: "audit.k8s.io/v1alpha1", - }, - PolicyFile: "/policy", - }, - Features: &apiserveroptions.FeatureOptions{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - Authentication: &kubeoptions.BuiltInAuthenticationOptions{ - Anonymous: &kubeoptions.AnonymousAuthenticationOptions{ - Allow: false, - }, - ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{ - ClientCA: "/client-ca", - }, - WebHook: &kubeoptions.WebHookAuthenticationOptions{ - CacheTTL: 180000000000, - ConfigFile: "/token-webhook-config", - Version: "v1beta1", - RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), - }, - BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, - OIDC: &kubeoptions.OIDCAuthenticationOptions{ - UsernameClaim: "sub", - SigningAlgs: []string{"RS256"}, - }, - RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{}, - ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ - Lookup: true, - ExtendExpiration: true, - }, - TokenFile: &kubeoptions.TokenFileAuthenticationOptions{}, - TokenSuccessCacheTTL: 10 * time.Second, - TokenFailureCacheTTL: 0, - }, - Authorization: &kubeoptions.BuiltInAuthorizationOptions{ - Modes: []string{"AlwaysDeny", "RBAC"}, - PolicyFile: "/policy", - WebhookConfigFile: "/webhook-config", - WebhookCacheAuthorizedTTL: 180000000000, - WebhookCacheUnauthorizedTTL: 60000000000, - WebhookVersion: "v1beta1", - WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), - }, - CloudProvider: &kubeoptions.CloudProviderOptions{ - CloudConfigFile: "/cloud-config", - CloudProvider: "azure", - }, - APIEnablement: &apiserveroptions.APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{}, - }, - EgressSelector: &apiserveroptions.EgressSelectorOptions{ - ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", - }, - EnableLogsHandler: false, - EnableAggregatorRouting: true, - ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", - ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", - Metrics: &metrics.Options{}, - Logs: logs.NewOptions(), - IdentityLeaseDurationSeconds: 3600, - IdentityLeaseRenewIntervalSeconds: 10, - } - - if !reflect.DeepEqual(expected, s) { - t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}))) - } -} diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go deleted file mode 100644 index 34b00a79e6238..0000000000000 --- a/cmd/kube-apiserver/app/options/validation.go +++ /dev/null @@ -1,180 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "errors" - "fmt" - "net" - "strings" - - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - genericfeatures "k8s.io/apiserver/pkg/features" - utilfeature "k8s.io/apiserver/pkg/util/feature" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/features" - netutils "k8s.io/utils/net" -) - -// TODO: Longer term we should read this from some config store, rather than a flag. -// validateClusterIPFlags is expected to be called after Complete() -func validateClusterIPFlags(options *ServerRunOptions) []error { - var errs []error - // maxCIDRBits is used to define the maximum CIDR size for the cluster ip(s) - const maxCIDRBits = 20 - - // validate that primary has been processed by user provided values or it has been defaulted - if options.PrimaryServiceClusterIPRange.IP == nil { - errs = append(errs, errors.New("--service-cluster-ip-range must contain at least one valid cidr")) - } - - serviceClusterIPRangeList := strings.Split(options.ServiceClusterIPRanges, ",") - if len(serviceClusterIPRangeList) > 2 { - errs = append(errs, errors.New("--service-cluster-ip-range must not contain more than two entries")) - } - - // Complete() expected to have set Primary* and Secondary* - // primary CIDR validation - if err := validateMaxCIDRRange(options.PrimaryServiceClusterIPRange, maxCIDRBits, "--service-cluster-ip-range"); err != nil { - errs = append(errs, err) - } - - // Secondary IP validation - // while api-server dualstack bits does not have dependency on EndPointSlice, its - // a good idea to have validation consistent across all components (ControllerManager - // needs EndPointSlice + DualStack feature flags). - secondaryServiceClusterIPRangeUsed := (options.SecondaryServiceClusterIPRange.IP != nil) - if secondaryServiceClusterIPRangeUsed && (!utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice)) { - errs = append(errs, fmt.Errorf("secondary service cluster-ip range(--service-cluster-ip-range[1]) can only be used if %v and %v feature is enabled", string(features.IPv6DualStack), string(features.EndpointSlice))) - } - - // note: While the cluster might be dualstack (i.e. pods with multiple IPs), the user may choose - // to only ingress traffic within and into the cluster on one IP family only. this family is decided - // by the range set on --service-cluster-ip-range. If/when the user decides to use dual stack services - // the Secondary* must be of different IPFamily than --service-cluster-ip-range - if secondaryServiceClusterIPRangeUsed { - // Should be dualstack IPFamily(PrimaryServiceClusterIPRange) != IPFamily(SecondaryServiceClusterIPRange) - dualstack, err := netutils.IsDualStackCIDRs([]*net.IPNet{&options.PrimaryServiceClusterIPRange, &options.SecondaryServiceClusterIPRange}) - if err != nil { - errs = append(errs, fmt.Errorf("error attempting to validate dualstack for --service-cluster-ip-range value error:%v", err)) - } - - if !dualstack { - errs = append(errs, errors.New("--service-cluster-ip-range[0] and --service-cluster-ip-range[1] must be of different IP family")) - } - - if err := validateMaxCIDRRange(options.SecondaryServiceClusterIPRange, maxCIDRBits, "--service-cluster-ip-range[1]"); err != nil { - errs = append(errs, err) - } - } - - return errs -} - -func validateMaxCIDRRange(cidr net.IPNet, maxCIDRBits int, cidrFlag string) error { - // Should be smallish sized cidr, this thing is kept in etcd - // bigger cidr (specially those offered by IPv6) will add no value - // significantly increase snapshotting time. - var ones, bits = cidr.Mask.Size() - if bits-ones > maxCIDRBits { - return fmt.Errorf("specified %s is too large; for %d-bit addresses, the mask must be >= %d", cidrFlag, bits, bits-maxCIDRBits) - } - - return nil -} - -func validateServiceNodePort(options *ServerRunOptions) []error { - var errs []error - - if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 { - errs = append(errs, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP", options.KubernetesServiceNodePort)) - } - - if options.KubernetesServiceNodePort > 0 && !options.ServiceNodePortRange.Contains(options.KubernetesServiceNodePort) { - errs = append(errs, fmt.Errorf("kubernetes service port range %v doesn't contain %v", options.ServiceNodePortRange, (options.KubernetesServiceNodePort))) - } - return errs -} - -func validateTokenRequest(options *ServerRunOptions) []error { - var errs []error - - enableAttempted := options.ServiceAccountSigningKeyFile != "" || - options.Authentication.ServiceAccounts.Issuer != "" || - len(options.Authentication.APIAudiences) != 0 - - enableSucceeded := options.ServiceAccountIssuer != nil - - if !enableAttempted { - errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags")) - } - - if enableAttempted && !enableSucceeded { - errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together")) - } - - return errs -} - -func validateAPIPriorityAndFairness(options *ServerRunOptions) []error { - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness { - // If none of the following runtime config options are specified, APF is - // assumed to be turned on. - enabledAPIString := options.APIEnablement.RuntimeConfig.String() - testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta1", "api/beta", "api/all"} // in the order of precedence - for _, testConfig := range testConfigs { - if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) { - return []error{fmt.Errorf("%s=false conflicts with APIPriorityAndFairness feature gate", testConfig)} - } - if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) { - return nil - } - } - } - - return nil -} - -// Validate checks ServerRunOptions and return a slice of found errs. -func (s *ServerRunOptions) Validate() []error { - var errs []error - if s.MasterCount <= 0 { - errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) - } - errs = append(errs, s.Etcd.Validate()...) - errs = append(errs, validateClusterIPFlags(s)...) - errs = append(errs, validateServiceNodePort(s)...) - errs = append(errs, validateAPIPriorityAndFairness(s)...) - errs = append(errs, s.SecureServing.Validate()...) - errs = append(errs, s.Authentication.Validate()...) - errs = append(errs, s.Authorization.Validate()...) - errs = append(errs, s.Audit.Validate()...) - errs = append(errs, s.Admission.Validate()...) - errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...) - errs = append(errs, validateTokenRequest(s)...) - errs = append(errs, s.Metrics.Validate()...) - errs = append(errs, s.Logs.Validate()...) - if s.IdentityLeaseDurationSeconds <= 0 { - errs = append(errs, fmt.Errorf("--identity-lease-duration-seconds should be a positive number, but value '%d' provided", s.IdentityLeaseDurationSeconds)) - } - if s.IdentityLeaseRenewIntervalSeconds <= 0 { - errs = append(errs, fmt.Errorf("--identity-lease-renew-interval-seconds should be a positive number, but value '%d' provided", s.IdentityLeaseRenewIntervalSeconds)) - } - - return errs -} diff --git a/cmd/kube-apiserver/app/options/validation_test.go b/cmd/kube-apiserver/app/options/validation_test.go deleted file mode 100644 index 24aa1fa972d92..0000000000000 --- a/cmd/kube-apiserver/app/options/validation_test.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "net" - "testing" - - utilfeature "k8s.io/apiserver/pkg/util/feature" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/kubernetes/pkg/features" -) - -func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *ServerRunOptions { - value := serviceCIDR - if len(secondaryServiceCIDR) > 0 { - value = value + "," + secondaryServiceCIDR - } - - var primaryCIDR, secondaryCIDR net.IPNet - if len(serviceCIDR) > 0 { - _, cidr, _ := net.ParseCIDR(serviceCIDR) - if cidr != nil { - primaryCIDR = *(cidr) - } - } - - if len(secondaryServiceCIDR) > 0 { - _, cidr, _ := net.ParseCIDR(secondaryServiceCIDR) - if cidr != nil { - secondaryCIDR = *(cidr) - } - } - return &ServerRunOptions{ - ServiceClusterIPRanges: value, - PrimaryServiceClusterIPRange: primaryCIDR, - SecondaryServiceClusterIPRange: secondaryCIDR, - } -} - -func TestClusterSerivceIPRange(t *testing.T) { - testCases := []struct { - name string - options *ServerRunOptions - enableDualStack bool - enableEndpointSlice bool - expectErrors bool - }{ - { - name: "no service cidr", - expectErrors: true, - options: makeOptionsWithCIDRs("", ""), - enableDualStack: false, - }, - { - name: "only secondary service cidr, dual stack gate on", - expectErrors: true, - options: makeOptionsWithCIDRs("", "10.0.0.0/16"), - enableDualStack: true, - enableEndpointSlice: true, - }, - { - name: "only secondary service cidr, dual stack gate off", - expectErrors: true, - options: makeOptionsWithCIDRs("", "10.0.0.0/16"), - enableDualStack: false, - }, - { - name: "primary and secondary are provided but not dual stack v4-v4", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/16", "11.0.0.0/16"), - enableDualStack: true, - enableEndpointSlice: true, - }, - { - name: "primary and secondary are provided but not dual stack v6-v6", - expectErrors: true, - options: makeOptionsWithCIDRs("2000::/108", "3000::/108"), - enableDualStack: true, - enableEndpointSlice: true, - }, - { - name: "valid dual stack with gate disabled", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"), - enableDualStack: false, - }, - { - name: "service cidr is too big", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/8", ""), - enableDualStack: true, - enableEndpointSlice: true, - }, - { - name: "dual-stack secondary cidr too big", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64"), - enableDualStack: true, - enableEndpointSlice: true, - }, - { - name: "valid v6-v4 dual stack + gate on + endpointSlice gate is on", - expectErrors: false, - options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"), - enableDualStack: true, - enableEndpointSlice: true, - }, - - { - name: "valid v4-v6 dual stack + gate on + endpointSlice is off", - expectErrors: true, - options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"), - enableDualStack: true, - enableEndpointSlice: false, - }, - /* success cases */ - { - name: "valid primary", - expectErrors: false, - options: makeOptionsWithCIDRs("10.0.0.0/16", ""), - enableDualStack: false, - }, - { - name: "valid v4-v6 dual stack + gate on", - expectErrors: false, - options: makeOptionsWithCIDRs("10.0.0.0/16", "3000::/108"), - enableDualStack: true, - enableEndpointSlice: true, - }, - { - name: "valid v6-v4 dual stack + gate on", - expectErrors: false, - options: makeOptionsWithCIDRs("3000::/108", "10.0.0.0/16"), - enableDualStack: true, - enableEndpointSlice: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.IPv6DualStack, tc.enableDualStack)() - defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EndpointSlice, tc.enableEndpointSlice)() - errs := validateClusterIPFlags(tc.options) - if len(errs) > 0 && !tc.expectErrors { - t.Errorf("expected no errors, errors found %+v", errs) - } - - if len(errs) == 0 && tc.expectErrors { - t.Errorf("expected errors, no errors found") - } - }) - } -} - -func getIPnetFromCIDR(cidr string) *net.IPNet { - _, ipnet, _ := net.ParseCIDR(cidr) - return ipnet -} - -func TestValidateMaxCIDRRange(t *testing.T) { - testCases := []struct { - // tc.cidr, tc.maxCIDRBits, tc.cidrFlag) tc.expectedErrorMessage - name string - cidr net.IPNet - maxCIDRBits int - cidrFlag string - expectedErrorMessage string - expectErrors bool - }{ - { - name: "valid ipv4 cidr", - cidr: *getIPnetFromCIDR("10.92.0.0/12"), - maxCIDRBits: 20, - cidrFlag: "--service-cluster-ip-range", - expectedErrorMessage: "", - expectErrors: false, - }, - { - name: "valid ipv6 cidr", - cidr: *getIPnetFromCIDR("3000::/108"), - maxCIDRBits: 20, - cidrFlag: "--service-cluster-ip-range", - expectedErrorMessage: "", - expectErrors: false, - }, - { - name: "ipv4 cidr to big", - cidr: *getIPnetFromCIDR("10.92.0.0/8"), - maxCIDRBits: 20, - cidrFlag: "--service-cluster-ip-range", - expectedErrorMessage: "specified --service-cluster-ip-range is too large; for 32-bit addresses, the mask must be >= 12", - expectErrors: true, - }, - { - name: "ipv6 cidr to big", - cidr: *getIPnetFromCIDR("3000::/64"), - maxCIDRBits: 20, - cidrFlag: "--service-cluster-ip-range", - expectedErrorMessage: "specified --service-cluster-ip-range is too large; for 128-bit addresses, the mask must be >= 108", - expectErrors: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - err := validateMaxCIDRRange(tc.cidr, tc.maxCIDRBits, tc.cidrFlag) - if err != nil && !tc.expectErrors { - t.Errorf("expected no errors, error found %+v", err) - } - - if err == nil && tc.expectErrors { - t.Errorf("expected errors, no errors found") - } - - if err != nil && tc.expectErrors && err.Error() != tc.expectedErrorMessage { - t.Errorf("Expected error message: \"%s\"\nGot: \"%s\"", tc.expectedErrorMessage, err.Error()) - } - }) - } -} diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go deleted file mode 100644 index 715e289f701a4..0000000000000 --- a/cmd/kube-apiserver/app/server.go +++ /dev/null @@ -1,775 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to create a Kubernetes -// APIServer by binding together the API, master and APIServer infrastructure. -// It can be configured and called directly or via the hyperkube framework. -package app - -import ( - "crypto/tls" - "fmt" - "net" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - utilnet "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/sets" - utilwait "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/authorization/authorizer" - openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" - genericfeatures "k8s.io/apiserver/pkg/features" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/egressselector" - "k8s.io/apiserver/pkg/server/filters" - serveroptions "k8s.io/apiserver/pkg/server/options" - serverstorage "k8s.io/apiserver/pkg/server/storage" - "k8s.io/apiserver/pkg/storage/etcd3/preflight" - "k8s.io/apiserver/pkg/util/feature" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" - "k8s.io/apiserver/pkg/util/webhook" - clientgoinformers "k8s.io/client-go/informers" - clientgoclientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/util/keyutil" - cloudprovider "k8s.io/cloud-provider" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/cli/globalflag" - _ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration - "k8s.io/component-base/term" - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - "k8s.io/klog/v2" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/capabilities" - "k8s.io/kubernetes/pkg/controlplane" - "k8s.io/kubernetes/pkg/controlplane/reconcilers" - "k8s.io/kubernetes/pkg/controlplane/tunneler" - "k8s.io/kubernetes/pkg/features" - generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" - "k8s.io/kubernetes/pkg/kubeapiserver" - kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" - kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" - "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" - rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" - "k8s.io/kubernetes/pkg/serviceaccount" -) - -const ( - etcdRetryLimit = 60 - etcdRetryInterval = 1 * time.Second -) - -// TODO: delete this check after insecure flags removed in v1.24 -func checkNonZeroInsecurePort(fs *pflag.FlagSet) error { - for _, name := range options.InsecurePortFlags { - val, err := fs.GetInt(name) - if err != nil { - return err - } - if val != 0 { - return fmt.Errorf("invalid port value %d: only zero is allowed", val) - } - } - return nil -} - -// NewAPIServerCommand creates a *cobra.Command object with default parameters -func NewAPIServerCommand() *cobra.Command { - s := options.NewServerRunOptions() - cmd := &cobra.Command{ - Use: "kube-apiserver", - Long: `The Kubernetes API server validates and configures data -for the api objects which include pods, services, replicationcontrollers, and -others. The API Server services REST operations and provides the frontend to the -cluster's shared state through which all other components interact.`, - - // stop printing usage when the command errors - SilenceUsage: true, - PersistentPreRunE: func(*cobra.Command, []string) error { - // silence client-go warnings. - // kube-apiserver loopback clients should not log self-issued warnings. - rest.SetDefaultWarningHandler(rest.NoWarnings{}) - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - verflag.PrintAndExitIfRequested() - fs := cmd.Flags() - cliflag.PrintFlags(fs) - - err := checkNonZeroInsecurePort(fs) - if err != nil { - return err - } - // set default options - completedOptions, err := Complete(s) - if err != nil { - return err - } - - // validate options - if errs := completedOptions.Validate(); len(errs) != 0 { - return utilerrors.NewAggregate(errs) - } - - return Run(completedOptions, genericapiserver.SetupSignalHandler()) - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - - fs := cmd.Flags() - namedFlagSets := s.Flags() - verflag.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) - options.AddCustomGlobalFlags(namedFlagSets.FlagSet("generic")) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - - usageFmt := "Usage:\n %s\n" - cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) - cmd.SetUsageFunc(func(cmd *cobra.Command) error { - fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) - return nil - }) - cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { - fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) - }) - - return cmd -} - -// Run runs the specified APIServer. This should never exit. -func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error { - // To help debugging, immediately log version - klog.Infof("Version: %+v", version.Get()) - - server, err := CreateServerChain(completeOptions, stopCh) - if err != nil { - return err - } - - prepared, err := server.PrepareRun() - if err != nil { - return err - } - - return prepared.Run(stopCh) -} - -// CreateServerChain creates the apiservers connected via delegation. -func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{}) (*aggregatorapiserver.APIAggregator, error) { - nodeTunneler, proxyTransport, err := CreateNodeDialer(completedOptions) - if err != nil { - return nil, err - } - - kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions, nodeTunneler, proxyTransport) - if err != nil { - return nil, err - } - - // If additional API servers are added, they should be gated. - apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount, - serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, kubeAPIServerConfig.GenericConfig.EgressSelector, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig)) - if err != nil { - return nil, err - } - apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate()) - if err != nil { - return nil, err - } - - kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer) - if err != nil { - return nil, err - } - - // aggregator comes last in the chain - aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, proxyTransport, pluginInitializer) - if err != nil { - return nil, err - } - aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers) - if err != nil { - // we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines - return nil, err - } - - return aggregatorServer, nil -} - -// CreateKubeAPIServer creates and wires a workable kube-apiserver -func CreateKubeAPIServer(kubeAPIServerConfig *controlplane.Config, delegateAPIServer genericapiserver.DelegationTarget) (*controlplane.Instance, error) { - kubeAPIServer, err := kubeAPIServerConfig.Complete().New(delegateAPIServer) - if err != nil { - return nil, err - } - - return kubeAPIServer, nil -} - -// CreateNodeDialer creates the dialer infrastructure to connect to the nodes. -func CreateNodeDialer(s completedServerRunOptions) (tunneler.Tunneler, *http.Transport, error) { - // Setup nodeTunneler if needed - var nodeTunneler tunneler.Tunneler - var proxyDialerFn utilnet.DialFunc - if len(s.SSHUser) > 0 { - // Get ssh key distribution func, if supported - var installSSHKey tunneler.InstallSSHKey - - cloudprovider.DeprecationWarningForProvider(s.CloudProvider.CloudProvider) - cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile) - if err != nil { - return nil, nil, fmt.Errorf("cloud provider could not be initialized: %v", err) - } - if cloud != nil { - if instances, supported := cloud.Instances(); supported { - installSSHKey = instances.AddSSHKeyToAllInstances - } - } - if s.KubeletConfig.Port == 0 { - return nil, nil, fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified") - } - if s.KubeletConfig.ReadOnlyPort == 0 { - return nil, nil, fmt.Errorf("must enable kubelet readonly port if proxy ssh-tunneling is specified") - } - // Set up the nodeTunneler - // TODO(cjcullen): If we want this to handle per-kubelet ports or other - // kubelet listen-addresses, we need to plumb through options. - healthCheckPath := &url.URL{ - Scheme: "http", - Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.ReadOnlyPort), 10)), - Path: "healthz", - } - nodeTunneler = tunneler.New(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSHKey) - - // Use the nodeTunneler's dialer when proxying to pods, services, and nodes - proxyDialerFn = nodeTunneler.Dial - } - // Proxying to pods and services is IP-based... don't expect to be able to verify the hostname - proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true} - proxyTransport := utilnet.SetTransportDefaults(&http.Transport{ - DialContext: proxyDialerFn, - TLSClientConfig: proxyTLSClientConfig, - }) - return nodeTunneler, proxyTransport, nil -} - -// CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them -func CreateKubeAPIServerConfig( - s completedServerRunOptions, - nodeTunneler tunneler.Tunneler, - proxyTransport *http.Transport, -) ( - *controlplane.Config, - aggregatorapiserver.ServiceResolver, - []admission.PluginInitializer, - error, -) { - genericConfig, versionedInformers, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport) - if err != nil { - return nil, nil, nil, err - } - - if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.Transport.ServerList[0]); err == nil && port != "0" && len(port) != 0 { - if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.Transport.ServerList}.CheckEtcdServers); err != nil { - return nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err) - } - } - - capabilities.Initialize(capabilities.Capabilities{ - AllowPrivileged: s.AllowPrivileged, - // TODO(vmarmol): Implement support for HostNetworkSources. - PrivilegedSources: capabilities.PrivilegedSources{ - HostNetworkSources: []string{}, - HostPIDSources: []string{}, - HostIPCSources: []string{}, - }, - PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, - }) - - s.Metrics.Apply() - serviceaccount.RegisterMetrics() - - s.Logs.Apply() - - serviceIPRange, apiServerServiceIP, err := controlplane.ServiceIPRange(s.PrimaryServiceClusterIPRange) - if err != nil { - return nil, nil, nil, err - } - - // defaults to empty range and ip - var secondaryServiceIPRange net.IPNet - // process secondary range only if provided by user - if s.SecondaryServiceClusterIPRange.IP != nil { - secondaryServiceIPRange, _, err = controlplane.ServiceIPRange(s.SecondaryServiceClusterIPRange) - if err != nil { - return nil, nil, nil, err - } - } - - config := &controlplane.Config{ - GenericConfig: genericConfig, - ExtraConfig: controlplane.ExtraConfig{ - APIResourceConfigSource: storageFactory.APIResourceConfigSource, - StorageFactory: storageFactory, - EventTTL: s.EventTTL, - KubeletClientConfig: s.KubeletConfig, - EnableLogsSupport: s.EnableLogsHandler, - ProxyTransport: proxyTransport, - - Tunneler: nodeTunneler, - - ServiceIPRange: serviceIPRange, - APIServerServiceIP: apiServerServiceIP, - SecondaryServiceIPRange: secondaryServiceIPRange, - - APIServerServicePort: 443, - - ServiceNodePortRange: s.ServiceNodePortRange, - KubernetesServiceNodePort: s.KubernetesServiceNodePort, - - EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType), - MasterCount: s.MasterCount, - - ServiceAccountIssuer: s.ServiceAccountIssuer, - ServiceAccountMaxExpiration: s.ServiceAccountTokenMaxExpiration, - ExtendExpiration: s.Authentication.ServiceAccounts.ExtendExpiration, - - VersionedInformers: versionedInformers, - - IdentityLeaseDurationSeconds: s.IdentityLeaseDurationSeconds, - IdentityLeaseRenewIntervalSeconds: s.IdentityLeaseRenewIntervalSeconds, - }, - } - - clientCAProvider, err := s.Authentication.ClientCert.GetClientCAContentProvider() - if err != nil { - return nil, nil, nil, err - } - config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider - - requestHeaderConfig, err := s.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() - if err != nil { - return nil, nil, nil, err - } - if requestHeaderConfig != nil { - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders - } - - if err := config.GenericConfig.AddPostStartHook("start-kube-apiserver-admission-initializer", admissionPostStartHook); err != nil { - return nil, nil, nil, err - } - - if nodeTunneler != nil { - // Use the nodeTunneler's dialer to connect to the kubelet - config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial - } - if config.GenericConfig.EgressSelector != nil { - // Use the config.GenericConfig.EgressSelector lookup to find the dialer to connect to the kubelet - config.ExtraConfig.KubeletClientConfig.Lookup = config.GenericConfig.EgressSelector.Lookup - - // Use the config.GenericConfig.EgressSelector lookup as the transport used by the "proxy" subresources. - networkContext := egressselector.Cluster.AsNetworkContext() - dialer, err := config.GenericConfig.EgressSelector.Lookup(networkContext) - if err != nil { - return nil, nil, nil, err - } - c := proxyTransport.Clone() - c.DialContext = dialer - config.ExtraConfig.ProxyTransport = c - } - - if utilfeature.DefaultFeatureGate.Enabled(features.ServiceAccountIssuerDiscovery) { - // Load the public keys. - var pubKeys []interface{} - for _, f := range s.Authentication.ServiceAccounts.KeyFiles { - keys, err := keyutil.PublicKeysFromFile(f) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to parse key file %q: %v", f, err) - } - pubKeys = append(pubKeys, keys...) - } - // Plumb the required metadata through ExtraConfig. - config.ExtraConfig.ServiceAccountIssuerURL = s.Authentication.ServiceAccounts.Issuer - config.ExtraConfig.ServiceAccountJWKSURI = s.Authentication.ServiceAccounts.JWKSURI - config.ExtraConfig.ServiceAccountPublicKeys = pubKeys - } - - return config, serviceResolver, pluginInitializers, nil -} - -// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it -func buildGenericConfig( - s *options.ServerRunOptions, - proxyTransport *http.Transport, -) ( - genericConfig *genericapiserver.Config, - versionedInformers clientgoinformers.SharedInformerFactory, - serviceResolver aggregatorapiserver.ServiceResolver, - pluginInitializers []admission.PluginInitializer, - admissionPostStartHook genericapiserver.PostStartHookFunc, - storageFactory *serverstorage.DefaultStorageFactory, - lastErr error, -) { - genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs) - genericConfig.MergedResourceConfig = controlplane.DefaultAPIResourceConfigSource() - - if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil { - return - } - - if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil { - return - } - if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { - return - } - if lastErr = s.APIEnablement.ApplyTo(genericConfig, controlplane.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil { - return - } - if lastErr = s.EgressSelector.ApplyTo(genericConfig); lastErr != nil { - return - } - - genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)) - genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" - genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( - sets.NewString("watch", "proxy"), - sets.NewString("attach", "exec", "proxy", "log", "portforward"), - ) - - kubeVersion := version.Get() - genericConfig.Version = &kubeVersion - - storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig() - storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig - completedStorageFactoryConfig, err := storageFactoryConfig.Complete(s.Etcd) - if err != nil { - lastErr = err - return - } - storageFactory, lastErr = completedStorageFactoryConfig.New() - if lastErr != nil { - return - } - if genericConfig.EgressSelector != nil { - storageFactory.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup - } - if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil { - return - } - - // Use protobufs for self-communication. - // Since not every generic apiserver has to support protobufs, we - // cannot default to it in generic apiserver and need to explicitly - // set it in kube-apiserver. - genericConfig.LoopbackClientConfig.ContentConfig.ContentType = "application/vnd.kubernetes.protobuf" - // Disable compression for self-communication, since we are going to be - // on a fast local network - genericConfig.LoopbackClientConfig.DisableCompression = true - - kubeClientConfig := genericConfig.LoopbackClientConfig - clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig) - if err != nil { - lastErr = fmt.Errorf("failed to create real external clientset: %v", err) - return - } - versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - - // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present - if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, clientgoExternalClient, versionedInformers); lastErr != nil { - return - } - - genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers) - if err != nil { - lastErr = fmt.Errorf("invalid authorization config: %v", err) - return - } - if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) { - genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName) - } - - lastErr = s.Audit.ApplyTo(genericConfig) - if lastErr != nil { - return - } - - admissionConfig := &kubeapiserveradmission.Config{ - ExternalInformers: versionedInformers, - LoopbackClientConfig: genericConfig.LoopbackClientConfig, - CloudConfigFile: s.CloudProvider.CloudConfigFile, - } - serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) - pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver) - if err != nil { - lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err) - return - } - - err = s.Admission.ApplyTo( - genericConfig, - versionedInformers, - kubeClientConfig, - feature.DefaultFeatureGate, - pluginInitializers...) - if err != nil { - lastErr = fmt.Errorf("failed to initialize admission: %v", err) - } - - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnablePriorityAndFairness { - genericConfig.FlowControl = BuildPriorityAndFairness(s, clientgoExternalClient, versionedInformers) - } - - return -} - -// BuildAuthorizer constructs the authorizer -func BuildAuthorizer(s *options.ServerRunOptions, EgressSelector *egressselector.EgressSelector, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) { - authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers) - - if EgressSelector != nil { - egressDialer, err := EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext()) - if err != nil { - return nil, nil, err - } - authorizationConfig.CustomDial = egressDialer - } - - return authorizationConfig.New() -} - -// BuildPriorityAndFairness constructs the guts of the API Priority and Fairness filter -func BuildPriorityAndFairness(s *options.ServerRunOptions, extclient clientgoclientset.Interface, versionedInformer clientgoinformers.SharedInformerFactory) utilflowcontrol.Interface { - return utilflowcontrol.New( - versionedInformer, - extclient.FlowcontrolV1beta1(), - s.GenericServerRunOptions.MaxRequestsInFlight+s.GenericServerRunOptions.MaxMutatingRequestsInFlight, - s.GenericServerRunOptions.RequestTimeout/4, - ) -} - -// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. -type completedServerRunOptions struct { - *options.ServerRunOptions -} - -// Complete set default ServerRunOptions. -// Should be called after kube-apiserver flags parsed. -func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) { - var options completedServerRunOptions - // set defaults - if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil { - return options, err - } - - // process s.ServiceClusterIPRange from list to Primary and Secondary - // we process secondary only if provided by user - apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(s.ServiceClusterIPRanges) - if err != nil { - return options, err - } - s.PrimaryServiceClusterIPRange = primaryServiceIPRange - s.SecondaryServiceClusterIPRange = secondaryServiceIPRange - - if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { - return options, fmt.Errorf("error creating self-signed certificates: %v", err) - } - - if len(s.GenericServerRunOptions.ExternalHost) == 0 { - if len(s.GenericServerRunOptions.AdvertiseAddress) > 0 { - s.GenericServerRunOptions.ExternalHost = s.GenericServerRunOptions.AdvertiseAddress.String() - } else { - if hostname, err := os.Hostname(); err == nil { - s.GenericServerRunOptions.ExternalHost = hostname - } else { - return options, fmt.Errorf("error finding host name: %v", err) - } - } - klog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost) - } - - s.Authentication.ApplyAuthorization(s.Authorization) - - // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling - // TokenRequest functionality. This defaulting was convenient, but messed up - // a lot of people when they rotated their serving cert with no idea it was - // connected to their service account keys. We are taking this opportunity to - // remove this problematic defaulting. - if s.ServiceAccountSigningKeyFile == "" { - // Default to the private server key for service account token signing - if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" { - if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) { - s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile} - } else { - klog.Warning("No TLS key provided, service account token authentication disabled") - } - } - } - - if s.ServiceAccountSigningKeyFile != "" && s.Authentication.ServiceAccounts.Issuer != "" { - sk, err := keyutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) - if err != nil { - return options, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) - } - if s.Authentication.ServiceAccounts.MaxExpiration != 0 { - lowBound := time.Hour - upBound := time.Duration(1<<32) * time.Second - if s.Authentication.ServiceAccounts.MaxExpiration < lowBound || - s.Authentication.ServiceAccounts.MaxExpiration > upBound { - return options, fmt.Errorf("the serviceaccount max expiration must be between 1 hour to 2^32 seconds") - } - if s.Authentication.ServiceAccounts.ExtendExpiration { - if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than 3607 seconds (currently %s)", s.Authentication.ServiceAccounts.MaxExpiration) - } - if s.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to 1 year, which is longer than service-account-max-token-expiration set to %s", s.Authentication.ServiceAccounts.MaxExpiration) - } - } - } - - s.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk) - if err != nil { - return options, fmt.Errorf("failed to build token generator: %v", err) - } - s.ServiceAccountTokenMaxExpiration = s.Authentication.ServiceAccounts.MaxExpiration - } - - if s.Etcd.EnableWatchCache { - sizes := kubeapiserver.DefaultWatchCacheSizes() - // Ensure that overrides parse correctly. - userSpecified, err := serveroptions.ParseWatchCacheSizes(s.Etcd.WatchCacheSizes) - if err != nil { - return options, err - } - for resource, size := range userSpecified { - sizes[resource] = size - } - s.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes) - if err != nil { - return options, err - } - } - - if s.APIEnablement.RuntimeConfig != nil { - for key, value := range s.APIEnablement.RuntimeConfig { - if key == "v1" || strings.HasPrefix(key, "v1/") || - key == "api/v1" || strings.HasPrefix(key, "api/v1/") { - delete(s.APIEnablement.RuntimeConfig, key) - s.APIEnablement.RuntimeConfig["/v1"] = value - } - if key == "api/legacy" { - delete(s.APIEnablement.RuntimeConfig, key) - } - } - } - options.ServerRunOptions = s - return options, nil -} - -func buildServiceResolver(enabledAggregatorRouting bool, hostname string, informer clientgoinformers.SharedInformerFactory) webhook.ServiceResolver { - var serviceResolver webhook.ServiceResolver - if enabledAggregatorRouting { - serviceResolver = aggregatorapiserver.NewEndpointServiceResolver( - informer.Core().V1().Services().Lister(), - informer.Core().V1().Endpoints().Lister(), - ) - } else { - serviceResolver = aggregatorapiserver.NewClusterIPServiceResolver( - informer.Core().V1().Services().Lister(), - ) - } - // resolve kubernetes.default.svc locally - if localHost, err := url.Parse(hostname); err == nil { - serviceResolver = aggregatorapiserver.NewLoopbackServiceResolver(serviceResolver, localHost) - } - return serviceResolver -} - -func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) { - serviceClusterIPRangeList := []string{} - if serviceClusterIPRanges != "" { - serviceClusterIPRangeList = strings.Split(serviceClusterIPRanges, ",") - } - - var apiServerServiceIP net.IP - var primaryServiceIPRange net.IPNet - var secondaryServiceIPRange net.IPNet - var err error - // nothing provided by user, use default range (only applies to the Primary) - if len(serviceClusterIPRangeList) == 0 { - var primaryServiceClusterCIDR net.IPNet - primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err) - } - return apiServerServiceIP, primaryServiceIPRange, net.IPNet{}, nil - } - - if len(serviceClusterIPRangeList) > 0 { - _, primaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[0]) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr") - } - - primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*(primaryServiceClusterCIDR)) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err) - } - } - - // user provided at least two entries - // note: validation asserts that the list is max of two dual stack entries - if len(serviceClusterIPRangeList) > 1 { - _, secondaryServiceClusterCIDR, err := net.ParseCIDR(serviceClusterIPRangeList[1]) - if err != nil { - return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[1] is not an ip net") - } - secondaryServiceIPRange = *secondaryServiceClusterCIDR - } - return apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, nil -} diff --git a/cmd/kube-apiserver/app/server_test.go b/cmd/kube-apiserver/app/server_test.go deleted file mode 100644 index cf53239d8c6a1..0000000000000 --- a/cmd/kube-apiserver/app/server_test.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "testing" -) - -func TestGetServiceIPAndRanges(t *testing.T) { - tests := []struct { - body string - apiServerServiceIP string - primaryServiceIPRange string - secondaryServiceIPRange string - expectedError bool - }{ - {"", "10.0.0.1", "10.0.0.0/24", "", false}, - {"192.0.2.1/24", "192.0.2.1", "192.0.2.0/24", "", false}, - {"192.0.2.1/24,192.168.128.0/17", "192.0.2.1", "192.0.2.0/24", "192.168.128.0/17", false}, - // Dual stack IPv4/IPv6 - {"192.0.2.1/24,2001:db2:1:3:4::1/112", "192.0.2.1", "192.0.2.0/24", "2001:db2:1:3:4::/112", false}, - // Dual stack IPv6/IPv4 - {"2001:db2:1:3:4::1/112,192.0.2.1/24", "2001:db2:1:3:4::1", "2001:db2:1:3:4::/112", "192.0.2.0/24", false}, - - {"192.0.2.1/30,192.168.128.0/17", "", "", "", true}, - // Invalid ip range[0] IPv4 mask - {"192.0.2.1/33,192.168.128.0/17", "", "", "", true}, - // Invalid ip range[1] IPv4 mask - {"192.0.2.1/24,192.168.128.0/33", "", "", "", true}, - // Invalid ip range[0] IPv6 mask - {"2001:db2:1:3:4::1/129,192.0.2.1/24", "", "", "", true}, - // Invalid ip range[1] IPv6 mask - {"192.0.2.1/24,2001:db2:1:3:4::1/129", "", "", "", true}, - // Invalid ip range[0] missing IPv4 mask - {"192.0.2.1,192.168.128.0/17", "", "", "", true}, - // Invalid ip range[1] missing IPv4 mask - {"192.0.2.1/24,192.168.128.1", "", "", "", true}, - // Invalid ip range[0] missing IPv6 mask - {"2001:db2:1:3:4::1,192.0.2.1/24", "", "", "", true}, - // Invalid ip range[1] missing IPv6 mask - {"192.0.2.1/24,2001:db2:1:3:4::1", "", "", "", true}, - // Invalid ip range[0] IP address format - {"bad.ip.range,192.168.0.2/24", "", "", "", true}, - // Invalid ip range[1] IP address format - {"192.168.0.2/24,bad.ip.range", "", "", "", true}, - } - - for _, test := range tests { - apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(test.body) - - if apiServerServiceIP.String() != test.apiServerServiceIP { - t.Errorf("expected apiServerServiceIP: %s, got: %s", test.apiServerServiceIP, apiServerServiceIP.String()) - } - - if primaryServiceIPRange.String() != test.primaryServiceIPRange { - t.Errorf("expected primaryServiceIPRange: %s, got: %s", test.primaryServiceIPRange, primaryServiceIPRange.String()) - } - - if secondaryServiceIPRange.String() != test.secondaryServiceIPRange { - t.Errorf("expected secondaryServiceIPRange: %s, got: %s", test.secondaryServiceIPRange, secondaryServiceIPRange.String()) - } - - if (err == nil) == test.expectedError { - t.Errorf("expected err to be: %t, but it was %t", test.expectedError, !test.expectedError) - } - } -} diff --git a/cmd/kube-apiserver/app/testing/BUILD b/cmd/kube-apiserver/app/testing/BUILD deleted file mode 100644 index baacf9a93a878..0000000000000 --- a/cmd/kube-apiserver/app/testing/BUILD +++ /dev/null @@ -1,41 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["testserver.go"], - data = glob(["testdata/**"]), - importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/testing", - deps = [ - "//cmd/kube-apiserver/app:go_default_library", - "//cmd/kube-apiserver/app/options:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/storageversion:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//test/utils:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt b/cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt deleted file mode 100644 index d144d04f5e56e..0000000000000 --- a/cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.crt +++ /dev/null @@ -1,38 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDTzCCAjegAwIBAgIBAjANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBcxMjcu -MC4wLjEtY2FAMTUzMTQ2NzU5MzAgFw0xODA3MTMwNjM5NTNaGA8yMTE4MDYxOTA2 -Mzk1M1owHzEdMBsGA1UEAwwUMTI3LjAuMC4xQDE1MzE0Njc1OTMwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDdTNF7rRKBvDtLOvCTDgj3utt+zv5u+23x -kCz2475DPnTZ7JK2ipCuHemyCY88M6VyaBkIqAVvvl3LZiS+Hu3gd+8elbdGrCxQ -sui1MrUcAg8OoBM+97UzoKC3HMFIFEpqzKjVJKr5PbV3F8XXIBQeS3YUCePo3m7u -OkGCXUXtWRtQTu4Dcq+tJKlJBsY+Q8CUvb1l0n5hafIFEMnFF/sKGP28CWd8gfzD -ZKKtVumvQlgcp1GdfxqKHfjQOtBo+ZBFiHgDGDrrghuQ2CxROvk5/bNrViqbWbw4 -lUbU3Yn18L4UHR5xOOvQyLP2QdWAaoPutT7Xba40RMgWYlsNRaatAgMBAAGjgZAw -gY0wDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB -/wQCMAAwWAYDVR0RBFEwT4IWa3ViZXJuZXRlcy5kZWZhdWx0LnN2Y4ISa3ViZXJu -ZXRlcy5kZWZhdWx0ggprdWJlcm5ldGVzgglsb2NhbGhvc3SHBH8AAAGHBAoAAAEw -DQYJKoZIhvcNAQELBQADggEBAFkRV1oBLrY3IJDI6E9nxAK30EdyfYZqvybPCZB8 -6AAErj+WleJVFi0rZJ3fRDoQ5Gelwe4Ud21DknW4+L7nZ8JRbzNkLTYTJxtkujSW -aEz7xKW1IxD+o9TEceqiVko4xGawXjUVTun7n0Upv6T4D4jC0GN9zu8oT6xbUHmd -WSSc2HjGLs8vF130xt2Oj0jx03i7AoJF4ZxMRt7dqSK7j5tfflfTS9Dxhmd9Gg5P -eGH4BWJ3IJI3r0+WUtiIgMSgV2ppTSNY2UNbNNpudsRCq55IzyHuRioFt/FH9t+8 -xFaar6D9RDsm87JCv5JZ3BoVZJglmX8iqye+OBXgHgMZxx4= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIC5DCCAcygAwIBAgIBATANBgkqhkiG9w0BAQsFADAiMSAwHgYDVQQDDBcxMjcu -MC4wLjEtY2FAMTUzMTQ2NzU5MzAgFw0xODA3MTMwNjM5NTNaGA8yMTE4MDYxOTA2 -Mzk1M1owIjEgMB4GA1UEAwwXMTI3LjAuMC4xLWNhQDE1MzE0Njc1OTMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRfS+rc4EPpDafeiRo+uIvV1812UQM -ddoaCrn2yIVSYiupsFc7goieXOpqxgI6ksUCMDUOfi3DQGC8067wX2HpMzz5J9yz -Qfamcg3SL7G9u5Vx+x+EU6qmBhXa4Z46JwTY0vYeccz2PR+Nx+HHO0DglIh3tip8 -ECQ2rtpMc5YxJOCwJg3zh8pnEqLNEahm3p1lNGLbY7Kpqp7al68ZVReVg/YaoJt5 -Voi7vbR38OWBChbBmwKRP4gJD8aKY2eY6Xgn8+UAAytYGOEp18y/eAvba7awKp56 -wG1Y3JqWD06D8NnUCPQOO/g/KyGU77sM66xdlsOwLpSbtwWLcjC3nnvHAgMBAAGj -IzAhMA4GA1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEB -CwUAA4IBAQCECJvqzrh57MrVT5sj+TDmhtSwkblBBqNG03X8gvmj7PsYAnZVpaev -KbN0xfDhK6yaCJX41sZCoTaQa55a0Y9n/Lr6d2LREYPf2NdhHU2aj1UjNhhamoZk -0/MJtn/7t6UmYsdFIRlYtLJZQRfNaeO+ULpjjQeGj+Y4mR87VzyDZk2zi/fLJCtk -aVKsI2Tan9KFzwmsCp/9RH7uPhOIFsaa8ePBCvzrahMrG+b9FGV670bQTS104Gyt -HB73ixOheUPL9PuvahXKz0xlJfeKu4nFFJkqUmThj2Ybv8cyzDNrSyDywZxzNe3e -nMA3i/kfmIj33gkmwcFgYPqfKleeVZQo ------END CERTIFICATE----- diff --git a/cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.key b/cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.key deleted file mode 100644 index ec43d5e876666..0000000000000 --- a/cmd/kube-apiserver/app/testing/testdata/127.0.0.1_10.0.0.1_kubernetes.default.svc-kubernetes.default-kubernetes-localhost.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEA3UzRe60Sgbw7Szrwkw4I97rbfs7+bvtt8ZAs9uO+Qz502eyS -toqQrh3psgmPPDOlcmgZCKgFb75dy2Ykvh7t4HfvHpW3RqwsULLotTK1HAIPDqAT -Pve1M6CgtxzBSBRKasyo1SSq+T21dxfF1yAUHkt2FAnj6N5u7jpBgl1F7VkbUE7u -A3KvrSSpSQbGPkPAlL29ZdJ+YWnyBRDJxRf7Chj9vAlnfIH8w2SirVbpr0JYHKdR -nX8aih340DrQaPmQRYh4Axg664IbkNgsUTr5Of2za1Yqm1m8OJVG1N2J9fC+FB0e -cTjr0Miz9kHVgGqD7rU+122uNETIFmJbDUWmrQIDAQABAoIBAFXzdhFhASUeZLEt -bS7Qbq85BfNUlgGo6eS+qJgjkEwxv9S8S6dWXiciOxgJGna5YHL093QjPilOeMN9 -IpwtCxr5ugfZAlwSlwuo0TU/QpRkQFDf31m/f8NTidhU9MT4DIc6ggB2w2kWjJp6 -wz5wmR/DE1NpG/ngGpmwSq1FaNlr3xz4e6b0A56ReqQr5YwYsZl2Fxf8sOBWTiPe -Iv41q8jyRXL2ytv9uTgdD7i+qLMz1/NGvy4ZWxD3yCMsDm5eEI8/4l2pOmRrrpKY -Fc14eUkbHBMyT6ibI4d7Y2aZJslq8d0HMMX1XNLvzLEnGT1+mrOjWwerI+60B0t1 -6EvTfUkCgYEA/rVROq6JupfnT7BM04jEx5UuaokcLoNpn6bptZwmMakioLjgZoa2 -XEZrNoRWVMQ82JuguxkLtUgLgqKQRreolDCyQtaGFjFnWdVs+1W0oIHY7oMdwOLh -XsQRtPW3HCK4HYZJcBBIh8t4USiBTrRtTpXmDinLkbCRXYOcehbRZ2cCgYEA3mwg -tsbnSDYTcFmPjJEGjISfMEjAHoka8ubPKzU6VylDnrb2k/NbDYL3NidzGbJaVJFk -YNfCsja4COz+0pBiMY2fBEzHU4RwDaRrxUr0fLVxvH7/E9JPP8y/e5CJR2Z2sDQa -yed3ArkNh0MaecGr+7IZFbv+Uj4QaBq3W77hGMsCgYB/keC1O2XQBvTbfLl92Sp1 -q8orobBzu23TgI3dX+/hP40QYohB0YbUgJCCJZX3rrGq64d9LfYnPYZGT5VjVekh -D6K4xykxRF03KSYEW9Cz81TrYNAuI3QtOpaDw+2KMfl1ECUH85/gI5CHVXouKT/1 -9C3dOiGzPnQQGjLtEzCeUQKBgFacZGDIM2e7Jvao6W0jTBmLyzFSIv3BBe1wU1vP -7lfiiaJUPNCAAwTP6tP7qKZo/SPROfU8D2S2ShOvtcrozlPdgf56p2OuPrQRQqYg -+fNV9GQiT9G4I4QEhsvnDI3xKGaU45mbuIwm4024o6al9AKe54W/HtmHsXvYa24e -dijhAoGARcbgcE/aT8jhdVHHCRBuSD4ZzXbB+JCetHsrjhOYnifc0graq0umiuRI -c0i+IT5OhGTdVbjnPgySHn/V/IuSYLLtKvfqSV8tQk3womXRPJ/K9BsFhelo1Vd5 -MTyZ2j0XjLWHOo0DKxIPLW3P7sBYAFM2Z+/RAe1uKjISmggDhBs= ------END RSA PRIVATE KEY----- diff --git a/cmd/kube-apiserver/app/testing/testdata/README.md b/cmd/kube-apiserver/app/testing/testdata/README.md deleted file mode 100644 index a78ddfbd05a3f..0000000000000 --- a/cmd/kube-apiserver/app/testing/testdata/README.md +++ /dev/null @@ -1 +0,0 @@ -Keys in this directory are generated for testing purposes only. diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go deleted file mode 100644 index 2babadc747cda..0000000000000 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ /dev/null @@ -1,322 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testing - -import ( - "context" - "fmt" - "io/ioutil" - "net" - "os" - "path" - "path/filepath" - "runtime" - "time" - - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/registry/generic/registry" - "k8s.io/apiserver/pkg/storage/storagebackend" - "k8s.io/apiserver/pkg/storageversion" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/util/cert" - "k8s.io/kubernetes/cmd/kube-apiserver/app" - "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - testutil "k8s.io/kubernetes/test/utils" -) - -// TearDownFunc is to be called to tear down a test server. -type TearDownFunc func() - -// TestServerInstanceOptions Instance options the TestServer -type TestServerInstanceOptions struct { - // DisableStorageCleanup Disable the automatic storage cleanup - DisableStorageCleanup bool - // Enable cert-auth for the kube-apiserver - EnableCertAuth bool - // Wrap the storage version interface of the created server's generic server. - StorageVersionWrapFunc func(storageversion.Manager) storageversion.Manager -} - -// TestServer return values supplied by kube-test-ApiServer -type TestServer struct { - ClientConfig *restclient.Config // Rest client config - ServerOpts *options.ServerRunOptions // ServerOpts - TearDownFn TearDownFunc // TearDown function - TmpDir string // Temp Dir used, by the apiserver -} - -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - -// NewDefaultTestServerOptions Default options for TestServer instances -func NewDefaultTestServerOptions() *TestServerInstanceOptions { - return &TestServerInstanceOptions{ - DisableStorageCleanup: false, - EnableCertAuth: true, - } -} - -// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func, -// and location of the tmpdir are returned. -// -// Note: we return a tear-down func instead of a stop channel because the later will leak temporary -// files that because Golang testing's call to os.Exit will not give a stop channel go routine -// enough time to remove temporary files. -func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) { - if instanceOptions == nil { - instanceOptions = NewDefaultTestServerOptions() - } - - // TODO : Remove TrackStorageCleanup below when PR - // https://github.com/kubernetes/kubernetes/pull/50690 - // merges as that shuts down storage properly - if !instanceOptions.DisableStorageCleanup { - registry.TrackStorageCleanup() - } - - stopCh := make(chan struct{}) - tearDown := func() { - if !instanceOptions.DisableStorageCleanup { - registry.CleanupStorage() - } - close(stopCh) - if len(result.TmpDir) != 0 { - os.RemoveAll(result.TmpDir) - } - } - defer func() { - if result.TearDownFn == nil { - tearDown() - } - }() - - result.TmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver") - if err != nil { - return result, fmt.Errorf("failed to create temp dir: %v", err) - } - - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - - s := options.NewServerRunOptions() - for _, f := range s.Flags().FlagSets { - fs.AddFlagSet(f) - } - - s.SecureServing.Listener, s.SecureServing.BindPort, err = createLocalhostListenerOnFreePort() - if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) - } - s.SecureServing.ServerCert.CertDirectory = result.TmpDir - - if instanceOptions.EnableCertAuth { - // create certificates for aggregation and client-cert auth - proxySigningKey, err := testutil.NewPrivateKey() - if err != nil { - return result, err - } - proxySigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "front-proxy-ca"}, proxySigningKey) - if err != nil { - return result, err - } - proxyCACertFile := path.Join(s.SecureServing.ServerCert.CertDirectory, "proxy-ca.crt") - if err := ioutil.WriteFile(proxyCACertFile, testutil.EncodeCertPEM(proxySigningCert), 0644); err != nil { - return result, err - } - s.Authentication.RequestHeader.ClientCAFile = proxyCACertFile - clientSigningKey, err := testutil.NewPrivateKey() - if err != nil { - return result, err - } - clientSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "client-ca"}, clientSigningKey) - if err != nil { - return result, err - } - clientCACertFile := path.Join(s.SecureServing.ServerCert.CertDirectory, "client-ca.crt") - if err := ioutil.WriteFile(clientCACertFile, testutil.EncodeCertPEM(clientSigningCert), 0644); err != nil { - return result, err - } - s.Authentication.ClientCert.ClientCA = clientCACertFile - } - - s.SecureServing.ExternalAddress = s.SecureServing.Listener.Addr().(*net.TCPAddr).IP // use listener addr although it is a loopback device - - pkgPath, err := pkgPath(t) - if err != nil { - return result, err - } - s.SecureServing.ServerCert.FixtureDirectory = filepath.Join(pkgPath, "testdata") - - s.ServiceClusterIPRanges = "10.0.0.0/16" - s.Etcd.StorageConfig = *storageConfig - s.APIEnablement.RuntimeConfig.Set("api/all=true") - - if err := fs.Parse(customFlags); err != nil { - return result, err - } - completedOptions, err := app.Complete(s) - if err != nil { - return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err) - } - - t.Logf("runtime-config=%v", completedOptions.APIEnablement.RuntimeConfig) - t.Logf("Starting kube-apiserver on port %d...", s.SecureServing.BindPort) - server, err := app.CreateServerChain(completedOptions, stopCh) - if instanceOptions.StorageVersionWrapFunc != nil { - server.GenericAPIServer.StorageVersionManager = instanceOptions.StorageVersionWrapFunc(server.GenericAPIServer.StorageVersionManager) - } - if err != nil { - return result, fmt.Errorf("failed to create server chain: %v", err) - } - - errCh := make(chan error) - go func(stopCh <-chan struct{}) { - prepared, err := server.PrepareRun() - if err != nil { - errCh <- err - } else if err := prepared.Run(stopCh); err != nil { - errCh <- err - } - }(stopCh) - - // skip healthz check when we test the storage version manager poststart hook - if instanceOptions.StorageVersionWrapFunc == nil { - t.Logf("Waiting for /healthz to be ok...") - - client, err := kubernetes.NewForConfig(server.GenericAPIServer.LoopbackClientConfig) - if err != nil { - return result, fmt.Errorf("failed to create a client: %v", err) - } - - // wait until healthz endpoint returns ok - err = wait.Poll(100*time.Millisecond, time.Minute, func() (bool, error) { - select { - case err := <-errCh: - return false, err - default: - } - - result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()) - status := 0 - result.StatusCode(&status) - if status == 200 { - return true, nil - } - return false, nil - }) - if err != nil { - return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) - } - - // wait until default namespace is created - err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { - select { - case err := <-errCh: - return false, err - default: - } - - if _, err := client.CoreV1().Namespaces().Get(context.TODO(), "default", metav1.GetOptions{}); err != nil { - if !errors.IsNotFound(err) { - t.Logf("Unable to get default namespace: %v", err) - } - return false, nil - } - return true, nil - }) - if err != nil { - return result, fmt.Errorf("failed to wait for default namespace to be created: %v", err) - } - } - - // from here the caller must call tearDown - result.ClientConfig = restclient.CopyConfig(server.GenericAPIServer.LoopbackClientConfig) - result.ClientConfig.QPS = 1000 - result.ClientConfig.Burst = 10000 - result.ServerOpts = s - result.TearDownFn = tearDown - - return result, nil -} - -// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, instanceOptions *TestServerInstanceOptions, flags []string, storageConfig *storagebackend.Config) *TestServer { - result, err := StartTestServer(t, instanceOptions, flags, storageConfig) - if err == nil { - return &result - } - - t.Fatalf("failed to launch server: %v", err) - return nil -} - -func createLocalhostListenerOnFreePort() (net.Listener, int, error) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return nil, 0, err - } - - // get port - tcpAddr, ok := ln.Addr().(*net.TCPAddr) - if !ok { - ln.Close() - return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) - } - - return ln, tcpAddr.Port, nil -} - -// pkgPath returns the absolute file path to this package's directory. With go -// test, we can just look at the runtime call stack. However, bazel compiles go -// binaries with the -trimpath option so the simple approach fails however we -// can consult environment variables to derive the path. -// -// The approach taken here works for both go test and bazel on the assumption -// that if and only if trimpath is passed, we are running under bazel. -func pkgPath(t Logger) (string, error) { - _, thisFile, _, ok := runtime.Caller(0) - if !ok { - return "", fmt.Errorf("failed to get current file") - } - - pkgPath := filepath.Dir(thisFile) - - // If we find bazel env variables, then -trimpath was passed so we need to - // construct the path from the environment. - if testSrcdir, testWorkspace := os.Getenv("TEST_SRCDIR"), os.Getenv("TEST_WORKSPACE"); testSrcdir != "" && testWorkspace != "" { - t.Logf("Detected bazel env varaiables: TEST_SRCDIR=%q TEST_WORKSPACE=%q", testSrcdir, testWorkspace) - pkgPath = filepath.Join(testSrcdir, testWorkspace, pkgPath) - } - - // If the path is still not absolute, something other than bazel compiled - // with -trimpath. - if !filepath.IsAbs(pkgPath) { - return "", fmt.Errorf("can't construct an absolute path from %q", pkgPath) - } - - t.Logf("Resolved testserver package path to: %q", pkgPath) - - return pkgPath, nil -} diff --git a/cmd/kube-controller-manager/BUILD b/cmd/kube-controller-manager/BUILD deleted file mode 100644 index f72156e9676e5..0000000000000 --- a/cmd/kube-controller-manager/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kube-controller-manager", - embed = [":go_default_library"], - pure = "on", - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["controller-manager.go"], - importpath = "k8s.io/kubernetes/cmd/kube-controller-manager", - deps = [ - "//cmd/kube-controller-manager/app:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-controller-manager/app:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kube-controller-manager/OWNERS b/cmd/kube-controller-manager/OWNERS deleted file mode 100644 index d8eba068cd0eb..0000000000000 --- a/cmd/kube-controller-manager/OWNERS +++ /dev/null @@ -1,56 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- cheftako -- deads2k -- lavalamp -- mikedanese -- sttts -reviewers: -- a-robinson -- brendandburns -- caesarxuchao -- cheftako -- cjcullen -- davidopp -- deads2k -- derekwaynecarr -- erictune -- errordeveloper -- feiskyer -- gmarek -- humblec -- ingvagabund -- janetkuo -- jayunit100 -- jlowdermilk -- johscheuer -- jsafrane -- justinsb -- lavalamp -- liggitt -- luxas -- madhusudancs -- mikedanese -- mml -- mwielgus -- nikhiljindal -- ping035627 -- piosz -- pmorie -- quinton-hoole -- resouer -- rootfs -- saad-ali -- screeley44 -- sjenning -- smarterclayton -- soltysh -- spiffxp -- stewart-yu -- sttts -- thockin -- timothysc -- wojtek-t -labels: -- sig/api-machinery diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD deleted file mode 100644 index bf098a302a94f..0000000000000 --- a/cmd/kube-controller-manager/app/BUILD +++ /dev/null @@ -1,200 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "apps.go", - "autoscaling.go", - "batch.go", - "bootstrap.go", - "certificates.go", - "cloudproviders.go", - "controllermanager.go", - "core.go", - "discovery.go", - "flags_providers.go", - "import_known_versions.go", - "plugins.go", - "plugins_providers.go", - "policy.go", - "rbac.go", - ], - importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kube-controller-manager/app/config:go_default_library", - "//cmd/kube-controller-manager/app/options:go_default_library", - "//pkg/apis/apps/install:go_default_library", - "//pkg/apis/authentication/install:go_default_library", - "//pkg/apis/authorization/install:go_default_library", - "//pkg/apis/autoscaling/install:go_default_library", - "//pkg/apis/batch/install:go_default_library", - "//pkg/apis/certificates/install:go_default_library", - "//pkg/apis/coordination/install:go_default_library", - "//pkg/apis/core/install:go_default_library", - "//pkg/apis/events/install:go_default_library", - "//pkg/apis/extensions/install:go_default_library", - "//pkg/apis/policy/install:go_default_library", - "//pkg/apis/rbac/install:go_default_library", - "//pkg/apis/scheduling/install:go_default_library", - "//pkg/apis/storage/install:go_default_library", - "//pkg/cloudprovider/providers:go_default_library", - "//pkg/controller:go_default_library", - "//pkg/controller/apis/config:go_default_library", - "//pkg/controller/bootstrap:go_default_library", - "//pkg/controller/certificates/approver:go_default_library", - "//pkg/controller/certificates/cleaner:go_default_library", - "//pkg/controller/certificates/rootcacertpublisher:go_default_library", - "//pkg/controller/certificates/signer:go_default_library", - "//pkg/controller/certificates/signer/config:go_default_library", - "//pkg/controller/clusterroleaggregation:go_default_library", - "//pkg/controller/cronjob:go_default_library", - "//pkg/controller/daemon:go_default_library", - "//pkg/controller/deployment:go_default_library", - "//pkg/controller/disruption:go_default_library", - "//pkg/controller/endpoint:go_default_library", - "//pkg/controller/endpointslice:go_default_library", - "//pkg/controller/endpointslicemirroring:go_default_library", - "//pkg/controller/garbagecollector:go_default_library", - "//pkg/controller/job:go_default_library", - "//pkg/controller/namespace:go_default_library", - "//pkg/controller/nodeipam:go_default_library", - "//pkg/controller/nodeipam/config:go_default_library", - "//pkg/controller/nodeipam/ipam:go_default_library", - "//pkg/controller/nodelifecycle:go_default_library", - "//pkg/controller/podautoscaler:go_default_library", - "//pkg/controller/podautoscaler/metrics:go_default_library", - "//pkg/controller/podgc:go_default_library", - "//pkg/controller/replicaset:go_default_library", - "//pkg/controller/replication:go_default_library", - "//pkg/controller/resourcequota:go_default_library", - "//pkg/controller/serviceaccount:go_default_library", - "//pkg/controller/statefulset:go_default_library", - "//pkg/controller/storageversiongc:go_default_library", - "//pkg/controller/ttl:go_default_library", - "//pkg/controller/ttlafterfinished:go_default_library", - "//pkg/controller/volume/attachdetach:go_default_library", - "//pkg/controller/volume/ephemeral:go_default_library", - "//pkg/controller/volume/expand:go_default_library", - "//pkg/controller/volume/persistentvolume:go_default_library", - "//pkg/controller/volume/persistentvolume/config:go_default_library", - "//pkg/controller/volume/pvcprotection:go_default_library", - "//pkg/controller/volume/pvprotection:go_default_library", - "//pkg/features:go_default_library", - "//pkg/quota/v1/install:go_default_library", - "//pkg/serviceaccount:go_default_library", - "//pkg/volume:go_default_library", - "//pkg/volume/awsebs:go_default_library", - "//pkg/volume/azure_file:go_default_library", - "//pkg/volume/azuredd:go_default_library", - "//pkg/volume/cinder:go_default_library", - "//pkg/volume/csi:go_default_library", - "//pkg/volume/csimigration:go_default_library", - "//pkg/volume/fc:go_default_library", - "//pkg/volume/flexvolume:go_default_library", - "//pkg/volume/flocker:go_default_library", - "//pkg/volume/gcepd:go_default_library", - "//pkg/volume/glusterfs:go_default_library", - "//pkg/volume/hostpath:go_default_library", - "//pkg/volume/iscsi:go_default_library", - "//pkg/volume/local:go_default_library", - "//pkg/volume/nfs:go_default_library", - "//pkg/volume/portworx:go_default_library", - "//pkg/volume/quobyte:go_default_library", - "//pkg/volume/rbd:go_default_library", - "//pkg/volume/scaleio:go_default_library", - "//pkg/volume/storageos:go_default_library", - "//pkg/volume/util:go_default_library", - "//pkg/volume/vsphere_volume:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/discovery/v1beta1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/features:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/quota/v1/generic:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/discovery/cached:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/informers/storage/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/metadata:go_default_library", - "//staging/src/k8s.io/client-go/metadata/metadatainformer:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/restmapper:go_default_library", - "//staging/src/k8s.io/client-go/scale:go_default_library", - "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", - "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/flowcontrol:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//staging/src/k8s.io/cloud-provider:go_default_library", - "//staging/src/k8s.io/cloud-provider/controllers/nodelifecycle:go_default_library", - "//staging/src/k8s.io/cloud-provider/controllers/route:go_default_library", - "//staging/src/k8s.io/cloud-provider/controllers/service:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", - "//staging/src/k8s.io/component-base/configz:go_default_library", - "//staging/src/k8s.io/component-base/featuregate:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/ratelimiter:go_default_library", - "//staging/src/k8s.io/component-base/term:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//staging/src/k8s.io/controller-manager/app:go_default_library", - "//staging/src/k8s.io/controller-manager/pkg/clientbuilder:go_default_library", - "//staging/src/k8s.io/controller-manager/pkg/informerfactory:go_default_library", - "//staging/src/k8s.io/csi-translation-lib:go_default_library", - "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", - "//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library", - "//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library", - "//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "certificates_test.go", - "core_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//pkg/controller/certificates/signer/config:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/discovery:go_default_library", - "//staging/src/k8s.io/client-go/discovery/fake:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-controller-manager/app/config:all-srcs", - "//cmd/kube-controller-manager/app/options:all-srcs", - "//cmd/kube-controller-manager/app/testing:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kube-controller-manager/app/apps.go b/cmd/kube-controller-manager/app/apps.go deleted file mode 100644 index 88727a1977e23..0000000000000 --- a/cmd/kube-controller-manager/app/apps.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "fmt" - "net/http" - "time" - - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/util/flowcontrol" - "k8s.io/kubernetes/pkg/controller/daemon" - "k8s.io/kubernetes/pkg/controller/deployment" - "k8s.io/kubernetes/pkg/controller/replicaset" - "k8s.io/kubernetes/pkg/controller/statefulset" -) - -func startDaemonSetController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}] { - return nil, false, nil - } - dsc, err := daemon.NewDaemonSetsController( - ctx.InformerFactory.Apps().V1().DaemonSets(), - ctx.InformerFactory.Apps().V1().ControllerRevisions(), - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().Nodes(), - ctx.ClientBuilder.ClientOrDie("daemon-set-controller"), - flowcontrol.NewBackOff(1*time.Second, 15*time.Minute), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating DaemonSets controller: %v", err) - } - go dsc.Run(int(ctx.ComponentConfig.DaemonSetController.ConcurrentDaemonSetSyncs), ctx.Stop) - return nil, true, nil -} - -func startStatefulSetController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"}] { - return nil, false, nil - } - go statefulset.NewStatefulSetController( - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Apps().V1().StatefulSets(), - ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), - ctx.InformerFactory.Apps().V1().ControllerRevisions(), - ctx.ClientBuilder.ClientOrDie("statefulset-controller"), - ).Run(int(ctx.ComponentConfig.StatefulSetController.ConcurrentStatefulSetSyncs), ctx.Stop) - return nil, true, nil -} - -func startReplicaSetController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"}] { - return nil, false, nil - } - go replicaset.NewReplicaSetController( - ctx.InformerFactory.Apps().V1().ReplicaSets(), - ctx.InformerFactory.Core().V1().Pods(), - ctx.ClientBuilder.ClientOrDie("replicaset-controller"), - replicaset.BurstReplicas, - ).Run(int(ctx.ComponentConfig.ReplicaSetController.ConcurrentRSSyncs), ctx.Stop) - return nil, true, nil -} - -func startDeploymentController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}] { - return nil, false, nil - } - dc, err := deployment.NewDeploymentController( - ctx.InformerFactory.Apps().V1().Deployments(), - ctx.InformerFactory.Apps().V1().ReplicaSets(), - ctx.InformerFactory.Core().V1().Pods(), - ctx.ClientBuilder.ClientOrDie("deployment-controller"), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating Deployment controller: %v", err) - } - go dc.Run(int(ctx.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs), ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/autoscaling.go b/cmd/kube-controller-manager/app/autoscaling.go deleted file mode 100644 index 98fb1b56a593c..0000000000000 --- a/cmd/kube-controller-manager/app/autoscaling.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "net/http" - - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/scale" - "k8s.io/kubernetes/pkg/controller/podautoscaler" - "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" - - resourceclient "k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1" - "k8s.io/metrics/pkg/client/custom_metrics" - "k8s.io/metrics/pkg/client/external_metrics" -) - -func startHPAController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "autoscaling", Version: "v1", Resource: "horizontalpodautoscalers"}] { - return nil, false, nil - } - - if ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerUseRESTClients { - // use the new-style clients if support for custom metrics is enabled - return startHPAControllerWithRESTClient(ctx) - } - - return startHPAControllerWithLegacyClient(ctx) -} - -func startHPAControllerWithRESTClient(ctx ControllerContext) (http.Handler, bool, error) { - clientConfig := ctx.ClientBuilder.ConfigOrDie("horizontal-pod-autoscaler") - hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler") - - apiVersionsGetter := custom_metrics.NewAvailableAPIsGetter(hpaClient.Discovery()) - // invalidate the discovery information roughly once per resync interval our API - // information is *at most* two resync intervals old. - go custom_metrics.PeriodicallyInvalidate( - apiVersionsGetter, - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration, - ctx.Stop) - - metricsClient := metrics.NewRESTMetricsClient( - resourceclient.NewForConfigOrDie(clientConfig), - custom_metrics.NewForConfig(clientConfig, ctx.RESTMapper, apiVersionsGetter), - external_metrics.NewForConfigOrDie(clientConfig), - ) - return startHPAControllerWithMetricsClient(ctx, metricsClient) -} - -func startHPAControllerWithLegacyClient(ctx ControllerContext) (http.Handler, bool, error) { - hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler") - metricsClient := metrics.NewHeapsterMetricsClient( - hpaClient, - metrics.DefaultHeapsterNamespace, - metrics.DefaultHeapsterScheme, - metrics.DefaultHeapsterService, - metrics.DefaultHeapsterPort, - ) - return startHPAControllerWithMetricsClient(ctx, metricsClient) -} - -func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient metrics.MetricsClient) (http.Handler, bool, error) { - hpaClient := ctx.ClientBuilder.ClientOrDie("horizontal-pod-autoscaler") - hpaClientConfig := ctx.ClientBuilder.ConfigOrDie("horizontal-pod-autoscaler") - - // we don't use cached discovery because DiscoveryScaleKindResolver does its own caching, - // so we want to re-fetch every time when we actually ask for it - scaleKindResolver := scale.NewDiscoveryScaleKindResolver(hpaClient.Discovery()) - scaleClient, err := scale.NewForConfig(hpaClientConfig, ctx.RESTMapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) - if err != nil { - return nil, false, err - } - - go podautoscaler.NewHorizontalController( - hpaClient.CoreV1(), - scaleClient, - hpaClient.AutoscalingV1(), - ctx.RESTMapper, - metricsClient, - ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(), - ctx.InformerFactory.Core().V1().Pods(), - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerSyncPeriod.Duration, - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerTolerance, - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, - ctx.ComponentConfig.HPAController.HorizontalPodAutoscalerInitialReadinessDelay.Duration, - ).Run(ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/batch.go b/cmd/kube-controller-manager/app/batch.go deleted file mode 100644 index f42dfca418e81..0000000000000 --- a/cmd/kube-controller-manager/app/batch.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "fmt" - "net/http" - - "k8s.io/apimachinery/pkg/runtime/schema" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/controller/cronjob" - "k8s.io/kubernetes/pkg/controller/job" - kubefeatures "k8s.io/kubernetes/pkg/features" -) - -func startJobController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"}] { - return nil, false, nil - } - go job.NewController( - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Batch().V1().Jobs(), - ctx.ClientBuilder.ClientOrDie("job-controller"), - ).Run(int(ctx.ComponentConfig.JobController.ConcurrentJobSyncs), ctx.Stop) - return nil, true, nil -} - -func startCronJobController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "batch", Version: "v1beta1", Resource: "cronjobs"}] { - return nil, false, nil - } - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.CronJobControllerV2) { - cj2c, err := cronjob.NewControllerV2(ctx.InformerFactory.Batch().V1().Jobs(), - ctx.InformerFactory.Batch().V1beta1().CronJobs(), - ctx.ClientBuilder.ClientOrDie("cronjob-controller"), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating CronJob controller V2: %v", err) - } - go cj2c.Run(int(ctx.ComponentConfig.CronJobController.ConcurrentCronJobSyncs), ctx.Stop) - return nil, true, nil - } - cjc, err := cronjob.NewController( - ctx.ClientBuilder.ClientOrDie("cronjob-controller"), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating CronJob controller: %v", err) - } - go cjc.Run(ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/bootstrap.go b/cmd/kube-controller-manager/app/bootstrap.go deleted file mode 100644 index 75e8a113f6726..0000000000000 --- a/cmd/kube-controller-manager/app/bootstrap.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "net/http" - - "k8s.io/kubernetes/pkg/controller/bootstrap" -) - -func startBootstrapSignerController(ctx ControllerContext) (http.Handler, bool, error) { - bsc, err := bootstrap.NewSigner( - ctx.ClientBuilder.ClientOrDie("bootstrap-signer"), - ctx.InformerFactory.Core().V1().Secrets(), - ctx.InformerFactory.Core().V1().ConfigMaps(), - bootstrap.DefaultSignerOptions(), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating BootstrapSigner controller: %v", err) - } - go bsc.Run(ctx.Stop) - return nil, true, nil -} - -func startTokenCleanerController(ctx ControllerContext) (http.Handler, bool, error) { - tcc, err := bootstrap.NewTokenCleaner( - ctx.ClientBuilder.ClientOrDie("token-cleaner"), - ctx.InformerFactory.Core().V1().Secrets(), - bootstrap.DefaultTokenCleanerOptions(), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating TokenCleaner controller: %v", err) - } - go tcc.Run(ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/certificates.go b/cmd/kube-controller-manager/app/certificates.go deleted file mode 100644 index f8768eda4f382..0000000000000 --- a/cmd/kube-controller-manager/app/certificates.go +++ /dev/null @@ -1,223 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "fmt" - "net/http" - - "k8s.io/apimachinery/pkg/runtime/schema" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/controller/certificates/approver" - "k8s.io/kubernetes/pkg/controller/certificates/cleaner" - "k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher" - "k8s.io/kubernetes/pkg/controller/certificates/signer" - csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config" - "k8s.io/kubernetes/pkg/features" -) - -func startCSRSigningController(ctx ControllerContext) (http.Handler, bool, error) { - gvr := schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1", Resource: "certificatesigningrequests"} - if !ctx.AvailableResources[gvr] { - klog.Warningf("Resource %s is not available now", gvr.String()) - return nil, false, nil - } - missingSingleSigningFile := ctx.ComponentConfig.CSRSigningController.ClusterSigningCertFile == "" || ctx.ComponentConfig.CSRSigningController.ClusterSigningKeyFile == "" - if missingSingleSigningFile && !anySpecificFilesSet(ctx.ComponentConfig.CSRSigningController) { - klog.V(2).Info("skipping CSR signer controller because no csr cert/key was specified") - return nil, false, nil - } - if !missingSingleSigningFile && anySpecificFilesSet(ctx.ComponentConfig.CSRSigningController) { - return nil, false, fmt.Errorf("cannot specify default and per controller certs at the same time") - } - - c := ctx.ClientBuilder.ClientOrDie("certificate-controller") - csrInformer := ctx.InformerFactory.Certificates().V1().CertificateSigningRequests() - certTTL := ctx.ComponentConfig.CSRSigningController.ClusterSigningDuration.Duration - - if kubeletServingSignerCertFile, kubeletServingSignerKeyFile := getKubeletServingSignerFiles(ctx.ComponentConfig.CSRSigningController); len(kubeletServingSignerCertFile) > 0 || len(kubeletServingSignerKeyFile) > 0 { - kubeletServingSigner, err := signer.NewKubeletServingCSRSigningController(c, csrInformer, kubeletServingSignerCertFile, kubeletServingSignerKeyFile, certTTL) - if err != nil { - return nil, false, fmt.Errorf("failed to start kubernetes.io/kubelet-serving certificate controller: %v", err) - } - go kubeletServingSigner.Run(1, ctx.Stop) - } else { - klog.V(2).Infof("skipping CSR signer controller %q because specific files were specified for other signers and not this one.", "kubernetes.io/kubelet-serving") - } - - if kubeletClientSignerCertFile, kubeletClientSignerKeyFile := getKubeletClientSignerFiles(ctx.ComponentConfig.CSRSigningController); len(kubeletClientSignerCertFile) > 0 || len(kubeletClientSignerKeyFile) > 0 { - kubeletClientSigner, err := signer.NewKubeletClientCSRSigningController(c, csrInformer, kubeletClientSignerCertFile, kubeletClientSignerKeyFile, certTTL) - if err != nil { - return nil, false, fmt.Errorf("failed to start kubernetes.io/kube-apiserver-client-kubelet certificate controller: %v", err) - } - go kubeletClientSigner.Run(1, ctx.Stop) - } else { - klog.V(2).Infof("skipping CSR signer controller %q because specific files were specified for other signers and not this one.", "kubernetes.io/kube-apiserver-client-kubelet") - } - - if kubeAPIServerSignerCertFile, kubeAPIServerSignerKeyFile := getKubeAPIServerClientSignerFiles(ctx.ComponentConfig.CSRSigningController); len(kubeAPIServerSignerCertFile) > 0 || len(kubeAPIServerSignerKeyFile) > 0 { - kubeAPIServerClientSigner, err := signer.NewKubeAPIServerClientCSRSigningController(c, csrInformer, kubeAPIServerSignerCertFile, kubeAPIServerSignerKeyFile, certTTL) - if err != nil { - return nil, false, fmt.Errorf("failed to start kubernetes.io/kube-apiserver-client certificate controller: %v", err) - } - go kubeAPIServerClientSigner.Run(1, ctx.Stop) - } else { - klog.V(2).Infof("skipping CSR signer controller %q because specific files were specified for other signers and not this one.", "kubernetes.io/kube-apiserver-client") - } - - if legacyUnknownSignerCertFile, legacyUnknownSignerKeyFile := getLegacyUnknownSignerFiles(ctx.ComponentConfig.CSRSigningController); len(legacyUnknownSignerCertFile) > 0 || len(legacyUnknownSignerKeyFile) > 0 { - legacyUnknownSigner, err := signer.NewLegacyUnknownCSRSigningController(c, csrInformer, legacyUnknownSignerCertFile, legacyUnknownSignerKeyFile, certTTL) - if err != nil { - return nil, false, fmt.Errorf("failed to start kubernetes.io/legacy-unknown certificate controller: %v", err) - } - go legacyUnknownSigner.Run(1, ctx.Stop) - } else { - klog.V(2).Infof("skipping CSR signer controller %q because specific files were specified for other signers and not this one.", "kubernetes.io/legacy-unknown") - } - - return nil, true, nil -} - -func areKubeletServingSignerFilesSpecified(config csrsigningconfig.CSRSigningControllerConfiguration) bool { - if len(config.KubeletServingSignerConfiguration.CertFile) > 0 || len(config.KubeletServingSignerConfiguration.KeyFile) > 0 { - // if only one is specified, it will error later during construction - return true - } - return false -} -func areKubeletClientSignerFilesSpecified(config csrsigningconfig.CSRSigningControllerConfiguration) bool { - if len(config.KubeletClientSignerConfiguration.CertFile) > 0 || len(config.KubeletClientSignerConfiguration.KeyFile) > 0 { - // if only one is specified, it will error later during construction - return true - } - return false -} - -func areKubeAPIServerClientSignerFilesSpecified(config csrsigningconfig.CSRSigningControllerConfiguration) bool { - if len(config.KubeAPIServerClientSignerConfiguration.CertFile) > 0 || len(config.KubeAPIServerClientSignerConfiguration.KeyFile) > 0 { - // if only one is specified, it will error later during construction - return true - } - return false -} - -func areLegacyUnknownSignerFilesSpecified(config csrsigningconfig.CSRSigningControllerConfiguration) bool { - if len(config.LegacyUnknownSignerConfiguration.CertFile) > 0 || len(config.LegacyUnknownSignerConfiguration.KeyFile) > 0 { - // if only one is specified, it will error later during construction - return true - } - return false -} - -func anySpecificFilesSet(config csrsigningconfig.CSRSigningControllerConfiguration) bool { - return areKubeletServingSignerFilesSpecified(config) || - areKubeletClientSignerFilesSpecified(config) || - areKubeAPIServerClientSignerFilesSpecified(config) || - areLegacyUnknownSignerFilesSpecified(config) -} - -func getKubeletServingSignerFiles(config csrsigningconfig.CSRSigningControllerConfiguration) (string, string) { - // if any cert/key is set for specific CSR signing loops, then the --cluster-signing-{cert,key}-file are not used for any CSR signing loop. - if anySpecificFilesSet(config) { - return config.KubeletServingSignerConfiguration.CertFile, config.KubeletServingSignerConfiguration.KeyFile - } - return config.ClusterSigningCertFile, config.ClusterSigningKeyFile -} - -func getKubeletClientSignerFiles(config csrsigningconfig.CSRSigningControllerConfiguration) (string, string) { - // if any cert/key is set for specific CSR signing loops, then the --cluster-signing-{cert,key}-file are not used for any CSR signing loop. - if anySpecificFilesSet(config) { - return config.KubeletClientSignerConfiguration.CertFile, config.KubeletClientSignerConfiguration.KeyFile - } - return config.ClusterSigningCertFile, config.ClusterSigningKeyFile -} - -func getKubeAPIServerClientSignerFiles(config csrsigningconfig.CSRSigningControllerConfiguration) (string, string) { - // if any cert/key is set for specific CSR signing loops, then the --cluster-signing-{cert,key}-file are not used for any CSR signing loop. - if anySpecificFilesSet(config) { - return config.KubeAPIServerClientSignerConfiguration.CertFile, config.KubeAPIServerClientSignerConfiguration.KeyFile - } - return config.ClusterSigningCertFile, config.ClusterSigningKeyFile -} - -func getLegacyUnknownSignerFiles(config csrsigningconfig.CSRSigningControllerConfiguration) (string, string) { - // if any cert/key is set for specific CSR signing loops, then the --cluster-signing-{cert,key}-file are not used for any CSR signing loop. - if anySpecificFilesSet(config) { - return config.LegacyUnknownSignerConfiguration.CertFile, config.LegacyUnknownSignerConfiguration.KeyFile - } - return config.ClusterSigningCertFile, config.ClusterSigningKeyFile -} - -func startCSRApprovingController(ctx ControllerContext) (http.Handler, bool, error) { - gvr := schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1", Resource: "certificatesigningrequests"} - if !ctx.AvailableResources[gvr] { - klog.Warningf("Resource %s is not available now", gvr.String()) - return nil, false, nil - } - - approver := approver.NewCSRApprovingController( - ctx.ClientBuilder.ClientOrDie("certificate-controller"), - ctx.InformerFactory.Certificates().V1().CertificateSigningRequests(), - ) - go approver.Run(1, ctx.Stop) - - return nil, true, nil -} - -func startCSRCleanerController(ctx ControllerContext) (http.Handler, bool, error) { - cleaner := cleaner.NewCSRCleanerController( - ctx.ClientBuilder.ClientOrDie("certificate-controller").CertificatesV1().CertificateSigningRequests(), - ctx.InformerFactory.Certificates().V1().CertificateSigningRequests(), - ) - go cleaner.Run(1, ctx.Stop) - return nil, true, nil -} - -func startRootCACertPublisher(ctx ControllerContext) (http.Handler, bool, error) { - if !utilfeature.DefaultFeatureGate.Enabled(features.RootCAConfigMap) { - return nil, false, nil - } - - var ( - rootCA []byte - err error - ) - if ctx.ComponentConfig.SAController.RootCAFile != "" { - if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil { - return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) - } - } else { - rootCA = ctx.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData - } - - sac, err := rootcacertpublisher.NewPublisher( - ctx.InformerFactory.Core().V1().ConfigMaps(), - ctx.InformerFactory.Core().V1().Namespaces(), - ctx.ClientBuilder.ClientOrDie("root-ca-cert-publisher"), - rootCA, - ) - if err != nil { - return nil, true, fmt.Errorf("error creating root CA certificate publisher: %v", err) - } - go sac.Run(1, ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/certificates_test.go b/cmd/kube-controller-manager/app/certificates_test.go deleted file mode 100644 index 7e724598f3de4..0000000000000 --- a/cmd/kube-controller-manager/app/certificates_test.go +++ /dev/null @@ -1,309 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config" -) - -func TestCertSpecified(t *testing.T) { - allConfig := csrsigningconfig.CSRSigningControllerConfiguration{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - KubeletServingSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-serving/cert-file", - KeyFile: "/cluster-signing-kubelet-serving/key-file", - }, - KubeletClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-client/cert-file", - KeyFile: "/cluster-signing-kubelet-client/key-file", - }, - KubeAPIServerClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kube-apiserver-client/cert-file", - KeyFile: "/cluster-signing-kube-apiserver-client/key-file", - }, - LegacyUnknownSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-legacy-unknown/cert-file", - KeyFile: "/cluster-signing-legacy-unknown/key-file", - }, - } - defaultOnly := csrsigningconfig.CSRSigningControllerConfiguration{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - } - specifiedOnly := csrsigningconfig.CSRSigningControllerConfiguration{ - KubeletServingSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-serving/cert-file", - KeyFile: "/cluster-signing-kubelet-serving/key-file", - }, - KubeletClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-client/cert-file", - KeyFile: "/cluster-signing-kubelet-client/key-file", - }, - KubeAPIServerClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kube-apiserver-client/cert-file", - KeyFile: "/cluster-signing-kube-apiserver-client/key-file", - }, - LegacyUnknownSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-legacy-unknown/cert-file", - KeyFile: "/cluster-signing-legacy-unknown/key-file", - }, - } - halfASpecified := csrsigningconfig.CSRSigningControllerConfiguration{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - KubeletServingSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-serving/cert-file", - KeyFile: "/cluster-signing-kubelet-serving/key-file", - }, - KubeletClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-client/cert-file", - KeyFile: "/cluster-signing-kubelet-client/key-file", - }, - } - halfBSpecified := csrsigningconfig.CSRSigningControllerConfiguration{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - KubeAPIServerClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kube-apiserver-client/cert-file", - KeyFile: "/cluster-signing-kube-apiserver-client/key-file", - }, - LegacyUnknownSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-legacy-unknown/cert-file", - KeyFile: "/cluster-signing-legacy-unknown/key-file", - }, - } - - tests := []struct { - name string - config csrsigningconfig.CSRSigningControllerConfiguration - specifiedFn func(config csrsigningconfig.CSRSigningControllerConfiguration) bool - expectedSpecified bool - filesFn func(config csrsigningconfig.CSRSigningControllerConfiguration) (string, string) - expectedCert string - expectedKey string - }{ - { - name: "allConfig-KubeletServingSignerFilesSpecified", - config: allConfig, - specifiedFn: areKubeletServingSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeletServingSignerFiles, - expectedCert: "/cluster-signing-kubelet-serving/cert-file", - expectedKey: "/cluster-signing-kubelet-serving/key-file", - }, - { - name: "defaultOnly-KubeletServingSignerFilesSpecified", - config: defaultOnly, - specifiedFn: areKubeletServingSignerFilesSpecified, - expectedSpecified: false, - filesFn: getKubeletServingSignerFiles, - expectedCert: "/cluster-signing-cert", - expectedKey: "/cluster-signing-key", - }, - { - name: "specifiedOnly-KubeletServingSignerFilesSpecified", - config: specifiedOnly, - specifiedFn: areKubeletServingSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeletServingSignerFiles, - expectedCert: "/cluster-signing-kubelet-serving/cert-file", - expectedKey: "/cluster-signing-kubelet-serving/key-file", - }, - { - name: "halfASpecified-KubeletServingSignerFilesSpecified", - config: halfASpecified, - specifiedFn: areKubeletServingSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeletServingSignerFiles, - expectedCert: "/cluster-signing-kubelet-serving/cert-file", - expectedKey: "/cluster-signing-kubelet-serving/key-file", - }, - { - name: "halfBSpecified-KubeletServingSignerFilesSpecified", - config: halfBSpecified, - specifiedFn: areKubeletServingSignerFilesSpecified, - expectedSpecified: false, - filesFn: getKubeletServingSignerFiles, - expectedCert: "", - expectedKey: "", - }, - - { - name: "allConfig-KubeletClientSignerFiles", - config: allConfig, - specifiedFn: areKubeletClientSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeletClientSignerFiles, - expectedCert: "/cluster-signing-kubelet-client/cert-file", - expectedKey: "/cluster-signing-kubelet-client/key-file", - }, - { - name: "defaultOnly-KubeletClientSignerFiles", - config: defaultOnly, - specifiedFn: areKubeletClientSignerFilesSpecified, - expectedSpecified: false, - filesFn: getKubeletClientSignerFiles, - expectedCert: "/cluster-signing-cert", - expectedKey: "/cluster-signing-key", - }, - { - name: "specifiedOnly-KubeletClientSignerFiles", - config: specifiedOnly, - specifiedFn: areKubeletClientSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeletClientSignerFiles, - expectedCert: "/cluster-signing-kubelet-client/cert-file", - expectedKey: "/cluster-signing-kubelet-client/key-file", - }, - { - name: "halfASpecified-KubeletClientSignerFiles", - config: halfASpecified, - specifiedFn: areKubeletClientSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeletClientSignerFiles, - expectedCert: "/cluster-signing-kubelet-client/cert-file", - expectedKey: "/cluster-signing-kubelet-client/key-file", - }, - { - name: "halfBSpecified-KubeletClientSignerFiles", - config: halfBSpecified, - specifiedFn: areKubeletClientSignerFilesSpecified, - expectedSpecified: false, - filesFn: getKubeletClientSignerFiles, - expectedCert: "", - expectedKey: "", - }, - - { - name: "allConfig-KubeletClientSignerFiles", - config: allConfig, - specifiedFn: areKubeAPIServerClientSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeAPIServerClientSignerFiles, - expectedCert: "/cluster-signing-kube-apiserver-client/cert-file", - expectedKey: "/cluster-signing-kube-apiserver-client/key-file", - }, - { - name: "defaultOnly-KubeletClientSignerFiles", - config: defaultOnly, - specifiedFn: areKubeAPIServerClientSignerFilesSpecified, - expectedSpecified: false, - filesFn: getKubeAPIServerClientSignerFiles, - expectedCert: "/cluster-signing-cert", - expectedKey: "/cluster-signing-key", - }, - { - name: "specifiedOnly-KubeletClientSignerFiles", - config: specifiedOnly, - specifiedFn: areKubeAPIServerClientSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeAPIServerClientSignerFiles, - expectedCert: "/cluster-signing-kube-apiserver-client/cert-file", - expectedKey: "/cluster-signing-kube-apiserver-client/key-file", - }, - { - name: "halfASpecified-KubeletClientSignerFiles", - config: halfASpecified, - specifiedFn: areKubeAPIServerClientSignerFilesSpecified, - expectedSpecified: false, - filesFn: getKubeAPIServerClientSignerFiles, - expectedCert: "", - expectedKey: "", - }, - { - name: "halfBSpecified-KubeletClientSignerFiles", - config: halfBSpecified, - specifiedFn: areKubeAPIServerClientSignerFilesSpecified, - expectedSpecified: true, - filesFn: getKubeAPIServerClientSignerFiles, - expectedCert: "/cluster-signing-kube-apiserver-client/cert-file", - expectedKey: "/cluster-signing-kube-apiserver-client/key-file", - }, - - { - name: "allConfig-LegacyUnknownSignerFiles", - config: allConfig, - specifiedFn: areLegacyUnknownSignerFilesSpecified, - expectedSpecified: true, - filesFn: getLegacyUnknownSignerFiles, - expectedCert: "/cluster-signing-legacy-unknown/cert-file", - expectedKey: "/cluster-signing-legacy-unknown/key-file", - }, - { - name: "defaultOnly-LegacyUnknownSignerFiles", - config: defaultOnly, - specifiedFn: areLegacyUnknownSignerFilesSpecified, - expectedSpecified: false, - filesFn: getLegacyUnknownSignerFiles, - expectedCert: "/cluster-signing-cert", - expectedKey: "/cluster-signing-key", - }, - { - name: "specifiedOnly-LegacyUnknownSignerFiles", - config: specifiedOnly, - specifiedFn: areLegacyUnknownSignerFilesSpecified, - expectedSpecified: true, - filesFn: getLegacyUnknownSignerFiles, - expectedCert: "/cluster-signing-legacy-unknown/cert-file", - expectedKey: "/cluster-signing-legacy-unknown/key-file", - }, - { - name: "halfASpecified-LegacyUnknownSignerFiles", - config: halfASpecified, - specifiedFn: areLegacyUnknownSignerFilesSpecified, - expectedSpecified: false, - filesFn: getLegacyUnknownSignerFiles, - expectedCert: "", - expectedKey: "", - }, - { - name: "halfBSpecified-LegacyUnknownSignerFiles", - config: halfBSpecified, - specifiedFn: areLegacyUnknownSignerFilesSpecified, - expectedSpecified: true, - filesFn: getLegacyUnknownSignerFiles, - expectedCert: "/cluster-signing-legacy-unknown/cert-file", - expectedKey: "/cluster-signing-legacy-unknown/key-file", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - actualSpecified := test.specifiedFn(test.config) - if actualSpecified != test.expectedSpecified { - t.Error(actualSpecified) - } - - actualCert, actualKey := test.filesFn(test.config) - if actualCert != test.expectedCert { - t.Error(actualCert) - } - if actualKey != test.expectedKey { - t.Error(actualKey) - } - }) - } -} diff --git a/cmd/kube-controller-manager/app/cloudproviders.go b/cmd/kube-controller-manager/app/cloudproviders.go deleted file mode 100644 index ee5b0babf1592..0000000000000 --- a/cmd/kube-controller-manager/app/cloudproviders.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "k8s.io/klog/v2" - - "k8s.io/client-go/informers" - cloudprovider "k8s.io/cloud-provider" -) - -// createCloudProvider helps consolidate what is needed for cloud providers, we explicitly list the things -// that the cloud providers need as parameters, so we can control -func createCloudProvider(cloudProvider string, externalCloudVolumePlugin string, cloudConfigFile string, - allowUntaggedCloud bool, sharedInformers informers.SharedInformerFactory) (cloudprovider.Interface, ControllerLoopMode, error) { - var cloud cloudprovider.Interface - var loopMode ControllerLoopMode - var err error - if cloudprovider.IsExternal(cloudProvider) { - loopMode = ExternalLoops - if externalCloudVolumePlugin == "" { - // externalCloudVolumePlugin is temporary until we split all cloud providers out. - // So we just tell the caller that we need to run ExternalLoops without any cloud provider. - return nil, loopMode, nil - } - cloud, err = cloudprovider.InitCloudProvider(externalCloudVolumePlugin, cloudConfigFile) - } else { - cloudprovider.DeprecationWarningForProvider(cloudProvider) - - loopMode = IncludeCloudLoops - cloud, err = cloudprovider.InitCloudProvider(cloudProvider, cloudConfigFile) - } - if err != nil { - return nil, loopMode, fmt.Errorf("cloud provider could not be initialized: %v", err) - } - - if cloud != nil && !cloud.HasClusterID() { - if allowUntaggedCloud { - klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") - } else { - return nil, loopMode, fmt.Errorf("no ClusterID Found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") - } - } - - if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { - informerUserCloud.SetInformers(sharedInformers) - } - return cloud, loopMode, err -} diff --git a/cmd/kube-controller-manager/app/config/BUILD b/cmd/kube-controller-manager/app/config/BUILD deleted file mode 100644 index 2d684633279f1..0000000000000 --- a/cmd/kube-controller-manager/app/config/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["config.go"], - importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/config", - visibility = ["//visibility:public"], - deps = [ - "//pkg/controller/apis/config:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/record:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kube-controller-manager/app/config/config.go b/cmd/kube-controller-manager/app/config/config.go deleted file mode 100644 index d3125b4a512e6..0000000000000 --- a/cmd/kube-controller-manager/app/config/config.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - apiserver "k8s.io/apiserver/pkg/server" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" -) - -// Config is the main context object for the controller manager. -type Config struct { - ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration - - SecureServing *apiserver.SecureServingInfo - // LoopbackClientConfig is a config for a privileged loopback connection - LoopbackClientConfig *restclient.Config - - // TODO: remove deprecated insecure serving - InsecureServing *apiserver.DeprecatedInsecureServingInfo - Authentication apiserver.AuthenticationInfo - Authorization apiserver.AuthorizationInfo - - // the general kube client - Client *clientset.Clientset - - // the client only used for leader election - LeaderElectionClient *clientset.Clientset - - // the rest config for the master - Kubeconfig *restclient.Config - - // the event sink - EventRecorder record.EventRecorder -} - -type completedConfig struct { - *Config -} - -// CompletedConfig same as Config, just to swap private object. -type CompletedConfig struct { - // Embed a private pointer that cannot be instantiated outside of this package. - *completedConfig -} - -// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() *CompletedConfig { - cc := completedConfig{c} - - apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) - - return &CompletedConfig{&cc} -} diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go deleted file mode 100644 index 5bc022dd2f043..0000000000000 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ /dev/null @@ -1,643 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "context" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "time" - - "github.com/spf13/cobra" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apimachinery/pkg/util/wait" - genericfeatures "k8s.io/apiserver/pkg/features" - "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/apiserver/pkg/server/mux" - utilfeature "k8s.io/apiserver/pkg/util/feature" - cacheddiscovery "k8s.io/client-go/discovery/cached" - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/metadata" - "k8s.io/client-go/metadata/metadatainformer" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - cloudprovider "k8s.io/cloud-provider" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/cli/globalflag" - "k8s.io/component-base/configz" - "k8s.io/component-base/term" - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/controller-manager/pkg/clientbuilder" - "k8s.io/controller-manager/pkg/informerfactory" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" - "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - "k8s.io/kubernetes/pkg/controller" - kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" - serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" - "k8s.io/kubernetes/pkg/serviceaccount" -) - -const ( - // ControllerStartJitter is the Jitter used when starting controller managers - ControllerStartJitter = 1.0 - // ConfigzName is the name used for register kube-controller manager /configz, same with GroupName. - ConfigzName = "kubecontrollermanager.config.k8s.io" -) - -// ControllerLoopMode is the kube-controller-manager's mode of running controller loops that are cloud provider dependent -type ControllerLoopMode int - -const ( - // IncludeCloudLoops means the kube-controller-manager include the controller loops that are cloud provider dependent - IncludeCloudLoops ControllerLoopMode = iota - // ExternalLoops means the kube-controller-manager exclude the controller loops that are cloud provider dependent - ExternalLoops -) - -// NewControllerManagerCommand creates a *cobra.Command object with default parameters -func NewControllerManagerCommand() *cobra.Command { - s, err := options.NewKubeControllerManagerOptions() - if err != nil { - klog.Fatalf("unable to initialize command options: %v", err) - } - - cmd := &cobra.Command{ - Use: "kube-controller-manager", - Long: `The Kubernetes controller manager is a daemon that embeds -the core control loops shipped with Kubernetes. In applications of robotics and -automation, a control loop is a non-terminating loop that regulates the state of -the system. In Kubernetes, a controller is a control loop that watches the shared -state of the cluster through the apiserver and makes changes attempting to move the -current state towards the desired state. Examples of controllers that ship with -Kubernetes today are the replication controller, endpoints controller, namespace -controller, and serviceaccounts controller.`, - PersistentPreRunE: func(*cobra.Command, []string) error { - // silence client-go warnings. - // kube-controller-manager generically watches APIs (including deprecated ones), - // and CI ensures it works properly against matching kube-apiserver versions. - restclient.SetDefaultWarningHandler(restclient.NoWarnings{}) - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - verflag.PrintAndExitIfRequested() - cliflag.PrintFlags(cmd.Flags()) - - c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - if err := Run(c.Complete(), wait.NeverStop); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - - fs := cmd.Flags() - namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List()) - verflag.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) - registerLegacyGlobalFlags(namedFlagSets) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - usageFmt := "Usage:\n %s\n" - cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) - cmd.SetUsageFunc(func(cmd *cobra.Command) error { - fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) - return nil - }) - cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { - fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) - }) - - return cmd -} - -// ResyncPeriod returns a function which generates a duration each time it is -// invoked; this is so that multiple controllers don't get into lock-step and all -// hammer the apiserver with list requests simultaneously. -func ResyncPeriod(c *config.CompletedConfig) func() time.Duration { - return func() time.Duration { - factor := rand.Float64() + 1 - return time.Duration(float64(c.ComponentConfig.Generic.MinResyncPeriod.Nanoseconds()) * factor) - } -} - -// Run runs the KubeControllerManagerOptions. This should never exit. -func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { - // To help debugging, immediately log version - klog.Infof("Version: %+v", version.Get()) - - if cfgz, err := configz.New(ConfigzName); err == nil { - cfgz.Set(c.ComponentConfig) - } else { - klog.Errorf("unable to register configz: %v", err) - } - - // Setup any healthz checks we will want to use. - var checks []healthz.HealthChecker - var electionChecker *leaderelection.HealthzAdaptor - if c.ComponentConfig.Generic.LeaderElection.LeaderElect { - electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20) - checks = append(checks, electionChecker) - } - - // Start the controller manager HTTP server - // unsecuredMux is the handler for these controller *after* authn/authz filters have been applied - var unsecuredMux *mux.PathRecorderMux - if c.SecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) - handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, &c.Authorization, &c.Authentication) - // TODO: handle stoppedCh returned by c.SecureServing.Serve - if _, err := c.SecureServing.Serve(handler, 0, stopCh); err != nil { - return err - } - } - if c.InsecureServing != nil { - unsecuredMux = genericcontrollermanager.NewBaseHandler(&c.ComponentConfig.Generic.Debugging, checks...) - insecureSuperuserAuthn := server.AuthenticationInfo{Authenticator: &server.InsecureSuperuser{}} - handler := genericcontrollermanager.BuildHandlerChain(unsecuredMux, nil, &insecureSuperuserAuthn) - if err := c.InsecureServing.Serve(handler, 0, stopCh); err != nil { - return err - } - } - - run := func(ctx context.Context) { - rootClientBuilder := clientbuilder.SimpleControllerClientBuilder{ - ClientConfig: c.Kubeconfig, - } - var clientBuilder clientbuilder.ControllerClientBuilder - if c.ComponentConfig.KubeCloudShared.UseServiceAccountCredentials { - if len(c.ComponentConfig.SAController.ServiceAccountKeyFile) == 0 { - // It's possible another controller process is creating the tokens for us. - // If one isn't, we'll timeout and exit when our client builder is unable to create the tokens. - klog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file") - } - - if shouldTurnOnDynamicClient(c.Client) { - klog.V(1).Infof("using dynamic client builder") - //Dynamic builder will use TokenRequest feature and refresh service account token periodically - clientBuilder = controller.NewDynamicClientBuilder( - restclient.AnonymousClientConfig(c.Kubeconfig), - c.Client.CoreV1(), - "kube-system") - } else { - klog.V(1).Infof("using legacy client builder") - clientBuilder = clientbuilder.SAControllerClientBuilder{ - ClientConfig: restclient.AnonymousClientConfig(c.Kubeconfig), - CoreClient: c.Client.CoreV1(), - AuthenticationClient: c.Client.AuthenticationV1(), - Namespace: "kube-system", - } - } - } else { - clientBuilder = rootClientBuilder - } - controllerContext, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, ctx.Done()) - if err != nil { - klog.Fatalf("error building controller context: %v", err) - } - saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController - - if err := StartControllers(controllerContext, saTokenControllerInitFunc, NewControllerInitializers(controllerContext.LoopMode), unsecuredMux); err != nil { - klog.Fatalf("error starting controllers: %v", err) - } - - controllerContext.InformerFactory.Start(controllerContext.Stop) - controllerContext.ObjectOrMetadataInformerFactory.Start(controllerContext.Stop) - close(controllerContext.InformersStarted) - - select {} - } - - if !c.ComponentConfig.Generic.LeaderElection.LeaderElect { - run(context.TODO()) - panic("unreachable") - } - - id, err := os.Hostname() - if err != nil { - return err - } - - // add a uniquifier so that two processes on the same host don't accidentally both become active - id = id + "_" + string(uuid.NewUUID()) - - rl, err := resourcelock.New(c.ComponentConfig.Generic.LeaderElection.ResourceLock, - c.ComponentConfig.Generic.LeaderElection.ResourceNamespace, - c.ComponentConfig.Generic.LeaderElection.ResourceName, - c.LeaderElectionClient.CoreV1(), - c.LeaderElectionClient.CoordinationV1(), - resourcelock.ResourceLockConfig{ - Identity: id, - EventRecorder: c.EventRecorder, - }) - if err != nil { - klog.Fatalf("error creating lock: %v", err) - } - - leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{ - Lock: rl, - LeaseDuration: c.ComponentConfig.Generic.LeaderElection.LeaseDuration.Duration, - RenewDeadline: c.ComponentConfig.Generic.LeaderElection.RenewDeadline.Duration, - RetryPeriod: c.ComponentConfig.Generic.LeaderElection.RetryPeriod.Duration, - Callbacks: leaderelection.LeaderCallbacks{ - OnStartedLeading: run, - OnStoppedLeading: func() { - klog.Fatalf("leaderelection lost") - }, - }, - WatchDog: electionChecker, - Name: "kube-controller-manager", - }) - panic("unreachable") -} - -// ControllerContext defines the context object for controller -type ControllerContext struct { - // ClientBuilder will provide a client for this controller to use - ClientBuilder clientbuilder.ControllerClientBuilder - - // InformerFactory gives access to informers for the controller. - InformerFactory informers.SharedInformerFactory - - // ObjectOrMetadataInformerFactory gives access to informers for typed resources - // and dynamic resources by their metadata. All generic controllers currently use - // object metadata - if a future controller needs access to the full object this - // would become GenericInformerFactory and take a dynamic client. - ObjectOrMetadataInformerFactory informerfactory.InformerFactory - - // ComponentConfig provides access to init options for a given controller - ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration - - // DeferredDiscoveryRESTMapper is a RESTMapper that will defer - // initialization of the RESTMapper until the first mapping is - // requested. - RESTMapper *restmapper.DeferredDiscoveryRESTMapper - - // AvailableResources is a map listing currently available resources - AvailableResources map[schema.GroupVersionResource]bool - - // Cloud is the cloud provider interface for the controllers to use. - // It must be initialized and ready to use. - Cloud cloudprovider.Interface - - // Control for which control loops to be run - // IncludeCloudLoops is for a kube-controller-manager running all loops - // ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager - LoopMode ControllerLoopMode - - // Stop is the stop channel - Stop <-chan struct{} - - // InformersStarted is closed after all of the controllers have been initialized and are running. After this point it is safe, - // for an individual controller to start the shared informers. Before it is closed, they should not. - InformersStarted chan struct{} - - // ResyncPeriod generates a duration each time it is invoked; this is so that - // multiple controllers don't get into lock-step and all hammer the apiserver - // with list requests simultaneously. - ResyncPeriod func() time.Duration -} - -// IsControllerEnabled checks if the context's controllers enabled or not -func (c ControllerContext) IsControllerEnabled(name string) bool { - return genericcontrollermanager.IsControllerEnabled(name, ControllersDisabledByDefault, c.ComponentConfig.Generic.Controllers) -} - -// InitFunc is used to launch a particular controller. It may run additional "should I activate checks". -// Any error returned will cause the controller process to `Fatal` -// The bool indicates whether the controller was enabled. -type InitFunc func(ctx ControllerContext) (debuggingHandler http.Handler, enabled bool, err error) - -// KnownControllers returns all known controllers's name -func KnownControllers() []string { - ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops)) - - // add "special" controllers that aren't initialized normally. These controllers cannot be initialized - // using a normal function. The only known special case is the SA token controller which *must* be started - // first to ensure that the SA tokens for future controllers will exist. Think very carefully before adding - // to this list. - ret.Insert( - saTokenControllerName, - ) - - return ret.List() -} - -// ControllersDisabledByDefault is the set of controllers which is disabled by default -var ControllersDisabledByDefault = sets.NewString( - "bootstrapsigner", - "tokencleaner", -) - -const ( - saTokenControllerName = "serviceaccount-token" -) - -// NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func) -// paired to their InitFunc. This allows for structured downstream composition and subdivision. -func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc { - controllers := map[string]InitFunc{} - controllers["endpoint"] = startEndpointController - controllers["endpointslice"] = startEndpointSliceController - controllers["endpointslicemirroring"] = startEndpointSliceMirroringController - controllers["replicationcontroller"] = startReplicationController - controllers["podgc"] = startPodGCController - controllers["resourcequota"] = startResourceQuotaController - controllers["namespace"] = startNamespaceController - controllers["serviceaccount"] = startServiceAccountController - controllers["garbagecollector"] = startGarbageCollectorController - controllers["daemonset"] = startDaemonSetController - controllers["job"] = startJobController - controllers["deployment"] = startDeploymentController - controllers["replicaset"] = startReplicaSetController - controllers["horizontalpodautoscaling"] = startHPAController - controllers["disruption"] = startDisruptionController - controllers["statefulset"] = startStatefulSetController - controllers["cronjob"] = startCronJobController - controllers["csrsigning"] = startCSRSigningController - controllers["csrapproving"] = startCSRApprovingController - controllers["csrcleaner"] = startCSRCleanerController - controllers["ttl"] = startTTLController - controllers["bootstrapsigner"] = startBootstrapSignerController - controllers["tokencleaner"] = startTokenCleanerController - controllers["nodeipam"] = startNodeIpamController - controllers["nodelifecycle"] = startNodeLifecycleController - if loopMode == IncludeCloudLoops { - controllers["service"] = startServiceController - controllers["route"] = startRouteController - controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController - // TODO: volume controller into the IncludeCloudLoops only set. - } - controllers["persistentvolume-binder"] = startPersistentVolumeBinderController - controllers["attachdetach"] = startAttachDetachController - controllers["persistentvolume-expander"] = startVolumeExpandController - controllers["clusterrole-aggregation"] = startClusterRoleAggregrationController - controllers["pvc-protection"] = startPVCProtectionController - controllers["pv-protection"] = startPVProtectionController - controllers["ttl-after-finished"] = startTTLAfterFinishedController - controllers["root-ca-cert-publisher"] = startRootCACertPublisher - controllers["ephemeral-volume"] = startEphemeralVolumeController - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) && - utilfeature.DefaultFeatureGate.Enabled(genericfeatures.StorageVersionAPI) { - controllers["storage-version-gc"] = startStorageVersionGCController - } - - return controllers -} - -// GetAvailableResources gets the map which contains all available resources of the apiserver -// TODO: In general, any controller checking this needs to be dynamic so -// users don't have to restart their controller manager if they change the apiserver. -// Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext. -func GetAvailableResources(clientBuilder clientbuilder.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) { - client := clientBuilder.ClientOrDie("controller-discovery") - discoveryClient := client.Discovery() - _, resourceMap, err := discoveryClient.ServerGroupsAndResources() - if err != nil { - utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err)) - } - if len(resourceMap) == 0 { - return nil, fmt.Errorf("unable to get any supported resources from server") - } - - allResources := map[schema.GroupVersionResource]bool{} - for _, apiResourceList := range resourceMap { - version, err := schema.ParseGroupVersion(apiResourceList.GroupVersion) - if err != nil { - return nil, err - } - for _, apiResource := range apiResourceList.APIResources { - allResources[version.WithResource(apiResource.Name)] = true - } - } - - return allResources, nil -} - -// CreateControllerContext creates a context struct containing references to resources needed by the -// controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for -// the shared-informers client and token controller. -func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clientBuilder clientbuilder.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) { - versionedClient := rootClientBuilder.ClientOrDie("shared-informers") - sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)()) - - metadataClient := metadata.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("metadata-informers")) - metadataInformers := metadatainformer.NewSharedInformerFactory(metadataClient, ResyncPeriod(s)()) - - // If apiserver is not running we should wait for some time and fail only then. This is particularly - // important when we start apiserver and controller manager at the same time. - if err := genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second); err != nil { - return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err) - } - - // Use a discovery client capable of being refreshed. - discoveryClient := rootClientBuilder.ClientOrDie("controller-discovery") - cachedClient := cacheddiscovery.NewMemCacheClient(discoveryClient.Discovery()) - restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedClient) - go wait.Until(func() { - restMapper.Reset() - }, 30*time.Second, stop) - - availableResources, err := GetAvailableResources(rootClientBuilder) - if err != nil { - return ControllerContext{}, err - } - - cloud, loopMode, err := createCloudProvider(s.ComponentConfig.KubeCloudShared.CloudProvider.Name, s.ComponentConfig.KubeCloudShared.ExternalCloudVolumePlugin, - s.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, s.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers) - if err != nil { - return ControllerContext{}, err - } - - ctx := ControllerContext{ - ClientBuilder: clientBuilder, - InformerFactory: sharedInformers, - ObjectOrMetadataInformerFactory: informerfactory.NewInformerFactory(sharedInformers, metadataInformers), - ComponentConfig: s.ComponentConfig, - RESTMapper: restMapper, - AvailableResources: availableResources, - Cloud: cloud, - LoopMode: loopMode, - Stop: stop, - InformersStarted: make(chan struct{}), - ResyncPeriod: ResyncPeriod(s), - } - return ctx, nil -} - -// StartControllers starts a set of controllers with a specified ControllerContext -func StartControllers(ctx ControllerContext, startSATokenController InitFunc, controllers map[string]InitFunc, unsecuredMux *mux.PathRecorderMux) error { - // Always start the SA token controller first using a full-power client, since it needs to mint tokens for the rest - // If this fails, just return here and fail since other controllers won't be able to get credentials. - if _, _, err := startSATokenController(ctx); err != nil { - return err - } - - // Initialize the cloud provider with a reference to the clientBuilder only after token controller - // has started in case the cloud provider uses the client builder. - if ctx.Cloud != nil { - ctx.Cloud.Initialize(ctx.ClientBuilder, ctx.Stop) - } - - for controllerName, initFn := range controllers { - if !ctx.IsControllerEnabled(controllerName) { - klog.Warningf("%q is disabled", controllerName) - continue - } - - time.Sleep(wait.Jitter(ctx.ComponentConfig.Generic.ControllerStartInterval.Duration, ControllerStartJitter)) - - klog.V(1).Infof("Starting %q", controllerName) - debugHandler, started, err := initFn(ctx) - if err != nil { - klog.Errorf("Error starting %q", controllerName) - return err - } - if !started { - klog.Warningf("Skipping %q", controllerName) - continue - } - if debugHandler != nil && unsecuredMux != nil { - basePath := "/debug/controllers/" + controllerName - unsecuredMux.UnlistedHandle(basePath, http.StripPrefix(basePath, debugHandler)) - unsecuredMux.UnlistedHandlePrefix(basePath+"/", http.StripPrefix(basePath, debugHandler)) - } - klog.Infof("Started %q", controllerName) - } - - return nil -} - -// serviceAccountTokenControllerStarter is special because it must run first to set up permissions for other controllers. -// It cannot use the "normal" client builder, so it tracks its own. It must also avoid being included in the "normal" -// init map so that it can always run first. -type serviceAccountTokenControllerStarter struct { - rootClientBuilder clientbuilder.ControllerClientBuilder -} - -func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.IsControllerEnabled(saTokenControllerName) { - klog.Warningf("%q is disabled", saTokenControllerName) - return nil, false, nil - } - - if len(ctx.ComponentConfig.SAController.ServiceAccountKeyFile) == 0 { - klog.Warningf("%q is disabled because there is no private key", saTokenControllerName) - return nil, false, nil - } - privateKey, err := keyutil.PrivateKeyFromFile(ctx.ComponentConfig.SAController.ServiceAccountKeyFile) - if err != nil { - return nil, true, fmt.Errorf("error reading key for service account token controller: %v", err) - } - - var rootCA []byte - if ctx.ComponentConfig.SAController.RootCAFile != "" { - if rootCA, err = readCA(ctx.ComponentConfig.SAController.RootCAFile); err != nil { - return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.SAController.RootCAFile, err) - } - } else { - rootCA = c.rootClientBuilder.ConfigOrDie("tokens-controller").CAData - } - - tokenGenerator, err := serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, privateKey) - if err != nil { - return nil, false, fmt.Errorf("failed to build token generator: %v", err) - } - controller, err := serviceaccountcontroller.NewTokensController( - ctx.InformerFactory.Core().V1().ServiceAccounts(), - ctx.InformerFactory.Core().V1().Secrets(), - c.rootClientBuilder.ClientOrDie("tokens-controller"), - serviceaccountcontroller.TokensControllerOptions{ - TokenGenerator: tokenGenerator, - RootCA: rootCA, - }, - ) - if err != nil { - return nil, true, fmt.Errorf("error creating Tokens controller: %v", err) - } - go controller.Run(int(ctx.ComponentConfig.SAController.ConcurrentSATokenSyncs), ctx.Stop) - - // start the first set of informers now so that other controllers can start - ctx.InformerFactory.Start(ctx.Stop) - - return nil, true, nil -} - -func readCA(file string) ([]byte, error) { - rootCA, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - if _, err := certutil.ParseCertsPEM(rootCA); err != nil { - return nil, err - } - - return rootCA, err -} - -func shouldTurnOnDynamicClient(client clientset.Interface) bool { - apiResourceList, err := client.Discovery().ServerResourcesForGroupVersion(v1.SchemeGroupVersion.String()) - if err != nil { - klog.Warningf("fetch api resource lists failed, use legacy client builder: %v", err) - return false - } - - for _, resource := range apiResourceList.APIResources { - if resource.Name == "serviceaccounts/token" && - resource.Group == "authentication.k8s.io" && - sets.NewString(resource.Verbs...).Has("create") { - return true - } - } - - return false -} diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go deleted file mode 100644 index 15dbc0586c8ca..0000000000000 --- a/cmd/kube-controller-manager/app/core.go +++ /dev/null @@ -1,687 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "errors" - "fmt" - "net" - "net/http" - "strings" - "time" - - "k8s.io/klog/v2" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/quota/v1/generic" - utilfeature "k8s.io/apiserver/pkg/util/feature" - storagev1informer "k8s.io/client-go/informers/storage/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/metadata" - restclient "k8s.io/client-go/rest" - cloudnodelifecyclecontroller "k8s.io/cloud-provider/controllers/nodelifecycle" - routecontroller "k8s.io/cloud-provider/controllers/route" - servicecontroller "k8s.io/cloud-provider/controllers/service" - "k8s.io/component-base/metrics/prometheus/ratelimiter" - csitrans "k8s.io/csi-translation-lib" - "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" - "k8s.io/kubernetes/pkg/controller" - endpointcontroller "k8s.io/kubernetes/pkg/controller/endpoint" - "k8s.io/kubernetes/pkg/controller/garbagecollector" - namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace" - nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam" - nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" - "k8s.io/kubernetes/pkg/controller/nodeipam/ipam" - lifecyclecontroller "k8s.io/kubernetes/pkg/controller/nodelifecycle" - "k8s.io/kubernetes/pkg/controller/podgc" - replicationcontroller "k8s.io/kubernetes/pkg/controller/replication" - resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota" - serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount" - "k8s.io/kubernetes/pkg/controller/storageversiongc" - ttlcontroller "k8s.io/kubernetes/pkg/controller/ttl" - "k8s.io/kubernetes/pkg/controller/ttlafterfinished" - "k8s.io/kubernetes/pkg/controller/volume/attachdetach" - "k8s.io/kubernetes/pkg/controller/volume/ephemeral" - "k8s.io/kubernetes/pkg/controller/volume/expand" - persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume" - "k8s.io/kubernetes/pkg/controller/volume/pvcprotection" - "k8s.io/kubernetes/pkg/controller/volume/pvprotection" - "k8s.io/kubernetes/pkg/features" - quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" - "k8s.io/kubernetes/pkg/volume/csimigration" - netutils "k8s.io/utils/net" -) - -const ( - // defaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr - defaultNodeMaskCIDRIPv4 = 24 - // defaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr - defaultNodeMaskCIDRIPv6 = 64 -) - -func startServiceController(ctx ControllerContext) (http.Handler, bool, error) { - serviceController, err := servicecontroller.New( - ctx.Cloud, - ctx.ClientBuilder.ClientOrDie("service-controller"), - ctx.InformerFactory.Core().V1().Services(), - ctx.InformerFactory.Core().V1().Nodes(), - ctx.ComponentConfig.KubeCloudShared.ClusterName, - utilfeature.DefaultFeatureGate, - ) - if err != nil { - // This error shouldn't fail. It lives like this as a legacy. - klog.Errorf("Failed to start service controller: %v", err) - return nil, false, nil - } - go serviceController.Run(ctx.Stop, int(ctx.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) - return nil, true, nil -} - -func startNodeIpamController(ctx ControllerContext) (http.Handler, bool, error) { - var serviceCIDR *net.IPNet - var secondaryServiceCIDR *net.IPNet - - // should we start nodeIPAM - if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs { - return nil, false, nil - } - - // failure: bad cidrs in config - clusterCIDRs, dualStack, err := processCIDRs(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR) - if err != nil { - return nil, false, err - } - - // failure: more than one cidr and dual stack is not enabled and/or endpoint slice is not enabled - if len(clusterCIDRs) > 1 && (!utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) || !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice)) { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack or EndpointSlice feature is not enabled", len(clusterCIDRs)) - } - - // failure: more than one cidr but they are not configured as dual stack - if len(clusterCIDRs) > 1 && !dualStack { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily)", len(clusterCIDRs)) - } - - // failure: more than cidrs is not allowed even with dual stack - if len(clusterCIDRs) > 2 { - return nil, false, fmt.Errorf("len of clusters is:%v > more than max allowed of 2", len(clusterCIDRs)) - } - - // service cidr processing - if len(strings.TrimSpace(ctx.ComponentConfig.NodeIPAMController.ServiceCIDR)) != 0 { - _, serviceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.NodeIPAMController.ServiceCIDR) - if err != nil { - klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.NodeIPAMController.ServiceCIDR, err) - } - } - - if len(strings.TrimSpace(ctx.ComponentConfig.NodeIPAMController.SecondaryServiceCIDR)) != 0 { - _, secondaryServiceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.NodeIPAMController.SecondaryServiceCIDR) - if err != nil { - klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.NodeIPAMController.SecondaryServiceCIDR, err) - } - } - - // the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided - if serviceCIDR != nil && secondaryServiceCIDR != nil { - // should have dual stack flag enabled - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - return nil, false, fmt.Errorf("secondary service cidr is provided and IPv6DualStack feature is not enabled") - } - - // should be dual stack (from different IPFamilies) - dualstackServiceCIDR, err := netutils.IsDualStackCIDRs([]*net.IPNet{serviceCIDR, secondaryServiceCIDR}) - if err != nil { - return nil, false, fmt.Errorf("failed to perform dualstack check on serviceCIDR and secondaryServiceCIDR error:%v", err) - } - if !dualstackServiceCIDR { - return nil, false, fmt.Errorf("serviceCIDR and secondaryServiceCIDR are not dualstack (from different IPfamiles)") - } - } - - var nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6 int - if dualStack { - // only --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 supported with dual stack clusters. - // --node-cidr-mask-size flag is incompatible with dual stack clusters. - nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(ctx.ComponentConfig.NodeIPAMController) - } else { - // only --node-cidr-mask-size supported with single stack clusters. - // --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 flags are incompatible with single stack clusters. - nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(ctx.ComponentConfig.NodeIPAMController) - } - - if err != nil { - return nil, false, err - } - - // get list of node cidr mask sizes - nodeCIDRMaskSizes := getNodeCIDRMaskSizes(clusterCIDRs, nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6) - - nodeIpamController, err := nodeipamcontroller.NewNodeIpamController( - ctx.InformerFactory.Core().V1().Nodes(), - ctx.Cloud, - ctx.ClientBuilder.ClientOrDie("node-controller"), - clusterCIDRs, - serviceCIDR, - secondaryServiceCIDR, - nodeCIDRMaskSizes, - ipam.CIDRAllocatorType(ctx.ComponentConfig.KubeCloudShared.CIDRAllocatorType), - ) - if err != nil { - return nil, true, err - } - go nodeIpamController.Run(ctx.Stop) - return nil, true, nil -} - -func startNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, error) { - lifecycleController, err := lifecyclecontroller.NewNodeLifecycleController( - ctx.InformerFactory.Coordination().V1().Leases(), - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().Nodes(), - ctx.InformerFactory.Apps().V1().DaemonSets(), - // node lifecycle controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), - ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, - ctx.ComponentConfig.NodeLifecycleController.NodeStartupGracePeriod.Duration, - ctx.ComponentConfig.NodeLifecycleController.NodeMonitorGracePeriod.Duration, - ctx.ComponentConfig.NodeLifecycleController.PodEvictionTimeout.Duration, - ctx.ComponentConfig.NodeLifecycleController.NodeEvictionRate, - ctx.ComponentConfig.NodeLifecycleController.SecondaryNodeEvictionRate, - ctx.ComponentConfig.NodeLifecycleController.LargeClusterSizeThreshold, - ctx.ComponentConfig.NodeLifecycleController.UnhealthyZoneThreshold, - ctx.ComponentConfig.NodeLifecycleController.EnableTaintManager, - ) - if err != nil { - return nil, true, err - } - go lifecycleController.Run(ctx.Stop) - return nil, true, nil -} - -func startCloudNodeLifecycleController(ctx ControllerContext) (http.Handler, bool, error) { - cloudNodeLifecycleController, err := cloudnodelifecyclecontroller.NewCloudNodeLifecycleController( - ctx.InformerFactory.Core().V1().Nodes(), - // cloud node lifecycle controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), - ctx.Cloud, - ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, - ) - if err != nil { - // the controller manager should continue to run if the "Instances" interface is not - // supported, though it's unlikely for a cloud provider to not support it - klog.Errorf("failed to start cloud node lifecycle controller: %v", err) - return nil, false, nil - } - - go cloudNodeLifecycleController.Run(ctx.Stop) - return nil, true, nil -} - -func startRouteController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { - klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) - return nil, false, nil - } - if ctx.Cloud == nil { - klog.Warning("configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.") - return nil, false, nil - } - routes, ok := ctx.Cloud.Routes() - if !ok { - klog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") - return nil, false, nil - } - - // failure: bad cidrs in config - clusterCIDRs, dualStack, err := processCIDRs(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR) - if err != nil { - return nil, false, err - } - - // failure: more than one cidr and dual stack is not enabled - if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs)) - } - - // failure: more than one cidr but they are not configured as dual stack - if len(clusterCIDRs) > 1 && !dualStack { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs)) - } - - // failure: more than cidrs is not allowed even with dual stack - if len(clusterCIDRs) > 2 { - return nil, false, fmt.Errorf("length of clusterCIDRs is:%v more than max allowed of 2", len(clusterCIDRs)) - } - - routeController := routecontroller.New(routes, - ctx.ClientBuilder.ClientOrDie("route-controller"), - ctx.InformerFactory.Core().V1().Nodes(), - ctx.ComponentConfig.KubeCloudShared.ClusterName, - clusterCIDRs) - go routeController.Run(ctx.Stop, ctx.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) - return nil, true, nil -} - -func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler, bool, error) { - plugins, err := ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration) - if err != nil { - return nil, true, fmt.Errorf("failed to probe volume plugins when starting persistentvolume controller: %v", err) - } - filteredDialOptions, err := options.ParseVolumeHostFilters( - ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostCIDRDenylist, - ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostAllowLocalLoopback) - if err != nil { - return nil, true, err - } - params := persistentvolumecontroller.ControllerParameters{ - KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"), - SyncPeriod: ctx.ComponentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod.Duration, - VolumePlugins: plugins, - Cloud: ctx.Cloud, - ClusterName: ctx.ComponentConfig.KubeCloudShared.ClusterName, - VolumeInformer: ctx.InformerFactory.Core().V1().PersistentVolumes(), - ClaimInformer: ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), - ClassInformer: ctx.InformerFactory.Storage().V1().StorageClasses(), - PodInformer: ctx.InformerFactory.Core().V1().Pods(), - NodeInformer: ctx.InformerFactory.Core().V1().Nodes(), - EnableDynamicProvisioning: ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration.EnableDynamicProvisioning, - FilteredDialOptions: filteredDialOptions, - } - volumeController, volumeControllerErr := persistentvolumecontroller.NewController(params) - if volumeControllerErr != nil { - return nil, true, fmt.Errorf("failed to construct persistentvolume controller: %v", volumeControllerErr) - } - go volumeController.Run(ctx.Stop) - return nil, true, nil -} - -func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, error) { - if ctx.ComponentConfig.AttachDetachController.ReconcilerSyncLoopPeriod.Duration < time.Second { - return nil, true, fmt.Errorf("duration time must be greater than one second as set via command line option reconcile-sync-loop-period") - } - - var ( - csiNodeInformer storagev1informer.CSINodeInformer - ) - if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) { - csiNodeInformer = ctx.InformerFactory.Storage().V1().CSINodes() - } - csiDriverInformer := ctx.InformerFactory.Storage().V1().CSIDrivers() - - plugins, err := ProbeAttachableVolumePlugins() - if err != nil { - return nil, true, fmt.Errorf("failed to probe volume plugins when starting attach/detach controller: %v", err) - } - - filteredDialOptions, err := options.ParseVolumeHostFilters( - ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostCIDRDenylist, - ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostAllowLocalLoopback) - if err != nil { - return nil, true, err - } - - attachDetachController, attachDetachControllerErr := - attachdetach.NewAttachDetachController( - ctx.ClientBuilder.ClientOrDie("attachdetach-controller"), - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().Nodes(), - ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), - ctx.InformerFactory.Core().V1().PersistentVolumes(), - csiNodeInformer, - csiDriverInformer, - ctx.InformerFactory.Storage().V1().VolumeAttachments(), - ctx.Cloud, - plugins, - GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration), - ctx.ComponentConfig.AttachDetachController.DisableAttachDetachReconcilerSync, - ctx.ComponentConfig.AttachDetachController.ReconcilerSyncLoopPeriod.Duration, - attachdetach.DefaultTimerConfig, - filteredDialOptions, - ) - if attachDetachControllerErr != nil { - return nil, true, fmt.Errorf("failed to start attach/detach controller: %v", attachDetachControllerErr) - } - go attachDetachController.Run(ctx.Stop) - return nil, true, nil -} - -func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, error) { - if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { - plugins, err := ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration) - if err != nil { - return nil, true, fmt.Errorf("failed to probe volume plugins when starting volume expand controller: %v", err) - } - csiTranslator := csitrans.New() - filteredDialOptions, err := options.ParseVolumeHostFilters( - ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostCIDRDenylist, - ctx.ComponentConfig.PersistentVolumeBinderController.VolumeHostAllowLocalLoopback) - if err != nil { - return nil, true, err - } - expandController, expandControllerErr := expand.NewExpandController( - ctx.ClientBuilder.ClientOrDie("expand-controller"), - ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), - ctx.InformerFactory.Core().V1().PersistentVolumes(), - ctx.Cloud, - plugins, - csiTranslator, - csimigration.NewPluginManager(csiTranslator), - filteredDialOptions, - ) - - if expandControllerErr != nil { - return nil, true, fmt.Errorf("failed to start volume expand controller: %v", expandControllerErr) - } - go expandController.Run(ctx.Stop) - return nil, true, nil - } - return nil, false, nil -} - -func startEphemeralVolumeController(ctx ControllerContext) (http.Handler, bool, error) { - if utilfeature.DefaultFeatureGate.Enabled(features.GenericEphemeralVolume) { - ephemeralController, err := ephemeral.NewController( - ctx.ClientBuilder.ClientOrDie("ephemeral-volume-controller"), - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().PersistentVolumeClaims()) - if err != nil { - return nil, true, fmt.Errorf("failed to start ephemeral volume controller: %v", err) - } - // TODO (before beta at the latest): make this configurable similar to the EndpointController - go ephemeralController.Run(1 /* int(ctx.ComponentConfig.EphemeralController.ConcurrentEphemeralVolumeSyncs) */, ctx.Stop) - return nil, true, nil - } - return nil, false, nil -} - -func startEndpointController(ctx ControllerContext) (http.Handler, bool, error) { - go endpointcontroller.NewEndpointController( - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().Services(), - ctx.InformerFactory.Core().V1().Endpoints(), - ctx.ClientBuilder.ClientOrDie("endpoint-controller"), - ctx.ComponentConfig.EndpointController.EndpointUpdatesBatchPeriod.Duration, - ).Run(int(ctx.ComponentConfig.EndpointController.ConcurrentEndpointSyncs), ctx.Stop) - return nil, true, nil -} - -func startReplicationController(ctx ControllerContext) (http.Handler, bool, error) { - go replicationcontroller.NewReplicationManager( - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().ReplicationControllers(), - ctx.ClientBuilder.ClientOrDie("replication-controller"), - replicationcontroller.BurstReplicas, - ).Run(int(ctx.ComponentConfig.ReplicationController.ConcurrentRCSyncs), ctx.Stop) - return nil, true, nil -} - -func startPodGCController(ctx ControllerContext) (http.Handler, bool, error) { - go podgc.NewPodGC( - ctx.ClientBuilder.ClientOrDie("pod-garbage-collector"), - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().Nodes(), - int(ctx.ComponentConfig.PodGCController.TerminatedPodGCThreshold), - ).Run(ctx.Stop) - return nil, true, nil -} - -func startResourceQuotaController(ctx ControllerContext) (http.Handler, bool, error) { - resourceQuotaControllerClient := ctx.ClientBuilder.ClientOrDie("resourcequota-controller") - discoveryFunc := resourceQuotaControllerClient.Discovery().ServerPreferredNamespacedResources - listerFuncForResource := generic.ListerFuncForResourceFunc(ctx.InformerFactory.ForResource) - quotaConfiguration := quotainstall.NewQuotaConfigurationForControllers(listerFuncForResource) - - resourceQuotaControllerOptions := &resourcequotacontroller.ControllerOptions{ - QuotaClient: resourceQuotaControllerClient.CoreV1(), - ResourceQuotaInformer: ctx.InformerFactory.Core().V1().ResourceQuotas(), - ResyncPeriod: controller.StaticResyncPeriodFunc(ctx.ComponentConfig.ResourceQuotaController.ResourceQuotaSyncPeriod.Duration), - InformerFactory: ctx.ObjectOrMetadataInformerFactory, - ReplenishmentResyncPeriod: ctx.ResyncPeriod, - DiscoveryFunc: discoveryFunc, - IgnoredResourcesFunc: quotaConfiguration.IgnoredResources, - InformersStarted: ctx.InformersStarted, - Registry: generic.NewRegistry(quotaConfiguration.Evaluators()), - } - if resourceQuotaControllerClient.CoreV1().RESTClient().GetRateLimiter() != nil { - if err := ratelimiter.RegisterMetricAndTrackRateLimiterUsage("resource_quota_controller", resourceQuotaControllerClient.CoreV1().RESTClient().GetRateLimiter()); err != nil { - return nil, true, err - } - } - - resourceQuotaController, err := resourcequotacontroller.NewController(resourceQuotaControllerOptions) - if err != nil { - return nil, false, err - } - go resourceQuotaController.Run(int(ctx.ComponentConfig.ResourceQuotaController.ConcurrentResourceQuotaSyncs), ctx.Stop) - - // Periodically the quota controller to detect new resource types - go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, ctx.Stop) - - return nil, true, nil -} - -func startNamespaceController(ctx ControllerContext) (http.Handler, bool, error) { - // the namespace cleanup controller is very chatty. It makes lots of discovery calls and then it makes lots of delete calls - // the ratelimiter negatively affects its speed. Deleting 100 total items in a namespace (that's only a few of each resource - // including events), takes ~10 seconds by default. - nsKubeconfig := ctx.ClientBuilder.ConfigOrDie("namespace-controller") - nsKubeconfig.QPS *= 20 - nsKubeconfig.Burst *= 100 - namespaceKubeClient := clientset.NewForConfigOrDie(nsKubeconfig) - return startModifiedNamespaceController(ctx, namespaceKubeClient, nsKubeconfig) -} - -func startModifiedNamespaceController(ctx ControllerContext, namespaceKubeClient clientset.Interface, nsKubeconfig *restclient.Config) (http.Handler, bool, error) { - - metadataClient, err := metadata.NewForConfig(nsKubeconfig) - if err != nil { - return nil, true, err - } - - discoverResourcesFn := namespaceKubeClient.Discovery().ServerPreferredNamespacedResources - - namespaceController := namespacecontroller.NewNamespaceController( - namespaceKubeClient, - metadataClient, - discoverResourcesFn, - ctx.InformerFactory.Core().V1().Namespaces(), - ctx.ComponentConfig.NamespaceController.NamespaceSyncPeriod.Duration, - v1.FinalizerKubernetes, - ) - go namespaceController.Run(int(ctx.ComponentConfig.NamespaceController.ConcurrentNamespaceSyncs), ctx.Stop) - - return nil, true, nil -} - -func startServiceAccountController(ctx ControllerContext) (http.Handler, bool, error) { - sac, err := serviceaccountcontroller.NewServiceAccountsController( - ctx.InformerFactory.Core().V1().ServiceAccounts(), - ctx.InformerFactory.Core().V1().Namespaces(), - ctx.ClientBuilder.ClientOrDie("service-account-controller"), - serviceaccountcontroller.DefaultServiceAccountsControllerOptions(), - ) - if err != nil { - return nil, true, fmt.Errorf("error creating ServiceAccount controller: %v", err) - } - go sac.Run(1, ctx.Stop) - return nil, true, nil -} - -func startTTLController(ctx ControllerContext) (http.Handler, bool, error) { - go ttlcontroller.NewTTLController( - ctx.InformerFactory.Core().V1().Nodes(), - ctx.ClientBuilder.ClientOrDie("ttl-controller"), - ).Run(5, ctx.Stop) - return nil, true, nil -} - -func startGarbageCollectorController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.ComponentConfig.GarbageCollectorController.EnableGarbageCollector { - return nil, false, nil - } - - gcClientset := ctx.ClientBuilder.ClientOrDie("generic-garbage-collector") - - config := ctx.ClientBuilder.ConfigOrDie("generic-garbage-collector") - metadataClient, err := metadata.NewForConfig(config) - if err != nil { - return nil, true, err - } - - ignoredResources := make(map[schema.GroupResource]struct{}) - for _, r := range ctx.ComponentConfig.GarbageCollectorController.GCIgnoredResources { - ignoredResources[schema.GroupResource{Group: r.Group, Resource: r.Resource}] = struct{}{} - } - garbageCollector, err := garbagecollector.NewGarbageCollector( - gcClientset, - metadataClient, - ctx.RESTMapper, - ignoredResources, - ctx.ObjectOrMetadataInformerFactory, - ctx.InformersStarted, - ) - if err != nil { - return nil, true, fmt.Errorf("failed to start the generic garbage collector: %v", err) - } - - // Start the garbage collector. - workers := int(ctx.ComponentConfig.GarbageCollectorController.ConcurrentGCSyncs) - go garbageCollector.Run(workers, ctx.Stop) - - // Periodically refresh the RESTMapper with new discovery information and sync - // the garbage collector. - go garbageCollector.Sync(gcClientset.Discovery(), 30*time.Second, ctx.Stop) - - return garbagecollector.NewDebugHandler(garbageCollector), true, nil -} - -func startPVCProtectionController(ctx ControllerContext) (http.Handler, bool, error) { - pvcProtectionController, err := pvcprotection.NewPVCProtectionController( - ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), - ctx.InformerFactory.Core().V1().Pods(), - ctx.ClientBuilder.ClientOrDie("pvc-protection-controller"), - utilfeature.DefaultFeatureGate.Enabled(features.StorageObjectInUseProtection), - utilfeature.DefaultFeatureGate.Enabled(features.StorageObjectInUseProtection), - ) - if err != nil { - return nil, true, fmt.Errorf("failed to start the pvc protection controller: %v", err) - } - go pvcProtectionController.Run(1, ctx.Stop) - return nil, true, nil -} - -func startPVProtectionController(ctx ControllerContext) (http.Handler, bool, error) { - go pvprotection.NewPVProtectionController( - ctx.InformerFactory.Core().V1().PersistentVolumes(), - ctx.ClientBuilder.ClientOrDie("pv-protection-controller"), - utilfeature.DefaultFeatureGate.Enabled(features.StorageObjectInUseProtection), - ).Run(1, ctx.Stop) - return nil, true, nil -} - -func startTTLAfterFinishedController(ctx ControllerContext) (http.Handler, bool, error) { - if !utilfeature.DefaultFeatureGate.Enabled(features.TTLAfterFinished) { - return nil, false, nil - } - go ttlafterfinished.New( - ctx.InformerFactory.Batch().V1().Jobs(), - ctx.ClientBuilder.ClientOrDie("ttl-after-finished-controller"), - ).Run(int(ctx.ComponentConfig.TTLAfterFinishedController.ConcurrentTTLSyncs), ctx.Stop) - return nil, true, nil -} - -// processCIDRs is a helper function that works on a comma separated cidrs and returns -// a list of typed cidrs -// a flag if cidrs represents a dual stack -// error if failed to parse any of the cidrs -func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) { - cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",") - - cidrs, err := netutils.ParseCIDRs(cidrsSplit) - if err != nil { - return nil, false, err - } - - // if cidrs has an error then the previous call will fail - // safe to ignore error checking on next call - dualstack, _ := netutils.IsDualStackCIDRs(cidrs) - - return cidrs, dualstack, nil -} - -// setNodeCIDRMaskSizes returns the IPv4 and IPv6 node cidr mask sizes. -// If --node-cidr-mask-size not set, then it will return default IPv4 and IPv6 cidr mask sizes. -func setNodeCIDRMaskSizes(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) { - ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6 - // NodeCIDRMaskSizeIPv4 and NodeCIDRMaskSizeIPv6 can be used only for dual-stack clusters - if cfg.NodeCIDRMaskSizeIPv4 != 0 || cfg.NodeCIDRMaskSizeIPv6 != 0 { - return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 are not allowed with non dual-stack clusters") - } - if cfg.NodeCIDRMaskSize != 0 { - ipv4Mask = int(cfg.NodeCIDRMaskSize) - ipv6Mask = int(cfg.NodeCIDRMaskSize) - } - return ipv4Mask, ipv6Mask, nil -} - -// setNodeCIDRMaskSizesDualStack returns the IPv4 and IPv6 node cidr mask sizes to the value provided -// for --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 respectively. If value not provided, -// then it will return default IPv4 and IPv6 cidr mask sizes. -func setNodeCIDRMaskSizesDualStack(cfg nodeipamconfig.NodeIPAMControllerConfiguration) (int, int, error) { - ipv4Mask, ipv6Mask := defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6 - // NodeCIDRMaskSize can be used only for single stack clusters - if cfg.NodeCIDRMaskSize != 0 { - return ipv4Mask, ipv6Mask, errors.New("usage of --node-cidr-mask-size is not allowed with dual-stack clusters") - } - if cfg.NodeCIDRMaskSizeIPv4 != 0 { - ipv4Mask = int(cfg.NodeCIDRMaskSizeIPv4) - } - if cfg.NodeCIDRMaskSizeIPv6 != 0 { - ipv6Mask = int(cfg.NodeCIDRMaskSizeIPv6) - } - return ipv4Mask, ipv6Mask, nil -} - -// getNodeCIDRMaskSizes is a helper function that helps the generate the node cidr mask -// sizes slice based on the cluster cidr slice -func getNodeCIDRMaskSizes(clusterCIDRs []*net.IPNet, maskSizeIPv4, maskSizeIPv6 int) []int { - nodeMaskCIDRs := make([]int, len(clusterCIDRs)) - - for idx, clusterCIDR := range clusterCIDRs { - if netutils.IsIPv6CIDR(clusterCIDR) { - nodeMaskCIDRs[idx] = maskSizeIPv6 - } else { - nodeMaskCIDRs[idx] = maskSizeIPv4 - } - } - return nodeMaskCIDRs -} - -func startStorageVersionGCController(ctx ControllerContext) (http.Handler, bool, error) { - go storageversiongc.NewStorageVersionGC( - ctx.ClientBuilder.ClientOrDie("storage-version-garbage-collector"), - ctx.InformerFactory.Coordination().V1().Leases(), - ctx.InformerFactory.Internal().V1alpha1().StorageVersions(), - ).Run(ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/core_test.go b/cmd/kube-controller-manager/app/core_test.go deleted file mode 100644 index 9fdc70dcb5a9b..0000000000000 --- a/cmd/kube-controller-manager/app/core_test.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "net/http" - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/discovery" - fakediscovery "k8s.io/client-go/discovery/fake" - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" - fakeclientset "k8s.io/client-go/kubernetes/fake" - restclient "k8s.io/client-go/rest" -) - -// TestClientBuilder inherits ClientBuilder and can accept a given fake clientset. -type TestClientBuilder struct { - clientset clientset.Interface -} - -func (TestClientBuilder) Config(name string) (*restclient.Config, error) { return nil, nil } -func (TestClientBuilder) ConfigOrDie(name string) *restclient.Config { - return &restclient.Config{} -} - -func (TestClientBuilder) Client(name string) (clientset.Interface, error) { return nil, nil } -func (m TestClientBuilder) ClientOrDie(name string) clientset.Interface { - return m.clientset -} - -// FakeDiscoveryWithError inherits DiscoveryInterface(via FakeDiscovery) with some methods accepting testing data. -type FakeDiscoveryWithError struct { - fakediscovery.FakeDiscovery - PossibleResources []*metav1.APIResourceList - Err error -} - -func (d FakeDiscoveryWithError) ServerPreferredNamespacedResources() ([]*metav1.APIResourceList, error) { - return d.PossibleResources, d.Err -} - -// FakeDiscoveryWithError inherits Clientset(via FakeClientset) with overridden Discovery method. -type FakeClientSet struct { - fakeclientset.Clientset - DiscoveryObj *FakeDiscoveryWithError -} - -func (c *FakeClientSet) Discovery() discovery.DiscoveryInterface { - return c.DiscoveryObj -} - -func (c *FakeClientSet) GetPossibleResources() []*metav1.APIResourceList { - return c.DiscoveryObj.PossibleResources -} - -// Create a fake Clientset with its Discovery method overridden. -func NewFakeClientset(fakeDiscovery FakeDiscoveryWithError) *FakeClientSet { - cs := &FakeClientSet{} - cs.DiscoveryObj = &fakeDiscovery - return cs -} - -func possibleDiscoveryResource() []*metav1.APIResourceList { - return []*metav1.APIResourceList{ - { - GroupVersion: "create/v1", - APIResources: []metav1.APIResource{ - { - Name: "jobs", - Verbs: []string{"create", "list", "watch", "delete"}, - ShortNames: []string{"jz"}, - Categories: []string{"all"}, - }, - }, - }, - } -} - -type controllerInitFunc func(ControllerContext) (http.Handler, bool, error) - -func TestController_DiscoveryError(t *testing.T) { - controllerInitFuncMap := map[string]controllerInitFunc{ - "ResourceQuotaController": startResourceQuotaController, - "GarbageCollectorController": startGarbageCollectorController, - "EndpointSliceController": startEndpointSliceController, - "EndpointSliceMirroringController": startEndpointSliceMirroringController, - } - - tcs := map[string]struct { - discoveryError error - expectedErr bool - possibleResources []*metav1.APIResourceList - }{ - "No Discovery Error": { - discoveryError: nil, - possibleResources: possibleDiscoveryResource(), - expectedErr: false, - }, - "Discovery Calls Partially Failed": { - discoveryError: new(discovery.ErrGroupDiscoveryFailed), - possibleResources: possibleDiscoveryResource(), - expectedErr: false, - }, - } - for name, test := range tcs { - testDiscovery := FakeDiscoveryWithError{Err: test.discoveryError, PossibleResources: test.possibleResources} - testClientset := NewFakeClientset(testDiscovery) - testClientBuilder := TestClientBuilder{clientset: testClientset} - testInformerFactory := informers.NewSharedInformerFactoryWithOptions(testClientset, time.Duration(1)) - ctx := ControllerContext{ - ClientBuilder: testClientBuilder, - InformerFactory: testInformerFactory, - ObjectOrMetadataInformerFactory: testInformerFactory, - InformersStarted: make(chan struct{}), - } - for funcName, controllerInit := range controllerInitFuncMap { - _, _, err := controllerInit(ctx) - if test.expectedErr != (err != nil) { - t.Errorf("%v test failed for use case: %v", funcName, name) - } - } - _, _, err := startModifiedNamespaceController( - ctx, testClientset, testClientBuilder.ConfigOrDie("namespace-controller")) - if test.expectedErr != (err != nil) { - t.Errorf("Namespace Controller test failed for use case: %v", name) - } - } -} diff --git a/cmd/kube-controller-manager/app/discovery.go b/cmd/kube-controller-manager/app/discovery.go deleted file mode 100644 index e0d2421e047a1..0000000000000 --- a/cmd/kube-controller-manager/app/discovery.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "net/http" - - discoveryv1beta1 "k8s.io/api/discovery/v1beta1" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/klog/v2" - endpointslicecontroller "k8s.io/kubernetes/pkg/controller/endpointslice" - endpointslicemirroringcontroller "k8s.io/kubernetes/pkg/controller/endpointslicemirroring" - "k8s.io/kubernetes/pkg/features" -) - -func startEndpointSliceController(ctx ControllerContext) (http.Handler, bool, error) { - if !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) { - klog.V(2).Infof("Not starting endpointslice-controller since EndpointSlice feature gate is disabled") - return nil, false, nil - } - - if !ctx.AvailableResources[discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices")] { - klog.Warningf("Not starting endpointslice-controller since discovery.k8s.io/v1beta1 resources are not available") - return nil, false, nil - } - - go endpointslicecontroller.NewController( - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Core().V1().Services(), - ctx.InformerFactory.Core().V1().Nodes(), - ctx.InformerFactory.Discovery().V1beta1().EndpointSlices(), - ctx.ComponentConfig.EndpointSliceController.MaxEndpointsPerSlice, - ctx.ClientBuilder.ClientOrDie("endpointslice-controller"), - ctx.ComponentConfig.EndpointSliceController.EndpointUpdatesBatchPeriod.Duration, - ).Run(int(ctx.ComponentConfig.EndpointSliceController.ConcurrentServiceEndpointSyncs), ctx.Stop) - return nil, true, nil -} - -func startEndpointSliceMirroringController(ctx ControllerContext) (http.Handler, bool, error) { - if !utilfeature.DefaultFeatureGate.Enabled(features.EndpointSlice) { - klog.V(2).Infof("Not starting endpointslicemirroring-controller since EndpointSlice feature gate is disabled") - return nil, false, nil - } - - if !ctx.AvailableResources[discoveryv1beta1.SchemeGroupVersion.WithResource("endpointslices")] { - klog.Warningf("Not starting endpointslicemirroring-controller since discovery.k8s.io/v1beta1 resources are not available") - return nil, false, nil - } - - go endpointslicemirroringcontroller.NewController( - ctx.InformerFactory.Core().V1().Endpoints(), - ctx.InformerFactory.Discovery().V1beta1().EndpointSlices(), - ctx.InformerFactory.Core().V1().Services(), - ctx.ComponentConfig.EndpointSliceMirroringController.MirroringMaxEndpointsPerSubset, - ctx.ClientBuilder.ClientOrDie("endpointslicemirroring-controller"), - ctx.ComponentConfig.EndpointSliceMirroringController.MirroringEndpointUpdatesBatchPeriod.Duration, - ).Run(int(ctx.ComponentConfig.EndpointSliceMirroringController.MirroringConcurrentServiceEndpointSyncs), ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/flags_providerless.go b/cmd/kube-controller-manager/app/flags_providerless.go deleted file mode 100644 index f08fc68e26b49..0000000000000 --- a/cmd/kube-controller-manager/app/flags_providerless.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - cliflag "k8s.io/component-base/cli/flag" -) - -func registerLegacyGlobalFlags(namedFlagSets cliflag.NamedFlagSets) { - // no-op when legacy cloud providers are not compiled -} diff --git a/cmd/kube-controller-manager/app/flags_providers.go b/cmd/kube-controller-manager/app/flags_providers.go deleted file mode 100644 index cb8a5a0cf44f4..0000000000000 --- a/cmd/kube-controller-manager/app/flags_providers.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/cli/globalflag" -) - -func registerLegacyGlobalFlags(namedFlagSets cliflag.NamedFlagSets) { - // hoist this flag from the global flagset to preserve the commandline until - // the gce cloudprovider is removed. - globalflag.Register(namedFlagSets.FlagSet("generic"), "cloud-provider-gce-lb-src-cidrs") - namedFlagSets.FlagSet("generic").MarkDeprecated("cloud-provider-gce-lb-src-cidrs", "This flag will be removed once the GCE Cloud Provider is removed from kube-controller-manager") -} diff --git a/cmd/kube-controller-manager/app/import_known_versions.go b/cmd/kube-controller-manager/app/import_known_versions.go deleted file mode 100644 index a76ce30997008..0000000000000 --- a/cmd/kube-controller-manager/app/import_known_versions.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app imports the API groups that the client will support -// TODO: Remove this file when namespace controller and garbage collector -// stops using legacyscheme.Registry.RESTMapper() -package app - -import ( - // These imports are the API groups the client will support. - _ "k8s.io/kubernetes/pkg/apis/apps/install" - _ "k8s.io/kubernetes/pkg/apis/authentication/install" - _ "k8s.io/kubernetes/pkg/apis/authorization/install" - _ "k8s.io/kubernetes/pkg/apis/autoscaling/install" - _ "k8s.io/kubernetes/pkg/apis/batch/install" - _ "k8s.io/kubernetes/pkg/apis/certificates/install" - _ "k8s.io/kubernetes/pkg/apis/coordination/install" - _ "k8s.io/kubernetes/pkg/apis/core/install" - _ "k8s.io/kubernetes/pkg/apis/events/install" - _ "k8s.io/kubernetes/pkg/apis/extensions/install" - _ "k8s.io/kubernetes/pkg/apis/policy/install" - _ "k8s.io/kubernetes/pkg/apis/rbac/install" - _ "k8s.io/kubernetes/pkg/apis/scheduling/install" - _ "k8s.io/kubernetes/pkg/apis/storage/install" -) diff --git a/cmd/kube-controller-manager/app/options/BUILD b/cmd/kube-controller-manager/app/options/BUILD deleted file mode 100644 index b9ce94c022206..0000000000000 --- a/cmd/kube-controller-manager/app/options/BUILD +++ /dev/null @@ -1,144 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "attachdetachcontroller.go", - "cronjobcontroller.go", - "csrsigningcontroller.go", - "daemonsetcontroller.go", - "deploymentcontroller.go", - "deprecatedcontroller.go", - "endpointcontroller.go", - "endpointslicecontroller.go", - "endpointslicemirroringcontroller.go", - "garbagecollectorcontroller.go", - "hpacontroller.go", - "jobcontroller.go", - "namespacecontroller.go", - "nodeipamcontroller.go", - "nodelifecyclecontroller.go", - "options.go", - "persistentvolumebindercontroller.go", - "podgccontroller.go", - "replicasetcontroller.go", - "replicationcontroller.go", - "resourcequotacontroller.go", - "serviceaccountcontroller.go", - "statefulsetcontroller.go", - "ttlafterfinishedcontroller.go", - ], - importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options", - deps = [ - "//cmd/kube-controller-manager/app/config:go_default_library", - "//pkg/cluster/ports:go_default_library", - "//pkg/controller/apis/config:go_default_library", - "//pkg/controller/apis/config/scheme:go_default_library", - "//pkg/controller/certificates/signer/config:go_default_library", - "//pkg/controller/cronjob/config:go_default_library", - "//pkg/controller/daemon/config:go_default_library", - "//pkg/controller/deployment/config:go_default_library", - "//pkg/controller/endpoint/config:go_default_library", - "//pkg/controller/endpointslice/config:go_default_library", - "//pkg/controller/endpointslicemirroring/config:go_default_library", - "//pkg/controller/garbagecollector:go_default_library", - "//pkg/controller/garbagecollector/config:go_default_library", - "//pkg/controller/job/config:go_default_library", - "//pkg/controller/namespace/config:go_default_library", - "//pkg/controller/nodeipam/config:go_default_library", - "//pkg/controller/nodelifecycle/config:go_default_library", - "//pkg/controller/podautoscaler/config:go_default_library", - "//pkg/controller/podgc/config:go_default_library", - "//pkg/controller/replicaset/config:go_default_library", - "//pkg/controller/replication/config:go_default_library", - "//pkg/controller/resourcequota/config:go_default_library", - "//pkg/controller/serviceaccount/config:go_default_library", - "//pkg/controller/statefulset/config:go_default_library", - "//pkg/controller/ttlafterfinished/config:go_default_library", - "//pkg/controller/volume/attachdetach/config:go_default_library", - "//pkg/controller/volume/persistentvolume/config:go_default_library", - "//pkg/features:go_default_library", - "//pkg/proxy/util:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/cloud-provider/options:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//staging/src/k8s.io/controller-manager/options:go_default_library", - "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["options_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kube-controller-manager/app/config:go_default_library", - "//pkg/controller/apis/config:go_default_library", - "//pkg/controller/certificates/signer/config:go_default_library", - "//pkg/controller/cronjob/config:go_default_library", - "//pkg/controller/daemon/config:go_default_library", - "//pkg/controller/deployment/config:go_default_library", - "//pkg/controller/endpoint/config:go_default_library", - "//pkg/controller/endpointslice/config:go_default_library", - "//pkg/controller/endpointslicemirroring/config:go_default_library", - "//pkg/controller/garbagecollector/config:go_default_library", - "//pkg/controller/job/config:go_default_library", - "//pkg/controller/namespace/config:go_default_library", - "//pkg/controller/nodeipam/config:go_default_library", - "//pkg/controller/nodelifecycle/config:go_default_library", - "//pkg/controller/podautoscaler/config:go_default_library", - "//pkg/controller/podgc/config:go_default_library", - "//pkg/controller/replicaset/config:go_default_library", - "//pkg/controller/replication/config:go_default_library", - "//pkg/controller/resourcequota/config:go_default_library", - "//pkg/controller/serviceaccount/config:go_default_library", - "//pkg/controller/statefulset/config:go_default_library", - "//pkg/controller/ttlafterfinished/config:go_default_library", - "//pkg/controller/volume/attachdetach/config:go_default_library", - "//pkg/controller/volume/persistentvolume/config:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/cloud-provider/config:go_default_library", - "//staging/src/k8s.io/cloud-provider/controllers/service/config:go_default_library", - "//staging/src/k8s.io/cloud-provider/options:go_default_library", - "//staging/src/k8s.io/component-base/config:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//staging/src/k8s.io/controller-manager/config:go_default_library", - "//staging/src/k8s.io/controller-manager/options:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) diff --git a/cmd/kube-controller-manager/app/options/attachdetachcontroller.go b/cmd/kube-controller-manager/app/options/attachdetachcontroller.go deleted file mode 100644 index 51c197b821373..0000000000000 --- a/cmd/kube-controller-manager/app/options/attachdetachcontroller.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - attachdetachconfig "k8s.io/kubernetes/pkg/controller/volume/attachdetach/config" -) - -// AttachDetachControllerOptions holds the AttachDetachController options. -type AttachDetachControllerOptions struct { - *attachdetachconfig.AttachDetachControllerConfiguration -} - -// AddFlags adds flags related to AttachDetachController for controller manager to the specified FlagSet. -func (o *AttachDetachControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.BoolVar(&o.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.") - fs.DurationVar(&o.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", o.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.") -} - -// ApplyTo fills up AttachDetachController config with options. -func (o *AttachDetachControllerOptions) ApplyTo(cfg *attachdetachconfig.AttachDetachControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.DisableAttachDetachReconcilerSync = o.DisableAttachDetachReconcilerSync - cfg.ReconcilerSyncLoopPeriod = o.ReconcilerSyncLoopPeriod - - return nil -} - -// Validate checks validation of AttachDetachControllerOptions. -func (o *AttachDetachControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/cronjobcontroller.go b/cmd/kube-controller-manager/app/options/cronjobcontroller.go deleted file mode 100644 index 48f03ca67c0e8..0000000000000 --- a/cmd/kube-controller-manager/app/options/cronjobcontroller.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - cronjobconfig "k8s.io/kubernetes/pkg/controller/cronjob/config" -) - -// CronJobControllerOptions holds the CronJobController options. -type CronJobControllerOptions struct { - *cronjobconfig.CronJobControllerConfiguration -} - -// AddFlags adds flags related to JobController for controller manager to the specified FlagSet. -func (o *CronJobControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } -} - -// ApplyTo fills up JobController config with options. -func (o *CronJobControllerOptions) ApplyTo(cfg *cronjobconfig.CronJobControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentCronJobSyncs = o.ConcurrentCronJobSyncs - - return nil -} - -// Validate checks validation of CronJobControllerOptions. -func (o *CronJobControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/csrsigningcontroller.go b/cmd/kube-controller-manager/app/options/csrsigningcontroller.go deleted file mode 100644 index ebb498868032d..0000000000000 --- a/cmd/kube-controller-manager/app/options/csrsigningcontroller.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - - "github.com/spf13/pflag" - - csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config" -) - -// CSRSigningControllerOptions holds the CSRSigningController options. -type CSRSigningControllerOptions struct { - *csrsigningconfig.CSRSigningControllerConfiguration -} - -// AddFlags adds flags related to CSRSigningController for controller manager to the specified FlagSet. -func (o *CSRSigningControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.StringVar(&o.ClusterSigningCertFile, "cluster-signing-cert-file", o.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates. If specified, no more specific --cluster-signing-* flag may be specified.") - fs.StringVar(&o.ClusterSigningKeyFile, "cluster-signing-key-file", o.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates. If specified, no more specific --cluster-signing-* flag may be specified.") - fs.StringVar(&o.KubeletServingSignerConfiguration.CertFile, "cluster-signing-kubelet-serving-cert-file", o.KubeletServingSignerConfiguration.CertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue certificates for the kubernetes.io/kubelet-serving signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.KubeletServingSignerConfiguration.KeyFile, "cluster-signing-kubelet-serving-key-file", o.KubeletServingSignerConfiguration.KeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign certificates for the kubernetes.io/kubelet-serving signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.KubeletClientSignerConfiguration.CertFile, "cluster-signing-kubelet-client-cert-file", o.KubeletClientSignerConfiguration.CertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue certificates for the kubernetes.io/kube-apiserver-client-kubelet signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.KubeletClientSignerConfiguration.KeyFile, "cluster-signing-kubelet-client-key-file", o.KubeletClientSignerConfiguration.KeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign certificates for the kubernetes.io/kube-apiserver-client-kubelet signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.KubeAPIServerClientSignerConfiguration.CertFile, "cluster-signing-kube-apiserver-client-cert-file", o.KubeAPIServerClientSignerConfiguration.CertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue certificates for the kubernetes.io/kube-apiserver-client signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.KubeAPIServerClientSignerConfiguration.KeyFile, "cluster-signing-kube-apiserver-client-key-file", o.KubeAPIServerClientSignerConfiguration.KeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign certificates for the kubernetes.io/kube-apiserver-client signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.LegacyUnknownSignerConfiguration.CertFile, "cluster-signing-legacy-unknown-cert-file", o.LegacyUnknownSignerConfiguration.CertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue certificates for the kubernetes.io/legacy-unknown signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.StringVar(&o.LegacyUnknownSignerConfiguration.KeyFile, "cluster-signing-legacy-unknown-key-file", o.LegacyUnknownSignerConfiguration.KeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign certificates for the kubernetes.io/legacy-unknown signer. If specified, --cluster-signing-{cert,key}-file must not be set.") - fs.DurationVar(&o.ClusterSigningDuration.Duration, "cluster-signing-duration", o.ClusterSigningDuration.Duration, "The length of duration signed certificates will be given.") - fs.DurationVar(&o.ClusterSigningDuration.Duration, "experimental-cluster-signing-duration", o.ClusterSigningDuration.Duration, "The length of duration signed certificates will be given.") - fs.MarkDeprecated("experimental-cluster-signing-duration", "use --cluster-signing-duration") -} - -// ApplyTo fills up CSRSigningController config with options. -func (o *CSRSigningControllerOptions) ApplyTo(cfg *csrsigningconfig.CSRSigningControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ClusterSigningCertFile = o.ClusterSigningCertFile - cfg.ClusterSigningKeyFile = o.ClusterSigningKeyFile - cfg.KubeletServingSignerConfiguration = o.KubeletServingSignerConfiguration - cfg.KubeletClientSignerConfiguration = o.KubeletClientSignerConfiguration - cfg.KubeAPIServerClientSignerConfiguration = o.KubeAPIServerClientSignerConfiguration - cfg.LegacyUnknownSignerConfiguration = o.LegacyUnknownSignerConfiguration - cfg.ClusterSigningDuration = o.ClusterSigningDuration - - return nil -} - -// Validate checks validation of CSRSigningControllerOptions. -func (o *CSRSigningControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - if err := csrSigningFilesValid(o.KubeletServingSignerConfiguration); err != nil { - errs = append(errs, fmt.Errorf("%q: %v", "cluster-signing-kubelet-serving", err)) - } - if err := csrSigningFilesValid(o.KubeletClientSignerConfiguration); err != nil { - errs = append(errs, fmt.Errorf("%q: %v", "cluster-signing-kube-apiserver-client", err)) - } - if err := csrSigningFilesValid(o.KubeAPIServerClientSignerConfiguration); err != nil { - errs = append(errs, fmt.Errorf("%q: %v", "cluster-signing-kube-apiserver", err)) - } - if err := csrSigningFilesValid(o.LegacyUnknownSignerConfiguration); err != nil { - errs = append(errs, fmt.Errorf("%q: %v", "cluster-signing-legacy-unknown", err)) - } - - singleSigningFile := len(o.ClusterSigningCertFile) > 0 || len(o.ClusterSigningKeyFile) > 0 - anySpecificFilesSet := len(o.KubeletServingSignerConfiguration.CertFile) > 0 || len(o.KubeletServingSignerConfiguration.KeyFile) > 0 || - len(o.KubeletClientSignerConfiguration.CertFile) > 0 || len(o.KubeletClientSignerConfiguration.KeyFile) > 0 || - len(o.KubeAPIServerClientSignerConfiguration.CertFile) > 0 || len(o.KubeAPIServerClientSignerConfiguration.KeyFile) > 0 || - len(o.LegacyUnknownSignerConfiguration.CertFile) > 0 || len(o.LegacyUnknownSignerConfiguration.KeyFile) > 0 - if singleSigningFile && anySpecificFilesSet { - errs = append(errs, fmt.Errorf("cannot specify --cluster-signing-{cert,key}-file and other --cluster-signing-*-file flags at the same time")) - } - - return errs -} - -// both must be specified or both must be empty -func csrSigningFilesValid(config csrsigningconfig.CSRSigningConfiguration) error { - switch { - case (len(config.CertFile) == 0) && (len(config.KeyFile) == 0): - return nil - case (len(config.CertFile) != 0) && (len(config.KeyFile) != 0): - return nil - case (len(config.CertFile) == 0) && (len(config.KeyFile) != 0): - return fmt.Errorf("cannot specify key without cert") - case (len(config.CertFile) != 0) && (len(config.KeyFile) == 0): - return fmt.Errorf("cannot specify cert without key") - } - - return fmt.Errorf("math broke") -} diff --git a/cmd/kube-controller-manager/app/options/daemonsetcontroller.go b/cmd/kube-controller-manager/app/options/daemonsetcontroller.go deleted file mode 100644 index 8506f8ec05c72..0000000000000 --- a/cmd/kube-controller-manager/app/options/daemonsetcontroller.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - daemonconfig "k8s.io/kubernetes/pkg/controller/daemon/config" -) - -// DaemonSetControllerOptions holds the DaemonSetController options. -type DaemonSetControllerOptions struct { - *daemonconfig.DaemonSetControllerConfiguration -} - -// AddFlags adds flags related to DaemonSetController for controller manager to the specified FlagSet. -func (o *DaemonSetControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } -} - -// ApplyTo fills up DaemonSetController config with options. -func (o *DaemonSetControllerOptions) ApplyTo(cfg *daemonconfig.DaemonSetControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentDaemonSetSyncs = o.ConcurrentDaemonSetSyncs - - return nil -} - -// Validate checks validation of DaemonSetControllerOptions. -func (o *DaemonSetControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/deploymentcontroller.go b/cmd/kube-controller-manager/app/options/deploymentcontroller.go deleted file mode 100644 index e352594e5edc4..0000000000000 --- a/cmd/kube-controller-manager/app/options/deploymentcontroller.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - deploymentconfig "k8s.io/kubernetes/pkg/controller/deployment/config" -) - -// DeploymentControllerOptions holds the DeploymentController options. -type DeploymentControllerOptions struct { - *deploymentconfig.DeploymentControllerConfiguration -} - -// AddFlags adds flags related to DeploymentController for controller manager to the specified FlagSet. -func (o *DeploymentControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", o.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load") - fs.DurationVar(&o.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", o.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.") -} - -// ApplyTo fills up DeploymentController config with options. -func (o *DeploymentControllerOptions) ApplyTo(cfg *deploymentconfig.DeploymentControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentDeploymentSyncs = o.ConcurrentDeploymentSyncs - cfg.DeploymentControllerSyncPeriod = o.DeploymentControllerSyncPeriod - - return nil -} - -// Validate checks validation of DeploymentControllerOptions. -func (o *DeploymentControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/deprecatedcontroller.go b/cmd/kube-controller-manager/app/options/deprecatedcontroller.go deleted file mode 100644 index 48ff5dd0d0a20..0000000000000 --- a/cmd/kube-controller-manager/app/options/deprecatedcontroller.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" -) - -// DeprecatedControllerOptions holds the DeprecatedController options, those option are deprecated. -// TODO remove these fields once the deprecated flags are removed. -type DeprecatedControllerOptions struct { - *kubectrlmgrconfig.DeprecatedControllerConfiguration -} - -// AddFlags adds flags related to DeprecatedController for controller manager to the specified FlagSet. -func (o *DeprecatedControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Float32Var(&o.DeletingPodsQPS, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.") - fs.MarkDeprecated("deleting-pods-qps", "This flag is currently no-op and will be deleted.") - fs.Int32Var(&o.DeletingPodsBurst, "deleting-pods-burst", 0, "Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter.") - fs.MarkDeprecated("deleting-pods-burst", "This flag is currently no-op and will be deleted.") - fs.Int32Var(&o.RegisterRetryCount, "register-retry-count", o.RegisterRetryCount, ""+ - "The number of retries for initial node registration. Retry interval equals node-sync-period.") - fs.MarkDeprecated("register-retry-count", "This flag is currently no-op and will be deleted.") -} - -// ApplyTo fills up DeprecatedController config with options. -func (o *DeprecatedControllerOptions) ApplyTo(cfg *kubectrlmgrconfig.DeprecatedControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.DeletingPodsQPS = o.DeletingPodsQPS - cfg.DeletingPodsBurst = o.DeletingPodsBurst - cfg.RegisterRetryCount = o.RegisterRetryCount - - return nil -} - -// Validate checks validation of DeprecatedControllerOptions. -func (o *DeprecatedControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/endpointcontroller.go b/cmd/kube-controller-manager/app/options/endpointcontroller.go deleted file mode 100644 index ae052154fefe1..0000000000000 --- a/cmd/kube-controller-manager/app/options/endpointcontroller.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - endpointconfig "k8s.io/kubernetes/pkg/controller/endpoint/config" -) - -// EndpointControllerOptions holds the EndPointController options. -type EndpointControllerOptions struct { - *endpointconfig.EndpointControllerConfiguration -} - -// AddFlags adds flags related to EndPointController for controller manager to the specified FlagSet. -func (o *EndpointControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", o.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load") - fs.DurationVar(&o.EndpointUpdatesBatchPeriod.Duration, "endpoint-updates-batch-period", o.EndpointUpdatesBatchPeriod.Duration, "The length of endpoint updates batching period. Processing of pod changes will be delayed by this duration to join them with potential upcoming updates and reduce the overall number of endpoints updates. Larger number = higher endpoint programming latency, but lower number of endpoints revision generated") -} - -// ApplyTo fills up EndPointController config with options. -func (o *EndpointControllerOptions) ApplyTo(cfg *endpointconfig.EndpointControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentEndpointSyncs = o.ConcurrentEndpointSyncs - cfg.EndpointUpdatesBatchPeriod = o.EndpointUpdatesBatchPeriod - - return nil -} - -// Validate checks validation of EndpointControllerOptions. -func (o *EndpointControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/endpointslicecontroller.go b/cmd/kube-controller-manager/app/options/endpointslicecontroller.go deleted file mode 100644 index c67e6750d468f..0000000000000 --- a/cmd/kube-controller-manager/app/options/endpointslicecontroller.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - - "github.com/spf13/pflag" - - endpointsliceconfig "k8s.io/kubernetes/pkg/controller/endpointslice/config" -) - -const ( - minConcurrentServiceEndpointSyncs = 1 - maxConcurrentServiceEndpointSyncs = 50 - minMaxEndpointsPerSlice = 1 - maxMaxEndpointsPerSlice = 1000 -) - -// EndpointSliceControllerOptions holds the EndpointSliceController options. -type EndpointSliceControllerOptions struct { - *endpointsliceconfig.EndpointSliceControllerConfiguration -} - -// AddFlags adds flags related to EndpointSliceController for controller manager to the specified FlagSet. -func (o *EndpointSliceControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentServiceEndpointSyncs, "concurrent-service-endpoint-syncs", o.ConcurrentServiceEndpointSyncs, "The number of service endpoint syncing operations that will be done concurrently. Larger number = faster endpoint slice updating, but more CPU (and network) load. Defaults to 5.") - fs.Int32Var(&o.MaxEndpointsPerSlice, "max-endpoints-per-slice", o.MaxEndpointsPerSlice, "The maximum number of endpoints that will be added to an EndpointSlice. More endpoints per slice will result in less endpoint slices, but larger resources. Defaults to 100.") - fs.DurationVar(&o.EndpointUpdatesBatchPeriod.Duration, "endpointslice-updates-batch-period", o.EndpointUpdatesBatchPeriod.Duration, "The length of endpoint slice updates batching period. Processing of pod changes will be delayed by this duration to join them with potential upcoming updates and reduce the overall number of endpoints updates. Larger number = higher endpoint programming latency, but lower number of endpoints revision generated") -} - -// ApplyTo fills up EndpointSliceController config with options. -func (o *EndpointSliceControllerOptions) ApplyTo(cfg *endpointsliceconfig.EndpointSliceControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentServiceEndpointSyncs = o.ConcurrentServiceEndpointSyncs - cfg.MaxEndpointsPerSlice = o.MaxEndpointsPerSlice - cfg.EndpointUpdatesBatchPeriod = o.EndpointUpdatesBatchPeriod - - return nil -} - -// Validate checks validation of EndpointSliceControllerOptions. -func (o *EndpointSliceControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - - if o.ConcurrentServiceEndpointSyncs < minConcurrentServiceEndpointSyncs { - errs = append(errs, fmt.Errorf("concurrent-service-endpoint-syncs must not be less than %d, but got %d", minConcurrentServiceEndpointSyncs, o.ConcurrentServiceEndpointSyncs)) - } else if o.ConcurrentServiceEndpointSyncs > maxConcurrentServiceEndpointSyncs { - errs = append(errs, fmt.Errorf("concurrent-service-endpoint-syncs must not be more than %d, but got %d", maxConcurrentServiceEndpointSyncs, o.ConcurrentServiceEndpointSyncs)) - } - - if o.MaxEndpointsPerSlice < minMaxEndpointsPerSlice { - errs = append(errs, fmt.Errorf("max-endpoints-per-slice must not be less than %d, but got %d", minMaxEndpointsPerSlice, o.MaxEndpointsPerSlice)) - } else if o.MaxEndpointsPerSlice > maxMaxEndpointsPerSlice { - errs = append(errs, fmt.Errorf("max-endpoints-per-slice must not be more than %d, but got %d", maxMaxEndpointsPerSlice, o.MaxEndpointsPerSlice)) - } - - return errs -} diff --git a/cmd/kube-controller-manager/app/options/endpointslicemirroringcontroller.go b/cmd/kube-controller-manager/app/options/endpointslicemirroringcontroller.go deleted file mode 100644 index b76f4e0298b71..0000000000000 --- a/cmd/kube-controller-manager/app/options/endpointslicemirroringcontroller.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - - "github.com/spf13/pflag" - - endpointslicemirroringconfig "k8s.io/kubernetes/pkg/controller/endpointslicemirroring/config" -) - -const ( - mirroringMinConcurrentServiceEndpointSyncs = 1 - mirroringMaxConcurrentServiceEndpointSyncs = 50 - mirroringMinMaxEndpointsPerSubset = 1 - mirroringMaxMaxEndpointsPerSubset = 1000 -) - -// EndpointSliceMirroringControllerOptions holds the -// EndpointSliceMirroringController options. -type EndpointSliceMirroringControllerOptions struct { - *endpointslicemirroringconfig.EndpointSliceMirroringControllerConfiguration -} - -// AddFlags adds flags related to EndpointSliceMirroringController for -// controller manager to the specified FlagSet. -func (o *EndpointSliceMirroringControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.MirroringConcurrentServiceEndpointSyncs, "mirroring-concurrent-service-endpoint-syncs", o.MirroringConcurrentServiceEndpointSyncs, "The number of service endpoint syncing operations that will be done concurrently by the EndpointSliceMirroring controller. Larger number = faster endpoint slice updating, but more CPU (and network) load. Defaults to 5.") - fs.Int32Var(&o.MirroringMaxEndpointsPerSubset, "mirroring-max-endpoints-per-subset", o.MirroringMaxEndpointsPerSubset, "The maximum number of endpoints that will be added to an EndpointSlice by the EndpointSliceMirroring controller. More endpoints per slice will result in less endpoint slices, but larger resources. Defaults to 100.") - fs.DurationVar(&o.MirroringEndpointUpdatesBatchPeriod.Duration, "mirroring-endpointslice-updates-batch-period", o.MirroringEndpointUpdatesBatchPeriod.Duration, "The length of EndpointSlice updates batching period for EndpointSliceMirroring controller. Processing of EndpointSlice changes will be delayed by this duration to join them with potential upcoming updates and reduce the overall number of EndpointSlice updates. Larger number = higher endpoint programming latency, but lower number of endpoints revision generated") -} - -// ApplyTo fills up EndpointSliceMirroringController config with options. -func (o *EndpointSliceMirroringControllerOptions) ApplyTo(cfg *endpointslicemirroringconfig.EndpointSliceMirroringControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.MirroringConcurrentServiceEndpointSyncs = o.MirroringConcurrentServiceEndpointSyncs - cfg.MirroringMaxEndpointsPerSubset = o.MirroringMaxEndpointsPerSubset - cfg.MirroringEndpointUpdatesBatchPeriod = o.MirroringEndpointUpdatesBatchPeriod - - return nil -} - -// Validate checks validation of EndpointSliceMirroringControllerOptions. -func (o *EndpointSliceMirroringControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - - if o.MirroringConcurrentServiceEndpointSyncs < mirroringMinConcurrentServiceEndpointSyncs { - errs = append(errs, fmt.Errorf("mirroring-concurrent-service-endpoint-syncs must not be less than %d, but got %d", mirroringMinConcurrentServiceEndpointSyncs, o.MirroringConcurrentServiceEndpointSyncs)) - } else if o.MirroringConcurrentServiceEndpointSyncs > mirroringMaxConcurrentServiceEndpointSyncs { - errs = append(errs, fmt.Errorf("mirroring-concurrent-service-endpoint-syncs must not be more than %d, but got %d", mirroringMaxConcurrentServiceEndpointSyncs, o.MirroringConcurrentServiceEndpointSyncs)) - } - - if o.MirroringMaxEndpointsPerSubset < mirroringMinMaxEndpointsPerSubset { - errs = append(errs, fmt.Errorf("mirroring-max-endpoints-per-subset must not be less than %d, but got %d", mirroringMinMaxEndpointsPerSubset, o.MirroringMaxEndpointsPerSubset)) - } else if o.MirroringMaxEndpointsPerSubset > mirroringMaxMaxEndpointsPerSubset { - errs = append(errs, fmt.Errorf("mirroring-max-endpoints-per-subset must not be more than %d, but got %d", mirroringMaxMaxEndpointsPerSubset, o.MirroringMaxEndpointsPerSubset)) - } - - return errs -} diff --git a/cmd/kube-controller-manager/app/options/garbagecollectorcontroller.go b/cmd/kube-controller-manager/app/options/garbagecollectorcontroller.go deleted file mode 100644 index 311917465296d..0000000000000 --- a/cmd/kube-controller-manager/app/options/garbagecollectorcontroller.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - garbagecollectorconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config" -) - -// GarbageCollectorControllerOptions holds the GarbageCollectorController options. -type GarbageCollectorControllerOptions struct { - *garbagecollectorconfig.GarbageCollectorControllerConfiguration -} - -// AddFlags adds flags related to GarbageCollectorController for controller manager to the specified FlagSet. -func (o *GarbageCollectorControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentGCSyncs, "concurrent-gc-syncs", o.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.") - fs.BoolVar(&o.EnableGarbageCollector, "enable-garbage-collector", o.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.") -} - -// ApplyTo fills up GarbageCollectorController config with options. -func (o *GarbageCollectorControllerOptions) ApplyTo(cfg *garbagecollectorconfig.GarbageCollectorControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentGCSyncs = o.ConcurrentGCSyncs - cfg.GCIgnoredResources = o.GCIgnoredResources - cfg.EnableGarbageCollector = o.EnableGarbageCollector - - return nil -} - -// Validate checks validation of GarbageCollectorController. -func (o *GarbageCollectorControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/hpacontroller.go b/cmd/kube-controller-manager/app/options/hpacontroller.go deleted file mode 100644 index 5a34eec508cc1..0000000000000 --- a/cmd/kube-controller-manager/app/options/hpacontroller.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - poautosclerconfig "k8s.io/kubernetes/pkg/controller/podautoscaler/config" -) - -// HPAControllerOptions holds the HPAController options. -type HPAControllerOptions struct { - *poautosclerconfig.HPAControllerConfiguration -} - -// AddFlags adds flags related to HPAController for controller manager to the specified FlagSet. -func (o *HPAControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.DurationVar(&o.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", o.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.") - fs.DurationVar(&o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", o.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.") - fs.MarkDeprecated("horizontal-pod-autoscaler-upscale-delay", "This flag is currently no-op and will be deleted.") - fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "horizontal-pod-autoscaler-downscale-stabilization", o.HorizontalPodAutoscalerDownscaleStabilizationWindow.Duration, "The period for which autoscaler will look backwards and not scale down below any recommendation it made during that period.") - fs.DurationVar(&o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", o.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.") - fs.MarkDeprecated("horizontal-pod-autoscaler-downscale-delay", "This flag is currently no-op and will be deleted.") - fs.Float64Var(&o.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", o.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.") - fs.BoolVar(&o.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", o.HorizontalPodAutoscalerUseRESTClients, "If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy. This is required for custom metrics support in the horizontal pod autoscaler.") - fs.DurationVar(&o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "horizontal-pod-autoscaler-cpu-initialization-period", o.HorizontalPodAutoscalerCPUInitializationPeriod.Duration, "The period after pod start when CPU samples might be skipped.") - fs.MarkDeprecated("horizontal-pod-autoscaler-use-rest-clients", "Heapster is no longer supported as a source for Horizontal Pod Autoscaler metrics.") - fs.DurationVar(&o.HorizontalPodAutoscalerInitialReadinessDelay.Duration, "horizontal-pod-autoscaler-initial-readiness-delay", o.HorizontalPodAutoscalerInitialReadinessDelay.Duration, "The period after pod start during which readiness changes will be treated as initial readiness.") -} - -// ApplyTo fills up HPAController config with options. -func (o *HPAControllerOptions) ApplyTo(cfg *poautosclerconfig.HPAControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.HorizontalPodAutoscalerSyncPeriod = o.HorizontalPodAutoscalerSyncPeriod - cfg.HorizontalPodAutoscalerDownscaleStabilizationWindow = o.HorizontalPodAutoscalerDownscaleStabilizationWindow - cfg.HorizontalPodAutoscalerTolerance = o.HorizontalPodAutoscalerTolerance - cfg.HorizontalPodAutoscalerUseRESTClients = o.HorizontalPodAutoscalerUseRESTClients - cfg.HorizontalPodAutoscalerCPUInitializationPeriod = o.HorizontalPodAutoscalerCPUInitializationPeriod - cfg.HorizontalPodAutoscalerInitialReadinessDelay = o.HorizontalPodAutoscalerInitialReadinessDelay - cfg.HorizontalPodAutoscalerUpscaleForbiddenWindow = o.HorizontalPodAutoscalerUpscaleForbiddenWindow - cfg.HorizontalPodAutoscalerDownscaleForbiddenWindow = o.HorizontalPodAutoscalerDownscaleForbiddenWindow - - return nil -} - -// Validate checks validation of HPAControllerOptions. -func (o *HPAControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/jobcontroller.go b/cmd/kube-controller-manager/app/options/jobcontroller.go deleted file mode 100644 index 2e7d66790b0bb..0000000000000 --- a/cmd/kube-controller-manager/app/options/jobcontroller.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - jobconfig "k8s.io/kubernetes/pkg/controller/job/config" -) - -// JobControllerOptions holds the JobController options. -type JobControllerOptions struct { - *jobconfig.JobControllerConfiguration -} - -// AddFlags adds flags related to JobController for controller manager to the specified FlagSet. -func (o *JobControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } -} - -// ApplyTo fills up JobController config with options. -func (o *JobControllerOptions) ApplyTo(cfg *jobconfig.JobControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentJobSyncs = o.ConcurrentJobSyncs - - return nil -} - -// Validate checks validation of JobControllerOptions. -func (o *JobControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/namespacecontroller.go b/cmd/kube-controller-manager/app/options/namespacecontroller.go deleted file mode 100644 index a8ce6801f3e4f..0000000000000 --- a/cmd/kube-controller-manager/app/options/namespacecontroller.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - namespaceconfig "k8s.io/kubernetes/pkg/controller/namespace/config" -) - -// NamespaceControllerOptions holds the NamespaceController options. -type NamespaceControllerOptions struct { - *namespaceconfig.NamespaceControllerConfiguration -} - -// AddFlags adds flags related to NamespaceController for controller manager to the specified FlagSet. -func (o *NamespaceControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.DurationVar(&o.NamespaceSyncPeriod.Duration, "namespace-sync-period", o.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates") - fs.Int32Var(&o.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", o.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load") -} - -// ApplyTo fills up NamespaceController config with options. -func (o *NamespaceControllerOptions) ApplyTo(cfg *namespaceconfig.NamespaceControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.NamespaceSyncPeriod = o.NamespaceSyncPeriod - cfg.ConcurrentNamespaceSyncs = o.ConcurrentNamespaceSyncs - - return nil -} - -// Validate checks validation of NamespaceControllerOptions. -func (o *NamespaceControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/nodeipamcontroller.go b/cmd/kube-controller-manager/app/options/nodeipamcontroller.go deleted file mode 100644 index 0a897e472bd5d..0000000000000 --- a/cmd/kube-controller-manager/app/options/nodeipamcontroller.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "strings" - - "github.com/spf13/pflag" - - nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" -) - -// NodeIPAMControllerOptions holds the NodeIpamController options. -type NodeIPAMControllerOptions struct { - *nodeipamconfig.NodeIPAMControllerConfiguration -} - -// AddFlags adds flags related to NodeIpamController for controller manager to the specified FlagSet. -func (o *NodeIPAMControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - fs.StringVar(&o.ServiceCIDR, "service-cluster-ip-range", o.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true") - fs.Int32Var(&o.NodeCIDRMaskSize, "node-cidr-mask-size", o.NodeCIDRMaskSize, "Mask size for node cidr in cluster. Default is 24 for IPv4 and 64 for IPv6.") - fs.Int32Var(&o.NodeCIDRMaskSizeIPv4, "node-cidr-mask-size-ipv4", o.NodeCIDRMaskSizeIPv4, "Mask size for IPv4 node cidr in dual-stack cluster. Default is 24.") - fs.Int32Var(&o.NodeCIDRMaskSizeIPv6, "node-cidr-mask-size-ipv6", o.NodeCIDRMaskSizeIPv6, "Mask size for IPv6 node cidr in dual-stack cluster. Default is 64.") -} - -// ApplyTo fills up NodeIpamController config with options. -func (o *NodeIPAMControllerOptions) ApplyTo(cfg *nodeipamconfig.NodeIPAMControllerConfiguration) error { - if o == nil { - return nil - } - - // split the cidrs list and assign primary and secondary - serviceCIDRList := strings.Split(o.ServiceCIDR, ",") - if len(serviceCIDRList) > 0 { - cfg.ServiceCIDR = serviceCIDRList[0] - } - if len(serviceCIDRList) > 1 { - cfg.SecondaryServiceCIDR = serviceCIDRList[1] - } - - cfg.NodeCIDRMaskSize = o.NodeCIDRMaskSize - cfg.NodeCIDRMaskSizeIPv4 = o.NodeCIDRMaskSizeIPv4 - cfg.NodeCIDRMaskSizeIPv6 = o.NodeCIDRMaskSizeIPv6 - - return nil -} - -// Validate checks validation of NodeIPAMControllerOptions. -func (o *NodeIPAMControllerOptions) Validate() []error { - if o == nil { - return nil - } - errs := make([]error, 0) - - serviceCIDRList := strings.Split(o.ServiceCIDR, ",") - if len(serviceCIDRList) > 2 { - errs = append(errs, fmt.Errorf("--service-cluster-ip-range can not contain more than two entries")) - } - - return errs -} diff --git a/cmd/kube-controller-manager/app/options/nodelifecyclecontroller.go b/cmd/kube-controller-manager/app/options/nodelifecyclecontroller.go deleted file mode 100644 index bded847a6d31c..0000000000000 --- a/cmd/kube-controller-manager/app/options/nodelifecyclecontroller.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - nodelifecycleconfig "k8s.io/kubernetes/pkg/controller/nodelifecycle/config" -) - -// NodeLifecycleControllerOptions holds the NodeLifecycleController options. -type NodeLifecycleControllerOptions struct { - *nodelifecycleconfig.NodeLifecycleControllerConfiguration -} - -// AddFlags adds flags related to NodeLifecycleController for controller manager to the specified FlagSet. -func (o *NodeLifecycleControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.DurationVar(&o.NodeStartupGracePeriod.Duration, "node-startup-grace-period", o.NodeStartupGracePeriod.Duration, - "Amount of time which we allow starting Node to be unresponsive before marking it unhealthy.") - fs.DurationVar(&o.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", o.NodeMonitorGracePeriod.Duration, - "Amount of time which we allow running Node to be unresponsive before marking it unhealthy. "+ - "Must be N times more than kubelet's nodeStatusUpdateFrequency, "+ - "where N means number of retries allowed for kubelet to post node status.") - fs.DurationVar(&o.PodEvictionTimeout.Duration, "pod-eviction-timeout", o.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.") - fs.Float32Var(&o.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.") - fs.Float32Var(&o.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.") - fs.Int32Var(&o.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.") - fs.Float32Var(&o.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ") - fs.BoolVar(&o.EnableTaintManager, "enable-taint-manager", o.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.") -} - -// ApplyTo fills up NodeLifecycleController config with options. -func (o *NodeLifecycleControllerOptions) ApplyTo(cfg *nodelifecycleconfig.NodeLifecycleControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.EnableTaintManager = o.EnableTaintManager - cfg.NodeStartupGracePeriod = o.NodeStartupGracePeriod - cfg.NodeMonitorGracePeriod = o.NodeMonitorGracePeriod - cfg.PodEvictionTimeout = o.PodEvictionTimeout - cfg.NodeEvictionRate = o.NodeEvictionRate - cfg.SecondaryNodeEvictionRate = o.SecondaryNodeEvictionRate - cfg.LargeClusterSizeThreshold = o.LargeClusterSizeThreshold - cfg.UnhealthyZoneThreshold = o.UnhealthyZoneThreshold - - return nil -} - -// Validate checks validation of NodeLifecycleControllerOptions. -func (o *NodeLifecycleControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go deleted file mode 100644 index d0475bd0c20b3..0000000000000 --- a/cmd/kube-controller-manager/app/options/options.go +++ /dev/null @@ -1,472 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package options provides the flags used for the controller manager. -// -package options - -import ( - "fmt" - "net" - - v1 "k8s.io/api/core/v1" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" - clientset "k8s.io/client-go/kubernetes" - clientgokubescheme "k8s.io/client-go/kubernetes/scheme" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/record" - cpoptions "k8s.io/cloud-provider/options" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - cmoptions "k8s.io/controller-manager/options" - kubectrlmgrconfigv1alpha1 "k8s.io/kube-controller-manager/config/v1alpha1" - kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" - "k8s.io/kubernetes/pkg/cluster/ports" - kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" - kubectrlmgrconfigscheme "k8s.io/kubernetes/pkg/controller/apis/config/scheme" - "k8s.io/kubernetes/pkg/controller/garbagecollector" - garbagecollectorconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config" - - // add the kubernetes feature gates - _ "k8s.io/kubernetes/pkg/features" -) - -const ( - // KubeControllerManagerUserAgent is the userAgent name when starting kube-controller managers. - KubeControllerManagerUserAgent = "kube-controller-manager" -) - -// KubeControllerManagerOptions is the main context object for the kube-controller manager. -type KubeControllerManagerOptions struct { - Generic *cmoptions.GenericControllerManagerConfigurationOptions - KubeCloudShared *cpoptions.KubeCloudSharedOptions - ServiceController *cpoptions.ServiceControllerOptions - - AttachDetachController *AttachDetachControllerOptions - CSRSigningController *CSRSigningControllerOptions - DaemonSetController *DaemonSetControllerOptions - DeploymentController *DeploymentControllerOptions - StatefulSetController *StatefulSetControllerOptions - DeprecatedFlags *DeprecatedControllerOptions - EndpointController *EndpointControllerOptions - EndpointSliceController *EndpointSliceControllerOptions - EndpointSliceMirroringController *EndpointSliceMirroringControllerOptions - GarbageCollectorController *GarbageCollectorControllerOptions - HPAController *HPAControllerOptions - JobController *JobControllerOptions - CronJobController *CronJobControllerOptions - NamespaceController *NamespaceControllerOptions - NodeIPAMController *NodeIPAMControllerOptions - NodeLifecycleController *NodeLifecycleControllerOptions - PersistentVolumeBinderController *PersistentVolumeBinderControllerOptions - PodGCController *PodGCControllerOptions - ReplicaSetController *ReplicaSetControllerOptions - ReplicationController *ReplicationControllerOptions - ResourceQuotaController *ResourceQuotaControllerOptions - SAController *SAControllerOptions - TTLAfterFinishedController *TTLAfterFinishedControllerOptions - - SecureServing *apiserveroptions.SecureServingOptionsWithLoopback - // TODO: remove insecure serving mode - InsecureServing *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback - Authentication *apiserveroptions.DelegatingAuthenticationOptions - Authorization *apiserveroptions.DelegatingAuthorizationOptions - Metrics *metrics.Options - Logs *logs.Options - - Master string - Kubeconfig string - ShowHiddenMetricsForVersion string -} - -// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config. -func NewKubeControllerManagerOptions() (*KubeControllerManagerOptions, error) { - componentConfig, err := NewDefaultComponentConfig(ports.InsecureKubeControllerManagerPort) - if err != nil { - return nil, err - } - - s := KubeControllerManagerOptions{ - Generic: cmoptions.NewGenericControllerManagerConfigurationOptions(&componentConfig.Generic), - KubeCloudShared: cpoptions.NewKubeCloudSharedOptions(&componentConfig.KubeCloudShared), - ServiceController: &cpoptions.ServiceControllerOptions{ - ServiceControllerConfiguration: &componentConfig.ServiceController, - }, - AttachDetachController: &AttachDetachControllerOptions{ - &componentConfig.AttachDetachController, - }, - CSRSigningController: &CSRSigningControllerOptions{ - &componentConfig.CSRSigningController, - }, - DaemonSetController: &DaemonSetControllerOptions{ - &componentConfig.DaemonSetController, - }, - DeploymentController: &DeploymentControllerOptions{ - &componentConfig.DeploymentController, - }, - StatefulSetController: &StatefulSetControllerOptions{ - &componentConfig.StatefulSetController, - }, - DeprecatedFlags: &DeprecatedControllerOptions{ - &componentConfig.DeprecatedController, - }, - EndpointController: &EndpointControllerOptions{ - &componentConfig.EndpointController, - }, - EndpointSliceController: &EndpointSliceControllerOptions{ - &componentConfig.EndpointSliceController, - }, - EndpointSliceMirroringController: &EndpointSliceMirroringControllerOptions{ - &componentConfig.EndpointSliceMirroringController, - }, - GarbageCollectorController: &GarbageCollectorControllerOptions{ - &componentConfig.GarbageCollectorController, - }, - HPAController: &HPAControllerOptions{ - &componentConfig.HPAController, - }, - JobController: &JobControllerOptions{ - &componentConfig.JobController, - }, - CronJobController: &CronJobControllerOptions{ - &componentConfig.CronJobController, - }, - NamespaceController: &NamespaceControllerOptions{ - &componentConfig.NamespaceController, - }, - NodeIPAMController: &NodeIPAMControllerOptions{ - &componentConfig.NodeIPAMController, - }, - NodeLifecycleController: &NodeLifecycleControllerOptions{ - &componentConfig.NodeLifecycleController, - }, - PersistentVolumeBinderController: &PersistentVolumeBinderControllerOptions{ - &componentConfig.PersistentVolumeBinderController, - }, - PodGCController: &PodGCControllerOptions{ - &componentConfig.PodGCController, - }, - ReplicaSetController: &ReplicaSetControllerOptions{ - &componentConfig.ReplicaSetController, - }, - ReplicationController: &ReplicationControllerOptions{ - &componentConfig.ReplicationController, - }, - ResourceQuotaController: &ResourceQuotaControllerOptions{ - &componentConfig.ResourceQuotaController, - }, - SAController: &SAControllerOptions{ - &componentConfig.SAController, - }, - TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{ - &componentConfig.TTLAfterFinishedController, - }, - SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), - InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindAddress: net.ParseIP(componentConfig.Generic.Address), - BindPort: int(componentConfig.Generic.Port), - BindNetwork: "tcp", - }).WithLoopback(), - Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), - Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), - } - - s.Authentication.RemoteKubeConfigFileOptional = true - s.Authorization.RemoteKubeConfigFileOptional = true - s.Authorization.AlwaysAllowPaths = []string{"/healthz"} - - // Set the PairName but leave certificate directory blank to generate in-memory by default - s.SecureServing.ServerCert.CertDirectory = "" - s.SecureServing.ServerCert.PairName = "kube-controller-manager" - s.SecureServing.BindPort = ports.KubeControllerManagerPort - - gcIgnoredResources := make([]garbagecollectorconfig.GroupResource, 0, len(garbagecollector.DefaultIgnoredResources())) - for r := range garbagecollector.DefaultIgnoredResources() { - gcIgnoredResources = append(gcIgnoredResources, garbagecollectorconfig.GroupResource{Group: r.Group, Resource: r.Resource}) - } - - s.GarbageCollectorController.GCIgnoredResources = gcIgnoredResources - s.Generic.LeaderElection.ResourceName = "kube-controller-manager" - s.Generic.LeaderElection.ResourceNamespace = "kube-system" - - return &s, nil -} - -// NewDefaultComponentConfig returns kube-controller manager configuration object. -func NewDefaultComponentConfig(insecurePort int32) (kubectrlmgrconfig.KubeControllerManagerConfiguration, error) { - versioned := kubectrlmgrconfigv1alpha1.KubeControllerManagerConfiguration{} - kubectrlmgrconfigscheme.Scheme.Default(&versioned) - - internal := kubectrlmgrconfig.KubeControllerManagerConfiguration{} - if err := kubectrlmgrconfigscheme.Scheme.Convert(&versioned, &internal, nil); err != nil { - return internal, err - } - internal.Generic.Port = insecurePort - return internal, nil -} - -// Flags returns flags for a specific APIServer by section name -func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledByDefaultControllers []string) cliflag.NamedFlagSets { - fss := cliflag.NamedFlagSets{} - s.Generic.AddFlags(&fss, allControllers, disabledByDefaultControllers) - s.KubeCloudShared.AddFlags(fss.FlagSet("generic")) - s.ServiceController.AddFlags(fss.FlagSet("service controller")) - - s.SecureServing.AddFlags(fss.FlagSet("secure serving")) - s.InsecureServing.AddUnqualifiedFlags(fss.FlagSet("insecure serving")) - s.Authentication.AddFlags(fss.FlagSet("authentication")) - s.Authorization.AddFlags(fss.FlagSet("authorization")) - - s.AttachDetachController.AddFlags(fss.FlagSet("attachdetach controller")) - s.CSRSigningController.AddFlags(fss.FlagSet("csrsigning controller")) - s.DeploymentController.AddFlags(fss.FlagSet("deployment controller")) - s.StatefulSetController.AddFlags(fss.FlagSet("statefulset controller")) - s.DaemonSetController.AddFlags(fss.FlagSet("daemonset controller")) - s.DeprecatedFlags.AddFlags(fss.FlagSet("deprecated")) - s.EndpointController.AddFlags(fss.FlagSet("endpoint controller")) - s.EndpointSliceController.AddFlags(fss.FlagSet("endpointslice controller")) - s.EndpointSliceMirroringController.AddFlags(fss.FlagSet("endpointslicemirroring controller")) - s.GarbageCollectorController.AddFlags(fss.FlagSet("garbagecollector controller")) - s.HPAController.AddFlags(fss.FlagSet("horizontalpodautoscaling controller")) - s.JobController.AddFlags(fss.FlagSet("job controller")) - s.CronJobController.AddFlags(fss.FlagSet("cronjob controller")) - s.NamespaceController.AddFlags(fss.FlagSet("namespace controller")) - s.NodeIPAMController.AddFlags(fss.FlagSet("nodeipam controller")) - s.NodeLifecycleController.AddFlags(fss.FlagSet("nodelifecycle controller")) - s.PersistentVolumeBinderController.AddFlags(fss.FlagSet("persistentvolume-binder controller")) - s.PodGCController.AddFlags(fss.FlagSet("podgc controller")) - s.ReplicaSetController.AddFlags(fss.FlagSet("replicaset controller")) - s.ReplicationController.AddFlags(fss.FlagSet("replicationcontroller")) - s.ResourceQuotaController.AddFlags(fss.FlagSet("resourcequota controller")) - s.SAController.AddFlags(fss.FlagSet("serviceaccount controller")) - s.TTLAfterFinishedController.AddFlags(fss.FlagSet("ttl-after-finished controller")) - s.Metrics.AddFlags(fss.FlagSet("metrics")) - s.Logs.AddFlags(fss.FlagSet("logs")) - - fs := fss.FlagSet("misc") - fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).") - fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.") - utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic")) - - return fss -} - -// ApplyTo fills up controller manager config with options. -func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) error { - if err := s.Generic.ApplyTo(&c.ComponentConfig.Generic); err != nil { - return err - } - if err := s.KubeCloudShared.ApplyTo(&c.ComponentConfig.KubeCloudShared); err != nil { - return err - } - if err := s.AttachDetachController.ApplyTo(&c.ComponentConfig.AttachDetachController); err != nil { - return err - } - if err := s.CSRSigningController.ApplyTo(&c.ComponentConfig.CSRSigningController); err != nil { - return err - } - if err := s.DaemonSetController.ApplyTo(&c.ComponentConfig.DaemonSetController); err != nil { - return err - } - if err := s.DeploymentController.ApplyTo(&c.ComponentConfig.DeploymentController); err != nil { - return err - } - if err := s.StatefulSetController.ApplyTo(&c.ComponentConfig.StatefulSetController); err != nil { - return err - } - if err := s.DeprecatedFlags.ApplyTo(&c.ComponentConfig.DeprecatedController); err != nil { - return err - } - if err := s.EndpointController.ApplyTo(&c.ComponentConfig.EndpointController); err != nil { - return err - } - if err := s.EndpointSliceController.ApplyTo(&c.ComponentConfig.EndpointSliceController); err != nil { - return err - } - if err := s.EndpointSliceMirroringController.ApplyTo(&c.ComponentConfig.EndpointSliceMirroringController); err != nil { - return err - } - if err := s.GarbageCollectorController.ApplyTo(&c.ComponentConfig.GarbageCollectorController); err != nil { - return err - } - if err := s.HPAController.ApplyTo(&c.ComponentConfig.HPAController); err != nil { - return err - } - if err := s.JobController.ApplyTo(&c.ComponentConfig.JobController); err != nil { - return err - } - if err := s.CronJobController.ApplyTo(&c.ComponentConfig.CronJobController); err != nil { - return err - } - if err := s.NamespaceController.ApplyTo(&c.ComponentConfig.NamespaceController); err != nil { - return err - } - if err := s.NodeIPAMController.ApplyTo(&c.ComponentConfig.NodeIPAMController); err != nil { - return err - } - if err := s.NodeLifecycleController.ApplyTo(&c.ComponentConfig.NodeLifecycleController); err != nil { - return err - } - if err := s.PersistentVolumeBinderController.ApplyTo(&c.ComponentConfig.PersistentVolumeBinderController); err != nil { - return err - } - if err := s.PodGCController.ApplyTo(&c.ComponentConfig.PodGCController); err != nil { - return err - } - if err := s.ReplicaSetController.ApplyTo(&c.ComponentConfig.ReplicaSetController); err != nil { - return err - } - if err := s.ReplicationController.ApplyTo(&c.ComponentConfig.ReplicationController); err != nil { - return err - } - if err := s.ResourceQuotaController.ApplyTo(&c.ComponentConfig.ResourceQuotaController); err != nil { - return err - } - if err := s.SAController.ApplyTo(&c.ComponentConfig.SAController); err != nil { - return err - } - if err := s.ServiceController.ApplyTo(&c.ComponentConfig.ServiceController); err != nil { - return err - } - if err := s.TTLAfterFinishedController.ApplyTo(&c.ComponentConfig.TTLAfterFinishedController); err != nil { - return err - } - if err := s.InsecureServing.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { - return err - } - if err := s.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { - return err - } - if s.SecureServing.BindPort != 0 || s.SecureServing.Listener != nil { - if err := s.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { - return err - } - if err := s.Authorization.ApplyTo(&c.Authorization); err != nil { - return err - } - } - - // sync back to component config - // TODO: find more elegant way than syncing back the values. - c.ComponentConfig.Generic.Port = int32(s.InsecureServing.BindPort) - c.ComponentConfig.Generic.Address = s.InsecureServing.BindAddress.String() - - return nil -} - -// Validate is used to validate the options and config before launching the controller manager -func (s *KubeControllerManagerOptions) Validate(allControllers []string, disabledByDefaultControllers []string) error { - var errs []error - - errs = append(errs, s.Generic.Validate(allControllers, disabledByDefaultControllers)...) - errs = append(errs, s.KubeCloudShared.Validate()...) - errs = append(errs, s.AttachDetachController.Validate()...) - errs = append(errs, s.CSRSigningController.Validate()...) - errs = append(errs, s.DaemonSetController.Validate()...) - errs = append(errs, s.DeploymentController.Validate()...) - errs = append(errs, s.StatefulSetController.Validate()...) - errs = append(errs, s.DeprecatedFlags.Validate()...) - errs = append(errs, s.EndpointController.Validate()...) - errs = append(errs, s.EndpointSliceController.Validate()...) - errs = append(errs, s.EndpointSliceMirroringController.Validate()...) - errs = append(errs, s.GarbageCollectorController.Validate()...) - errs = append(errs, s.HPAController.Validate()...) - errs = append(errs, s.JobController.Validate()...) - errs = append(errs, s.CronJobController.Validate()...) - errs = append(errs, s.NamespaceController.Validate()...) - errs = append(errs, s.NodeIPAMController.Validate()...) - errs = append(errs, s.NodeLifecycleController.Validate()...) - errs = append(errs, s.PersistentVolumeBinderController.Validate()...) - errs = append(errs, s.PodGCController.Validate()...) - errs = append(errs, s.ReplicaSetController.Validate()...) - errs = append(errs, s.ReplicationController.Validate()...) - errs = append(errs, s.ResourceQuotaController.Validate()...) - errs = append(errs, s.SAController.Validate()...) - errs = append(errs, s.ServiceController.Validate()...) - errs = append(errs, s.TTLAfterFinishedController.Validate()...) - errs = append(errs, s.SecureServing.Validate()...) - errs = append(errs, s.InsecureServing.Validate()...) - errs = append(errs, s.Authentication.Validate()...) - errs = append(errs, s.Authorization.Validate()...) - errs = append(errs, s.Metrics.Validate()...) - errs = append(errs, s.Logs.Validate()...) - - // TODO: validate component config, master and kubeconfig - - return utilerrors.NewAggregate(errs) -} - -// Config return a controller manager config objective -func (s KubeControllerManagerOptions) Config(allControllers []string, disabledByDefaultControllers []string) (*kubecontrollerconfig.Config, error) { - if err := s.Validate(allControllers, disabledByDefaultControllers); err != nil { - return nil, err - } - - if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { - return nil, fmt.Errorf("error creating self-signed certificates: %v", err) - } - - kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) - if err != nil { - return nil, err - } - kubeconfig.DisableCompression = true - kubeconfig.ContentConfig.AcceptContentTypes = s.Generic.ClientConnection.AcceptContentTypes - kubeconfig.ContentConfig.ContentType = s.Generic.ClientConnection.ContentType - kubeconfig.QPS = s.Generic.ClientConnection.QPS - kubeconfig.Burst = int(s.Generic.ClientConnection.Burst) - - client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, KubeControllerManagerUserAgent)) - if err != nil { - return nil, err - } - - // shallow copy, do not modify the kubeconfig.Timeout. - config := *kubeconfig - config.Timeout = s.Generic.LeaderElection.RenewDeadline.Duration - leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "leader-election")) - - eventRecorder := createRecorder(client, KubeControllerManagerUserAgent) - - c := &kubecontrollerconfig.Config{ - Client: client, - Kubeconfig: kubeconfig, - EventRecorder: eventRecorder, - LeaderElectionClient: leaderElectionClient, - } - if err := s.ApplyTo(c); err != nil { - return nil, err - } - s.Metrics.Apply() - - s.Logs.Apply() - - return c, nil -} - -func createRecorder(kubeClient clientset.Interface, userAgent string) record.EventRecorder { - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartStructuredLogging(0) - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) - return eventBroadcaster.NewRecorder(clientgokubescheme.Scheme, v1.EventSource{Component: userAgent}) -} diff --git a/cmd/kube-controller-manager/app/options/options_test.go b/cmd/kube-controller-manager/app/options/options_test.go deleted file mode 100644 index a04f30853a524..0000000000000 --- a/cmd/kube-controller-manager/app/options/options_test.go +++ /dev/null @@ -1,666 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "net" - "reflect" - "sort" - "testing" - "time" - - "github.com/spf13/pflag" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/diff" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - cpconfig "k8s.io/cloud-provider/config" - serviceconfig "k8s.io/cloud-provider/controllers/service/config" - cpoptions "k8s.io/cloud-provider/options" - componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - cmconfig "k8s.io/controller-manager/config" - cmoptions "k8s.io/controller-manager/options" - kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" - kubectrlmgrconfig "k8s.io/kubernetes/pkg/controller/apis/config" - csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config" - cronjobconfig "k8s.io/kubernetes/pkg/controller/cronjob/config" - daemonconfig "k8s.io/kubernetes/pkg/controller/daemon/config" - deploymentconfig "k8s.io/kubernetes/pkg/controller/deployment/config" - endpointconfig "k8s.io/kubernetes/pkg/controller/endpoint/config" - endpointsliceconfig "k8s.io/kubernetes/pkg/controller/endpointslice/config" - endpointslicemirroringconfig "k8s.io/kubernetes/pkg/controller/endpointslicemirroring/config" - garbagecollectorconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config" - jobconfig "k8s.io/kubernetes/pkg/controller/job/config" - namespaceconfig "k8s.io/kubernetes/pkg/controller/namespace/config" - nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config" - nodelifecycleconfig "k8s.io/kubernetes/pkg/controller/nodelifecycle/config" - poautosclerconfig "k8s.io/kubernetes/pkg/controller/podautoscaler/config" - podgcconfig "k8s.io/kubernetes/pkg/controller/podgc/config" - replicasetconfig "k8s.io/kubernetes/pkg/controller/replicaset/config" - replicationconfig "k8s.io/kubernetes/pkg/controller/replication/config" - resourcequotaconfig "k8s.io/kubernetes/pkg/controller/resourcequota/config" - serviceaccountconfig "k8s.io/kubernetes/pkg/controller/serviceaccount/config" - statefulsetconfig "k8s.io/kubernetes/pkg/controller/statefulset/config" - ttlafterfinishedconfig "k8s.io/kubernetes/pkg/controller/ttlafterfinished/config" - attachdetachconfig "k8s.io/kubernetes/pkg/controller/volume/attachdetach/config" - persistentvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/config" -) - -var args = []string{ - "--address=192.168.4.10", - "--allocate-node-cidrs=true", - "--attach-detach-reconcile-sync-period=30s", - "--cidr-allocator-type=CloudAllocator", - "--cloud-config=/cloud-config", - "--cloud-provider=gce", - "--cluster-cidr=1.2.3.4/24", - "--cluster-name=k8s", - "--cluster-signing-cert-file=/cluster-signing-cert", - "--cluster-signing-key-file=/cluster-signing-key", - "--cluster-signing-kubelet-serving-cert-file=/cluster-signing-kubelet-serving/cert-file", - "--cluster-signing-kubelet-serving-key-file=/cluster-signing-kubelet-serving/key-file", - "--cluster-signing-kubelet-client-cert-file=/cluster-signing-kubelet-client/cert-file", - "--cluster-signing-kubelet-client-key-file=/cluster-signing-kubelet-client/key-file", - "--cluster-signing-kube-apiserver-client-cert-file=/cluster-signing-kube-apiserver-client/cert-file", - "--cluster-signing-kube-apiserver-client-key-file=/cluster-signing-kube-apiserver-client/key-file", - "--cluster-signing-legacy-unknown-cert-file=/cluster-signing-legacy-unknown/cert-file", - "--cluster-signing-legacy-unknown-key-file=/cluster-signing-legacy-unknown/key-file", - "--concurrent-deployment-syncs=10", - "--concurrent-statefulset-syncs=15", - "--concurrent-endpoint-syncs=10", - "--concurrent-service-endpoint-syncs=10", - "--concurrent-gc-syncs=30", - "--concurrent-namespace-syncs=20", - "--concurrent-replicaset-syncs=10", - "--concurrent-resource-quota-syncs=10", - "--concurrent-service-syncs=2", - "--concurrent-serviceaccount-token-syncs=10", - "--concurrent_rc_syncs=10", - "--configure-cloud-routes=false", - "--contention-profiling=true", - "--controller-start-interval=2m", - "--controllers=foo,bar", - "--deployment-controller-sync-period=45s", - "--disable-attach-detach-reconcile-sync=true", - "--enable-dynamic-provisioning=false", - "--enable-garbage-collector=false", - "--enable-hostpath-provisioner=true", - "--enable-taint-manager=false", - "--cluster-signing-duration=10h", - "--flex-volume-plugin-dir=/flex-volume-plugin", - "--volume-host-cidr-denylist=127.0.0.1/28,feed::/16", - "--volume-host-allow-local-loopback=false", - "--horizontal-pod-autoscaler-downscale-delay=2m", - "--horizontal-pod-autoscaler-sync-period=45s", - "--horizontal-pod-autoscaler-upscale-delay=1m", - "--horizontal-pod-autoscaler-downscale-stabilization=3m", - "--horizontal-pod-autoscaler-cpu-initialization-period=90s", - "--horizontal-pod-autoscaler-initial-readiness-delay=50s", - "--http2-max-streams-per-connection=47", - "--kube-api-burst=100", - "--kube-api-content-type=application/json", - "--kube-api-qps=50.0", - "--kubeconfig=/kubeconfig", - "--large-cluster-size-threshold=100", - "--leader-elect=false", - "--leader-elect-lease-duration=30s", - "--leader-elect-renew-deadline=15s", - "--leader-elect-resource-lock=configmap", - "--leader-elect-retry-period=5s", - "--master=192.168.4.20", - "--max-endpoints-per-slice=200", - "--min-resync-period=8h", - "--mirroring-concurrent-service-endpoint-syncs=2", - "--mirroring-max-endpoints-per-subset=1000", - "--namespace-sync-period=10m", - "--node-cidr-mask-size=48", - "--node-cidr-mask-size-ipv4=48", - "--node-cidr-mask-size-ipv6=108", - "--node-eviction-rate=0.2", - "--node-monitor-grace-period=30s", - "--node-monitor-period=10s", - "--node-startup-grace-period=30s", - "--pod-eviction-timeout=2m", - "--port=10000", - "--profiling=false", - "--pv-recycler-increment-timeout-nfs=45", - "--pv-recycler-minimum-timeout-hostpath=45", - "--pv-recycler-minimum-timeout-nfs=200", - "--pv-recycler-timeout-increment-hostpath=45", - "--pvclaimbinder-sync-period=30s", - "--resource-quota-sync-period=10m", - "--route-reconciliation-period=30s", - "--secondary-node-eviction-rate=0.05", - "--service-account-private-key-file=/service-account-private-key", - "--terminated-pod-gc-threshold=12000", - "--unhealthy-zone-threshold=0.6", - "--use-service-account-credentials=true", - "--cert-dir=/a/b/c", - "--bind-address=192.168.4.21", - "--secure-port=10001", - "--concurrent-ttl-after-finished-syncs=8", -} - -func TestAddFlags(t *testing.T) { - fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) - s, _ := NewKubeControllerManagerOptions() - for _, f := range s.Flags([]string{""}, []string{""}).FlagSets { - fs.AddFlagSet(f) - } - - fs.Parse(args) - // Sort GCIgnoredResources because it's built from a map, which means the - // insertion order is random. - sort.Sort(sortedGCIgnoredResources(s.GarbageCollectorController.GCIgnoredResources)) - - expected := &KubeControllerManagerOptions{ - Generic: &cmoptions.GenericControllerManagerConfigurationOptions{ - GenericControllerManagerConfiguration: &cmconfig.GenericControllerManagerConfiguration{ - Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - ContentType: "application/json", - QPS: 50.0, - Burst: 100, - }, - ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - ResourceLock: "configmap", - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, - ResourceName: "kube-controller-manager", - ResourceNamespace: "kube-system", - }, - Controllers: []string{"foo", "bar"}, - }, - Debugging: &cmoptions.DebuggingOptions{ - DebuggingConfiguration: &componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: false, - EnableContentionProfiling: true, - }, - }, - }, - KubeCloudShared: &cpoptions.KubeCloudSharedOptions{ - KubeCloudSharedConfiguration: &cpconfig.KubeCloudSharedConfiguration{ - UseServiceAccountCredentials: true, - RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, - ClusterName: "k8s", - ClusterCIDR: "1.2.3.4/24", - AllocateNodeCIDRs: true, - CIDRAllocatorType: "CloudAllocator", - ConfigureCloudRoutes: false, - }, - CloudProvider: &cpoptions.CloudProviderOptions{ - CloudProviderConfiguration: &cpconfig.CloudProviderConfiguration{ - Name: "gce", - CloudConfigFile: "/cloud-config", - }, - }, - }, - ServiceController: &cpoptions.ServiceControllerOptions{ - ServiceControllerConfiguration: &serviceconfig.ServiceControllerConfiguration{ - ConcurrentServiceSyncs: 2, - }, - }, - AttachDetachController: &AttachDetachControllerOptions{ - &attachdetachconfig.AttachDetachControllerConfiguration{ - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, - DisableAttachDetachReconcilerSync: true, - }, - }, - CSRSigningController: &CSRSigningControllerOptions{ - &csrsigningconfig.CSRSigningControllerConfiguration{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - KubeletServingSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-serving/cert-file", - KeyFile: "/cluster-signing-kubelet-serving/key-file", - }, - KubeletClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-client/cert-file", - KeyFile: "/cluster-signing-kubelet-client/key-file", - }, - KubeAPIServerClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kube-apiserver-client/cert-file", - KeyFile: "/cluster-signing-kube-apiserver-client/key-file", - }, - LegacyUnknownSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-legacy-unknown/cert-file", - KeyFile: "/cluster-signing-legacy-unknown/key-file", - }, - }, - }, - DaemonSetController: &DaemonSetControllerOptions{ - &daemonconfig.DaemonSetControllerConfiguration{ - ConcurrentDaemonSetSyncs: 2, - }, - }, - DeploymentController: &DeploymentControllerOptions{ - &deploymentconfig.DeploymentControllerConfiguration{ - ConcurrentDeploymentSyncs: 10, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - }, - }, - StatefulSetController: &StatefulSetControllerOptions{ - &statefulsetconfig.StatefulSetControllerConfiguration{ - ConcurrentStatefulSetSyncs: 15, - }, - }, - DeprecatedFlags: &DeprecatedControllerOptions{ - &kubectrlmgrconfig.DeprecatedControllerConfiguration{ - DeletingPodsQPS: 0.1, - RegisterRetryCount: 10, - }, - }, - EndpointController: &EndpointControllerOptions{ - &endpointconfig.EndpointControllerConfiguration{ - ConcurrentEndpointSyncs: 10, - }, - }, - EndpointSliceController: &EndpointSliceControllerOptions{ - &endpointsliceconfig.EndpointSliceControllerConfiguration{ - ConcurrentServiceEndpointSyncs: 10, - MaxEndpointsPerSlice: 200, - }, - }, - EndpointSliceMirroringController: &EndpointSliceMirroringControllerOptions{ - &endpointslicemirroringconfig.EndpointSliceMirroringControllerConfiguration{ - MirroringConcurrentServiceEndpointSyncs: 2, - MirroringMaxEndpointsPerSubset: 1000, - }, - }, - GarbageCollectorController: &GarbageCollectorControllerOptions{ - &garbagecollectorconfig.GarbageCollectorControllerConfiguration{ - ConcurrentGCSyncs: 30, - GCIgnoredResources: []garbagecollectorconfig.GroupResource{ - {Group: "", Resource: "events"}, - }, - EnableGarbageCollector: false, - }, - }, - HPAController: &HPAControllerOptions{ - &poautosclerconfig.HPAControllerConfiguration{ - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, - HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute}, - HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second}, - HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second}, - HorizontalPodAutoscalerTolerance: 0.1, - HorizontalPodAutoscalerUseRESTClients: true, - }, - }, - JobController: &JobControllerOptions{ - &jobconfig.JobControllerConfiguration{ - ConcurrentJobSyncs: 5, - }, - }, - CronJobController: &CronJobControllerOptions{ - &cronjobconfig.CronJobControllerConfiguration{ - ConcurrentCronJobSyncs: 5, - }, - }, - NamespaceController: &NamespaceControllerOptions{ - &namespaceconfig.NamespaceControllerConfiguration{ - NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - ConcurrentNamespaceSyncs: 20, - }, - }, - NodeIPAMController: &NodeIPAMControllerOptions{ - &nodeipamconfig.NodeIPAMControllerConfiguration{ - NodeCIDRMaskSize: 48, - NodeCIDRMaskSizeIPv4: 48, - NodeCIDRMaskSizeIPv6: 108, - }, - }, - NodeLifecycleController: &NodeLifecycleControllerOptions{ - &nodelifecycleconfig.NodeLifecycleControllerConfiguration{ - EnableTaintManager: false, - NodeEvictionRate: 0.2, - SecondaryNodeEvictionRate: 0.05, - NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, - LargeClusterSizeThreshold: 100, - UnhealthyZoneThreshold: 0.6, - }, - }, - PersistentVolumeBinderController: &PersistentVolumeBinderControllerOptions{ - &persistentvolumeconfig.PersistentVolumeBinderControllerConfiguration{ - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - VolumeConfiguration: persistentvolumeconfig.VolumeConfiguration{ - EnableDynamicProvisioning: false, - EnableHostPathProvisioning: true, - FlexVolumePluginDir: "/flex-volume-plugin", - PersistentVolumeRecyclerConfiguration: persistentvolumeconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 200, - IncrementTimeoutNFS: 45, - MinimumTimeoutHostPath: 45, - IncrementTimeoutHostPath: 45, - }, - }, - VolumeHostCIDRDenylist: []string{"127.0.0.1/28", "feed::/16"}, - VolumeHostAllowLocalLoopback: false, - }, - }, - PodGCController: &PodGCControllerOptions{ - &podgcconfig.PodGCControllerConfiguration{ - TerminatedPodGCThreshold: 12000, - }, - }, - ReplicaSetController: &ReplicaSetControllerOptions{ - &replicasetconfig.ReplicaSetControllerConfiguration{ - ConcurrentRSSyncs: 10, - }, - }, - ReplicationController: &ReplicationControllerOptions{ - &replicationconfig.ReplicationControllerConfiguration{ - ConcurrentRCSyncs: 10, - }, - }, - ResourceQuotaController: &ResourceQuotaControllerOptions{ - &resourcequotaconfig.ResourceQuotaControllerConfiguration{ - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - ConcurrentResourceQuotaSyncs: 10, - }, - }, - SAController: &SAControllerOptions{ - &serviceaccountconfig.SAControllerConfiguration{ - ServiceAccountKeyFile: "/service-account-private-key", - ConcurrentSATokenSyncs: 10, - }, - }, - TTLAfterFinishedController: &TTLAfterFinishedControllerOptions{ - &ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration{ - ConcurrentTTLSyncs: 8, - }, - }, - SecureServing: (&apiserveroptions.SecureServingOptions{ - BindPort: 10001, - BindAddress: net.ParseIP("192.168.4.21"), - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/a/b/c", - PairName: "kube-controller-manager", - }, - HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), - InsecureServing: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindAddress: net.ParseIP("192.168.4.10"), - BindPort: int(10000), - BindNetwork: "tcp", - }).WithLoopback(), - Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - ClientTimeout: 10 * time.Second, - WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), - ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, - RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ - UsernameHeaders: []string{"x-remote-user"}, - GroupHeaders: []string{"x-remote-group"}, - ExtraHeaderPrefixes: []string{"x-remote-extra-"}, - }, - RemoteKubeConfigFileOptional: true, - }, - Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ - AllowCacheTTL: 10 * time.Second, - DenyCacheTTL: 10 * time.Second, - ClientTimeout: 10 * time.Second, - WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), - RemoteKubeConfigFileOptional: true, - AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* - }, - Kubeconfig: "/kubeconfig", - Master: "192.168.4.20", - Metrics: &metrics.Options{}, - Logs: logs.NewOptions(), - } - - // Sort GCIgnoredResources because it's built from a map, which means the - // insertion order is random. - sort.Sort(sortedGCIgnoredResources(expected.GarbageCollectorController.GCIgnoredResources)) - - if !reflect.DeepEqual(expected, s) { - t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s)) - } -} - -func TestApplyTo(t *testing.T) { - fs := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError) - s, _ := NewKubeControllerManagerOptions() - // flag set to parse the args that are required to start the kube controller manager - for _, f := range s.Flags([]string{""}, []string{""}).FlagSets { - fs.AddFlagSet(f) - } - - fs.Parse(args) - // Sort GCIgnoredResources because it's built from a map, which means the - // insertion order is random. - sort.Sort(sortedGCIgnoredResources(s.GarbageCollectorController.GCIgnoredResources)) - - expected := &kubecontrollerconfig.Config{ - ComponentConfig: kubectrlmgrconfig.KubeControllerManagerConfiguration{ - Generic: cmconfig.GenericControllerManagerConfiguration{ - Port: 10252, // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - Address: "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config - MinResyncPeriod: metav1.Duration{Duration: 8 * time.Hour}, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - ContentType: "application/json", - QPS: 50.0, - Burst: 100, - }, - ControllerStartInterval: metav1.Duration{Duration: 2 * time.Minute}, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - ResourceLock: "configmap", - LeaderElect: false, - LeaseDuration: metav1.Duration{Duration: 30 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 15 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 5 * time.Second}, - ResourceName: "kube-controller-manager", - ResourceNamespace: "kube-system", - }, - Controllers: []string{"foo", "bar"}, - Debugging: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: false, - EnableContentionProfiling: true, - }, - }, - KubeCloudShared: cpconfig.KubeCloudSharedConfiguration{ - UseServiceAccountCredentials: true, - RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeMonitorPeriod: metav1.Duration{Duration: 10 * time.Second}, - ClusterName: "k8s", - ClusterCIDR: "1.2.3.4/24", - AllocateNodeCIDRs: true, - CIDRAllocatorType: "CloudAllocator", - ConfigureCloudRoutes: false, - CloudProvider: cpconfig.CloudProviderConfiguration{ - Name: "gce", - CloudConfigFile: "/cloud-config", - }, - }, - ServiceController: serviceconfig.ServiceControllerConfiguration{ - ConcurrentServiceSyncs: 2, - }, - AttachDetachController: attachdetachconfig.AttachDetachControllerConfiguration{ - ReconcilerSyncLoopPeriod: metav1.Duration{Duration: 30 * time.Second}, - DisableAttachDetachReconcilerSync: true, - }, - CSRSigningController: csrsigningconfig.CSRSigningControllerConfiguration{ - ClusterSigningCertFile: "/cluster-signing-cert", - ClusterSigningKeyFile: "/cluster-signing-key", - ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour}, - KubeletServingSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-serving/cert-file", - KeyFile: "/cluster-signing-kubelet-serving/key-file", - }, - KubeletClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kubelet-client/cert-file", - KeyFile: "/cluster-signing-kubelet-client/key-file", - }, - KubeAPIServerClientSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-kube-apiserver-client/cert-file", - KeyFile: "/cluster-signing-kube-apiserver-client/key-file", - }, - LegacyUnknownSignerConfiguration: csrsigningconfig.CSRSigningConfiguration{ - CertFile: "/cluster-signing-legacy-unknown/cert-file", - KeyFile: "/cluster-signing-legacy-unknown/key-file", - }, - }, - DaemonSetController: daemonconfig.DaemonSetControllerConfiguration{ - ConcurrentDaemonSetSyncs: 2, - }, - DeploymentController: deploymentconfig.DeploymentControllerConfiguration{ - ConcurrentDeploymentSyncs: 10, - DeploymentControllerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - }, - StatefulSetController: statefulsetconfig.StatefulSetControllerConfiguration{ - ConcurrentStatefulSetSyncs: 15, - }, - DeprecatedController: kubectrlmgrconfig.DeprecatedControllerConfiguration{ - DeletingPodsQPS: 0.1, - RegisterRetryCount: 10, - }, - EndpointController: endpointconfig.EndpointControllerConfiguration{ - ConcurrentEndpointSyncs: 10, - }, - EndpointSliceController: endpointsliceconfig.EndpointSliceControllerConfiguration{ - ConcurrentServiceEndpointSyncs: 10, - MaxEndpointsPerSlice: 200, - }, - EndpointSliceMirroringController: endpointslicemirroringconfig.EndpointSliceMirroringControllerConfiguration{ - MirroringConcurrentServiceEndpointSyncs: 2, - MirroringMaxEndpointsPerSubset: 1000, - }, - GarbageCollectorController: garbagecollectorconfig.GarbageCollectorControllerConfiguration{ - ConcurrentGCSyncs: 30, - GCIgnoredResources: []garbagecollectorconfig.GroupResource{ - {Group: "", Resource: "events"}, - }, - EnableGarbageCollector: false, - }, - HPAController: poautosclerconfig.HPAControllerConfiguration{ - HorizontalPodAutoscalerSyncPeriod: metav1.Duration{Duration: 45 * time.Second}, - HorizontalPodAutoscalerUpscaleForbiddenWindow: metav1.Duration{Duration: 1 * time.Minute}, - HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute}, - HorizontalPodAutoscalerDownscaleStabilizationWindow: metav1.Duration{Duration: 3 * time.Minute}, - HorizontalPodAutoscalerCPUInitializationPeriod: metav1.Duration{Duration: 90 * time.Second}, - HorizontalPodAutoscalerInitialReadinessDelay: metav1.Duration{Duration: 50 * time.Second}, - HorizontalPodAutoscalerTolerance: 0.1, - HorizontalPodAutoscalerUseRESTClients: true, - }, - JobController: jobconfig.JobControllerConfiguration{ - ConcurrentJobSyncs: 5, - }, - CronJobController: cronjobconfig.CronJobControllerConfiguration{ - ConcurrentCronJobSyncs: 5, - }, - NamespaceController: namespaceconfig.NamespaceControllerConfiguration{ - NamespaceSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - ConcurrentNamespaceSyncs: 20, - }, - NodeIPAMController: nodeipamconfig.NodeIPAMControllerConfiguration{ - NodeCIDRMaskSize: 48, - NodeCIDRMaskSizeIPv4: 48, - NodeCIDRMaskSizeIPv6: 108, - }, - NodeLifecycleController: nodelifecycleconfig.NodeLifecycleControllerConfiguration{ - EnableTaintManager: false, - NodeEvictionRate: 0.2, - SecondaryNodeEvictionRate: 0.05, - NodeMonitorGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - NodeStartupGracePeriod: metav1.Duration{Duration: 30 * time.Second}, - PodEvictionTimeout: metav1.Duration{Duration: 2 * time.Minute}, - LargeClusterSizeThreshold: 100, - UnhealthyZoneThreshold: 0.6, - }, - PersistentVolumeBinderController: persistentvolumeconfig.PersistentVolumeBinderControllerConfiguration{ - PVClaimBinderSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - VolumeConfiguration: persistentvolumeconfig.VolumeConfiguration{ - EnableDynamicProvisioning: false, - EnableHostPathProvisioning: true, - FlexVolumePluginDir: "/flex-volume-plugin", - PersistentVolumeRecyclerConfiguration: persistentvolumeconfig.PersistentVolumeRecyclerConfiguration{ - MaximumRetry: 3, - MinimumTimeoutNFS: 200, - IncrementTimeoutNFS: 45, - MinimumTimeoutHostPath: 45, - IncrementTimeoutHostPath: 45, - }, - }, - VolumeHostCIDRDenylist: []string{"127.0.0.1/28", "feed::/16"}, - VolumeHostAllowLocalLoopback: false, - }, - PodGCController: podgcconfig.PodGCControllerConfiguration{ - TerminatedPodGCThreshold: 12000, - }, - ReplicaSetController: replicasetconfig.ReplicaSetControllerConfiguration{ - ConcurrentRSSyncs: 10, - }, - ReplicationController: replicationconfig.ReplicationControllerConfiguration{ - ConcurrentRCSyncs: 10, - }, - ResourceQuotaController: resourcequotaconfig.ResourceQuotaControllerConfiguration{ - ResourceQuotaSyncPeriod: metav1.Duration{Duration: 10 * time.Minute}, - ConcurrentResourceQuotaSyncs: 10, - }, - SAController: serviceaccountconfig.SAControllerConfiguration{ - ServiceAccountKeyFile: "/service-account-private-key", - ConcurrentSATokenSyncs: 10, - }, - TTLAfterFinishedController: ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration{ - ConcurrentTTLSyncs: 8, - }, - }, - } - - // Sort GCIgnoredResources because it's built from a map, which means the - // insertion order is random. - sort.Sort(sortedGCIgnoredResources(expected.ComponentConfig.GarbageCollectorController.GCIgnoredResources)) - - c := &kubecontrollerconfig.Config{} - s.ApplyTo(c) - - if !reflect.DeepEqual(expected.ComponentConfig, c.ComponentConfig) { - t.Errorf("Got different configuration than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected.ComponentConfig, c.ComponentConfig)) - } -} - -type sortedGCIgnoredResources []garbagecollectorconfig.GroupResource - -func (r sortedGCIgnoredResources) Len() int { - return len(r) -} - -func (r sortedGCIgnoredResources) Less(i, j int) bool { - if r[i].Group < r[j].Group { - return true - } else if r[i].Group > r[j].Group { - return false - } - return r[i].Resource < r[j].Resource -} - -func (r sortedGCIgnoredResources) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} diff --git a/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go b/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go deleted file mode 100644 index 4be5a29a9ca30..0000000000000 --- a/cmd/kube-controller-manager/app/options/persistentvolumebindercontroller.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - - "github.com/spf13/pflag" - - persistentvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/config" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" - netutils "k8s.io/utils/net" -) - -// PersistentVolumeBinderControllerOptions holds the PersistentVolumeBinderController options. -type PersistentVolumeBinderControllerOptions struct { - *persistentvolumeconfig.PersistentVolumeBinderControllerConfiguration -} - -// AddFlags adds flags related to PersistentVolumeBinderController for controller manager to the specified FlagSet. -func (o *PersistentVolumeBinderControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.DurationVar(&o.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", o.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims") - fs.StringVar(&o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling") - fs.Int32Var(&o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod") - fs.Int32Var(&o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod") - fs.StringVar(&o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "pv-recycler-pod-template-filepath-hostpath", o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "The file path to a pod definition used as a template for HostPath persistent volume recycling. This is for development and testing only and will not work in a multi-node cluster.") - fs.Int32Var(&o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "pv-recycler-minimum-timeout-hostpath", o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod. This is for development and testing only and will not work in a multi-node cluster.") - fs.Int32Var(&o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "pv-recycler-timeout-increment-hostpath", o.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod. This is for development and testing only and will not work in a multi-node cluster.") - fs.BoolVar(&o.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", o.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features. HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.") - fs.BoolVar(&o.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", o.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.") - fs.StringVar(&o.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", o.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.") - fs.StringSliceVar(&o.VolumeHostCIDRDenylist, "volume-host-cidr-denylist", o.VolumeHostCIDRDenylist, "A comma-separated list of CIDR ranges to avoid from volume plugins.") - fs.BoolVar(&o.VolumeHostAllowLocalLoopback, "volume-host-allow-local-loopback", o.VolumeHostAllowLocalLoopback, "If false, deny local loopback IPs in addition to any CIDR ranges in --volume-host-cidr-denylist") -} - -// ApplyTo fills up PersistentVolumeBinderController config with options. -func (o *PersistentVolumeBinderControllerOptions) ApplyTo(cfg *persistentvolumeconfig.PersistentVolumeBinderControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.PVClaimBinderSyncPeriod = o.PVClaimBinderSyncPeriod - cfg.VolumeConfiguration = o.VolumeConfiguration - cfg.VolumeHostCIDRDenylist = o.VolumeHostCIDRDenylist - cfg.VolumeHostAllowLocalLoopback = o.VolumeHostAllowLocalLoopback - - return nil -} - -// Validate checks validation of PersistentVolumeBinderControllerOptions. -func (o *PersistentVolumeBinderControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - if _, err := ParseVolumeHostFilters(o.VolumeHostCIDRDenylist, o.VolumeHostAllowLocalLoopback); err != nil { - errs = append(errs, fmt.Errorf("Bad --volume-host-ip-denylist/--volume-host-allow-local-loopback %w", err)) - } - return errs -} - -// ParseVolumeHostFilters process the --volume-host-ip-denylist and --volume-host-allow-local-loopback flags. -func ParseVolumeHostFilters(denylist []string, allowLocalLoopback bool) (*proxyutil.FilteredDialOptions, error) { - denyCIDRs, err := netutils.ParseCIDRs(denylist) - if err != nil { - return nil, err - } - return &proxyutil.FilteredDialOptions{DialHostCIDRDenylist: denyCIDRs, AllowLocalLoopback: allowLocalLoopback}, nil -} diff --git a/cmd/kube-controller-manager/app/options/podgccontroller.go b/cmd/kube-controller-manager/app/options/podgccontroller.go deleted file mode 100644 index c7d76a5e56573..0000000000000 --- a/cmd/kube-controller-manager/app/options/podgccontroller.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - podgcconfig "k8s.io/kubernetes/pkg/controller/podgc/config" -) - -// PodGCControllerOptions holds the PodGCController options. -type PodGCControllerOptions struct { - *podgcconfig.PodGCControllerConfiguration -} - -// AddFlags adds flags related to PodGCController for controller manager to the specified FlagSet. -func (o *PodGCControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", o.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.") -} - -// ApplyTo fills up PodGCController config with options. -func (o *PodGCControllerOptions) ApplyTo(cfg *podgcconfig.PodGCControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.TerminatedPodGCThreshold = o.TerminatedPodGCThreshold - - return nil -} - -// Validate checks validation of PodGCControllerOptions. -func (o *PodGCControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/replicasetcontroller.go b/cmd/kube-controller-manager/app/options/replicasetcontroller.go deleted file mode 100644 index 8b6c3be1051f4..0000000000000 --- a/cmd/kube-controller-manager/app/options/replicasetcontroller.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - replicasetconfig "k8s.io/kubernetes/pkg/controller/replicaset/config" -) - -// ReplicaSetControllerOptions holds the ReplicaSetController options. -type ReplicaSetControllerOptions struct { - *replicasetconfig.ReplicaSetControllerConfiguration -} - -// AddFlags adds flags related to ReplicaSetController for controller manager to the specified FlagSet. -func (o *ReplicaSetControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentRSSyncs, "concurrent-replicaset-syncs", o.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load") -} - -// ApplyTo fills up ReplicaSetController config with options. -func (o *ReplicaSetControllerOptions) ApplyTo(cfg *replicasetconfig.ReplicaSetControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentRSSyncs = o.ConcurrentRSSyncs - - return nil -} - -// Validate checks validation of ReplicaSetControllerOptions. -func (o *ReplicaSetControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/replicationcontroller.go b/cmd/kube-controller-manager/app/options/replicationcontroller.go deleted file mode 100644 index 492e0aa66efb2..0000000000000 --- a/cmd/kube-controller-manager/app/options/replicationcontroller.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - replicationconfig "k8s.io/kubernetes/pkg/controller/replication/config" -) - -// ReplicationControllerOptions holds the ReplicationController options. -type ReplicationControllerOptions struct { - *replicationconfig.ReplicationControllerConfiguration -} - -// AddFlags adds flags related to ReplicationController for controller manager to the specified FlagSet. -func (o *ReplicationControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentRCSyncs, "concurrent_rc_syncs", o.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load") -} - -// ApplyTo fills up ReplicationController config with options. -func (o *ReplicationControllerOptions) ApplyTo(cfg *replicationconfig.ReplicationControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentRCSyncs = o.ConcurrentRCSyncs - - return nil -} - -// Validate checks validation of ReplicationControllerOptions. -func (o *ReplicationControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/resourcequotacontroller.go b/cmd/kube-controller-manager/app/options/resourcequotacontroller.go deleted file mode 100644 index 4bf1084dcb760..0000000000000 --- a/cmd/kube-controller-manager/app/options/resourcequotacontroller.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - resourcequotaconfig "k8s.io/kubernetes/pkg/controller/resourcequota/config" -) - -// ResourceQuotaControllerOptions holds the ResourceQuotaController options. -type ResourceQuotaControllerOptions struct { - *resourcequotaconfig.ResourceQuotaControllerConfiguration -} - -// AddFlags adds flags related to ResourceQuotaController for controller manager to the specified FlagSet. -func (o *ResourceQuotaControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.DurationVar(&o.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", o.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system") - fs.Int32Var(&o.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", o.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load") -} - -// ApplyTo fills up ResourceQuotaController config with options. -func (o *ResourceQuotaControllerOptions) ApplyTo(cfg *resourcequotaconfig.ResourceQuotaControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ResourceQuotaSyncPeriod = o.ResourceQuotaSyncPeriod - cfg.ConcurrentResourceQuotaSyncs = o.ConcurrentResourceQuotaSyncs - - return nil -} - -// Validate checks validation of ResourceQuotaControllerOptions. -func (o *ResourceQuotaControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/serviceaccountcontroller.go b/cmd/kube-controller-manager/app/options/serviceaccountcontroller.go deleted file mode 100644 index 571838c5db53e..0000000000000 --- a/cmd/kube-controller-manager/app/options/serviceaccountcontroller.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - serviceaccountconfig "k8s.io/kubernetes/pkg/controller/serviceaccount/config" -) - -// SAControllerOptions holds the ServiceAccountController options. -type SAControllerOptions struct { - *serviceaccountconfig.SAControllerConfiguration -} - -// AddFlags adds flags related to ServiceAccountController for controller manager to the specified FlagSet -func (o *SAControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.StringVar(&o.ServiceAccountKeyFile, "service-account-private-key-file", o.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.") - fs.Int32Var(&o.ConcurrentSATokenSyncs, "concurrent-serviceaccount-token-syncs", o.ConcurrentSATokenSyncs, "The number of service account token objects that are allowed to sync concurrently. Larger number = more responsive token generation, but more CPU (and network) load") - fs.StringVar(&o.RootCAFile, "root-ca-file", o.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.") -} - -// ApplyTo fills up ServiceAccountController config with options. -func (o *SAControllerOptions) ApplyTo(cfg *serviceaccountconfig.SAControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ServiceAccountKeyFile = o.ServiceAccountKeyFile - cfg.ConcurrentSATokenSyncs = o.ConcurrentSATokenSyncs - cfg.RootCAFile = o.RootCAFile - - return nil -} - -// Validate checks validation of ServiceAccountControllerOptions. -func (o *SAControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/options/statefulsetcontroller.go b/cmd/kube-controller-manager/app/options/statefulsetcontroller.go deleted file mode 100644 index fedf06893f8c1..0000000000000 --- a/cmd/kube-controller-manager/app/options/statefulsetcontroller.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - - "github.com/spf13/pflag" - - statefulsetconfig "k8s.io/kubernetes/pkg/controller/statefulset/config" -) - -// StatefulSetControllerOptions holds the StatefulSetController options. -type StatefulSetControllerOptions struct { - *statefulsetconfig.StatefulSetControllerConfiguration -} - -// AddFlags adds flags related to StatefulSetController for controller manager to the specified FlagSet. -func (o *StatefulSetControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentStatefulSetSyncs, "concurrent-statefulset-syncs", o.ConcurrentStatefulSetSyncs, "The number of statefulset objects that are allowed to sync concurrently. Larger number = more responsive statefulsets, but more CPU (and network) load") -} - -// ApplyTo fills up StatefulSetController config with options. -func (o *StatefulSetControllerOptions) ApplyTo(cfg *statefulsetconfig.StatefulSetControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentStatefulSetSyncs = o.ConcurrentStatefulSetSyncs - - return nil -} - -// Validate checks validation of StatefulSetControllerOptions. -func (o *StatefulSetControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - if o.ConcurrentStatefulSetSyncs < 1 { - errs = append(errs, fmt.Errorf("concurrent-statefulset-syncs must be greater than 0, but got %d", o.ConcurrentStatefulSetSyncs)) - } - return errs -} diff --git a/cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go b/cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go deleted file mode 100644 index 749d562cc825a..0000000000000 --- a/cmd/kube-controller-manager/app/options/ttlafterfinishedcontroller.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" - - ttlafterfinishedconfig "k8s.io/kubernetes/pkg/controller/ttlafterfinished/config" -) - -// TTLAfterFinishedControllerOptions holds the TTLAfterFinishedController options. -type TTLAfterFinishedControllerOptions struct { - *ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration -} - -// AddFlags adds flags related to TTLAfterFinishedController for controller manager to the specified FlagSet. -func (o *TTLAfterFinishedControllerOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.Int32Var(&o.ConcurrentTTLSyncs, "concurrent-ttl-after-finished-syncs", o.ConcurrentTTLSyncs, "The number of TTL-after-finished controller workers that are allowed to sync concurrently.") -} - -// ApplyTo fills up TTLAfterFinishedController config with options. -func (o *TTLAfterFinishedControllerOptions) ApplyTo(cfg *ttlafterfinishedconfig.TTLAfterFinishedControllerConfiguration) error { - if o == nil { - return nil - } - - cfg.ConcurrentTTLSyncs = o.ConcurrentTTLSyncs - - return nil -} - -// Validate checks validation of TTLAfterFinishedControllerOptions. -func (o *TTLAfterFinishedControllerOptions) Validate() []error { - if o == nil { - return nil - } - - errs := []error{} - return errs -} diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go deleted file mode 100644 index f9556d696cebc..0000000000000 --- a/cmd/kube-controller-manager/app/plugins.go +++ /dev/null @@ -1,174 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - // This file exists to force the desired plugin implementations to be linked. - // This should probably be part of some configuration fed into the build for a - // given binary target. - - "fmt" - - "k8s.io/klog/v2" - - // Cloud providers - cloudprovider "k8s.io/cloud-provider" - // ensure the cloud providers are installed - _ "k8s.io/kubernetes/pkg/cloudprovider/providers" - // Volume plugins - "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/csi" - "k8s.io/kubernetes/pkg/volume/fc" - "k8s.io/kubernetes/pkg/volume/flexvolume" - "k8s.io/kubernetes/pkg/volume/flocker" - "k8s.io/kubernetes/pkg/volume/glusterfs" - "k8s.io/kubernetes/pkg/volume/hostpath" - "k8s.io/kubernetes/pkg/volume/iscsi" - "k8s.io/kubernetes/pkg/volume/local" - "k8s.io/kubernetes/pkg/volume/nfs" - "k8s.io/kubernetes/pkg/volume/portworx" - "k8s.io/kubernetes/pkg/volume/quobyte" - "k8s.io/kubernetes/pkg/volume/rbd" - "k8s.io/kubernetes/pkg/volume/scaleio" - "k8s.io/kubernetes/pkg/volume/storageos" - volumeutil "k8s.io/kubernetes/pkg/volume/util" - - utilfeature "k8s.io/apiserver/pkg/util/feature" - persistentvolumeconfig "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/config" - "k8s.io/kubernetes/pkg/features" - "k8s.io/utils/exec" -) - -// ProbeAttachableVolumePlugins collects all volume plugins for the attach/ -// detach controller. -// The list of plugins is manually compiled. This code and the plugin -// initialization code for kubelet really, really need a through refactor. -func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) { - var err error - allPlugins := []volume.VolumePlugin{} - allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate) - if err != nil { - return allPlugins, err - } - allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - return allPlugins, nil -} - -// GetDynamicPluginProber gets the probers of dynamically discoverable plugins -// for the attach/detach controller. -// Currently only Flexvolume plugins are dynamically discoverable. -func GetDynamicPluginProber(config persistentvolumeconfig.VolumeConfiguration) volume.DynamicPluginProber { - return flexvolume.GetDynamicPluginProber(config.FlexVolumePluginDir, exec.New() /*exec.Interface*/) -} - -// ProbeExpandableVolumePlugins returns volume plugins which are expandable -func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) { - var err error - allPlugins := []volume.VolumePlugin{} - allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate) - if err != nil { - return allPlugins, err - } - allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) - return allPlugins, nil -} - -// ProbeControllerVolumePlugins collects all persistent volume plugins into an -// easy to use list. Only volume plugins that implement any of -// provisioner/recycler/deleter interface should be returned. -func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) { - allPlugins := []volume.VolumePlugin{} - - // The list of plugins to probe is decided by this binary, not - // by dynamic linking or other "magic". Plugins will be analyzed and - // initialized later. - - // Each plugin can make use of VolumeConfig. The single arg to this func contains *all* enumerated - // options meant to configure volume plugins. From that single config, create an instance of volume.VolumeConfig - // for a specific plugin and pass that instance to the plugin's ProbeVolumePlugins(config) func. - - // HostPath recycling is for testing and development purposes only! - hostPathConfig := volume.VolumeConfig{ - RecyclerMinimumTimeout: int(config.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath), - RecyclerTimeoutIncrement: int(config.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath), - RecyclerPodTemplate: volume.NewPersistentVolumeRecyclerPodTemplate(), - ProvisioningEnabled: config.EnableHostPathProvisioning, - } - if err := AttemptToLoadRecycler(config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, &hostPathConfig); err != nil { - klog.Fatalf("Could not create hostpath recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, err) - } - allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(hostPathConfig)...) - - nfsConfig := volume.VolumeConfig{ - RecyclerMinimumTimeout: int(config.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS), - RecyclerTimeoutIncrement: int(config.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS), - RecyclerPodTemplate: volume.NewPersistentVolumeRecyclerPodTemplate(), - } - if err := AttemptToLoadRecycler(config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, &nfsConfig); err != nil { - klog.Fatalf("Could not create NFS recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, err) - } - allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...) - allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...) - // add rbd provisioner - allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...) - var err error - allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate) - if err != nil { - return allPlugins, err - } - - allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, local.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) - - if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - } - - return allPlugins, nil -} - -// AttemptToLoadRecycler tries decoding a pod from a filepath for use as a recycler for a volume. -// If successful, this method will set the recycler on the config. -// If unsuccessful, an error is returned. Function is exported for reuse downstream. -func AttemptToLoadRecycler(path string, config *volume.VolumeConfig) error { - if path != "" { - recyclerPod, err := volumeutil.LoadPodFromFile(path) - if err != nil { - return err - } - if err = volume.ValidateRecyclerPodTemplate(recyclerPod); err != nil { - return fmt.Errorf("pod specification (%v): %v", path, err) - } - config.RecyclerPodTemplate = recyclerPod - } - return nil -} diff --git a/cmd/kube-controller-manager/app/plugins_providerless.go b/cmd/kube-controller-manager/app/plugins_providerless.go deleted file mode 100644 index f0ded7da355a8..0000000000000 --- a/cmd/kube-controller-manager/app/plugins_providerless.go +++ /dev/null @@ -1,40 +0,0 @@ -// +build providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "k8s.io/component-base/featuregate" - - "k8s.io/kubernetes/pkg/volume" -) - -func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - // no-op when compiled without legacy cloud providers - return allPlugins, nil -} - -func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - // no-op when compiled without legacy cloud providers - return allPlugins, nil -} - -func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - // no-op when compiled without legacy cloud providers - return allPlugins, nil -} diff --git a/cmd/kube-controller-manager/app/plugins_providers.go b/cmd/kube-controller-manager/app/plugins_providers.go deleted file mode 100644 index 093e236dc23df..0000000000000 --- a/cmd/kube-controller-manager/app/plugins_providers.go +++ /dev/null @@ -1,100 +0,0 @@ -// +build !providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "k8s.io/component-base/featuregate" - "k8s.io/csi-translation-lib/plugins" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/awsebs" - "k8s.io/kubernetes/pkg/volume/azure_file" - "k8s.io/kubernetes/pkg/volume/azuredd" - "k8s.io/kubernetes/pkg/volume/cinder" - "k8s.io/kubernetes/pkg/volume/csimigration" - "k8s.io/kubernetes/pkg/volume/gcepd" - "k8s.io/kubernetes/pkg/volume/vsphere_volume" -) - -type probeFn func() []volume.VolumePlugin - -func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { - // Skip appending the in-tree plugin to the list of plugins to be probed/initialized - // if the CSIMigration feature flag and plugin specific feature flag indicating - // CSI migration is complete - err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) - if err != nil { - klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) - // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly - } - if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { - klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) - return plugins, nil - } - plugins = append(plugins, fn()...) - return plugins, nil -} - -type pluginInfo struct { - pluginMigrationFeature featuregate.Feature - pluginMigrationCompleteFeature featuregate.Feature - pluginProbeFunction probeFn -} - -func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} - - var err error - for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) - if err != nil { - return allPlugins, err - } - } - return allPlugins, nil -} - -func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - return appendLegacyProviderVolumes(allPlugins, featureGate) -} - -func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} - - var err error - for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) - if err != nil { - return allPlugins, err - } - } - return allPlugins, nil -} diff --git a/cmd/kube-controller-manager/app/policy.go b/cmd/kube-controller-manager/app/policy.go deleted file mode 100644 index a3aa4aae2106d..0000000000000 --- a/cmd/kube-controller-manager/app/policy.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a server that runs a set of active -// components. This includes replication controllers, service endpoints and -// nodes. -// -package app - -import ( - "net/http" - - "k8s.io/klog/v2" - - "k8s.io/apimachinery/pkg/runtime/schema" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/scale" - "k8s.io/kubernetes/pkg/controller/disruption" - kubefeatures "k8s.io/kubernetes/pkg/features" -) - -func startDisruptionController(ctx ControllerContext) (http.Handler, bool, error) { - var group = "policy" - var version = "v1beta1" - var resource = "poddisruptionbudgets" - - if !ctx.AvailableResources[schema.GroupVersionResource{Group: group, Version: version, Resource: resource}] { - klog.Infof( - "Refusing to start disruption because resource %q in group %q is not available.", - resource, group+"/"+version) - return nil, false, nil - } - if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodDisruptionBudget) { - klog.Infof("Refusing to start disruption because the PodDisruptionBudget feature is disabled") - return nil, false, nil - } - - client := ctx.ClientBuilder.ClientOrDie("disruption-controller") - config := ctx.ClientBuilder.ConfigOrDie("disruption-controller") - scaleKindResolver := scale.NewDiscoveryScaleKindResolver(client.Discovery()) - scaleClient, err := scale.NewForConfig(config, ctx.RESTMapper, dynamic.LegacyAPIPathResolverFunc, scaleKindResolver) - if err != nil { - return nil, false, err - } - - go disruption.NewDisruptionController( - ctx.InformerFactory.Core().V1().Pods(), - ctx.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(), - ctx.InformerFactory.Core().V1().ReplicationControllers(), - ctx.InformerFactory.Apps().V1().ReplicaSets(), - ctx.InformerFactory.Apps().V1().Deployments(), - ctx.InformerFactory.Apps().V1().StatefulSets(), - client, - ctx.RESTMapper, - scaleClient, - ).Run(ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/rbac.go b/cmd/kube-controller-manager/app/rbac.go deleted file mode 100644 index 135dbf1a18e2f..0000000000000 --- a/cmd/kube-controller-manager/app/rbac.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "net/http" - - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/kubernetes/pkg/controller/clusterroleaggregation" -) - -func startClusterRoleAggregrationController(ctx ControllerContext) (http.Handler, bool, error) { - if !ctx.AvailableResources[schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}] { - return nil, false, nil - } - go clusterroleaggregation.NewClusterRoleAggregation( - ctx.InformerFactory.Rbac().V1().ClusterRoles(), - ctx.ClientBuilder.ClientOrDie("clusterrole-aggregation-controller").RbacV1(), - ).Run(5, ctx.Stop) - return nil, true, nil -} diff --git a/cmd/kube-controller-manager/app/testing/BUILD b/cmd/kube-controller-manager/app/testing/BUILD deleted file mode 100644 index 75c8ff763e01d..0000000000000 --- a/cmd/kube-controller-manager/app/testing/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_library( - name = "go_default_library", - srcs = ["testserver.go"], - importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kube-controller-manager/app:go_default_library", - "//cmd/kube-controller-manager/app/config:go_default_library", - "//cmd/kube-controller-manager/app/options:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) diff --git a/cmd/kube-controller-manager/app/testing/testserver.go b/cmd/kube-controller-manager/app/testing/testserver.go deleted file mode 100644 index 7d3f0c5579a3b..0000000000000 --- a/cmd/kube-controller-manager/app/testing/testserver.go +++ /dev/null @@ -1,183 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testing - -import ( - "context" - "fmt" - "io/ioutil" - "net" - "os" - "time" - - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/kubernetes/cmd/kube-controller-manager/app" - kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config" - "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" -) - -// TearDownFunc is to be called to tear down a test server. -type TearDownFunc func() - -// TestServer return values supplied by kube-test-ApiServer -type TestServer struct { - LoopbackClientConfig *restclient.Config // Rest client config using the magic token - Options *options.KubeControllerManagerOptions - Config *kubecontrollerconfig.Config - TearDownFn TearDownFunc // TearDown function - TmpDir string // Temp Dir used, by the apiserver -} - -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - -// StartTestServer starts a kube-controller-manager. A rest client config and a tear-down func, -// and location of the tmpdir are returned. -// -// Note: we return a tear-down func instead of a stop channel because the later will leak temporary -// files that because Golang testing's call to os.Exit will not give a stop channel go routine -// enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { - stopCh := make(chan struct{}) - tearDown := func() { - close(stopCh) - if len(result.TmpDir) != 0 { - os.RemoveAll(result.TmpDir) - } - } - defer func() { - if result.TearDownFn == nil { - tearDown() - } - }() - - result.TmpDir, err = ioutil.TempDir("", "kube-controller-manager") - if err != nil { - return result, fmt.Errorf("failed to create temp dir: %v", err) - } - - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - - s, err := options.NewKubeControllerManagerOptions() - if err != nil { - return TestServer{}, err - } - all, disabled := app.KnownControllers(), app.ControllersDisabledByDefault.List() - namedFlagSets := s.Flags(all, disabled) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - fs.Parse(customFlags) - - if s.SecureServing.BindPort != 0 { - s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() - if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) - } - s.SecureServing.ServerCert.CertDirectory = result.TmpDir - - t.Logf("kube-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) - } - - if s.InsecureServing.BindPort != 0 { - s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort() - if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) - } - - t.Logf("kube-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) - } - - config, err := s.Config(all, disabled) - if err != nil { - return result, fmt.Errorf("failed to create config from options: %v", err) - } - - errCh := make(chan error) - go func(stopCh <-chan struct{}) { - if err := app.Run(config.Complete(), stopCh); err != nil { - errCh <- err - } - }(stopCh) - - t.Logf("Waiting for /healthz to be ok...") - client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) - if err != nil { - return result, fmt.Errorf("failed to create a client: %v", err) - } - err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { - select { - case err := <-errCh: - return false, err - default: - } - - result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()) - status := 0 - result.StatusCode(&status) - if status == 200 { - return true, nil - } - return false, nil - }) - if err != nil { - return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) - } - - // from here the caller must call tearDown - result.LoopbackClientConfig = config.LoopbackClientConfig - result.Options = s - result.Config = config - result.TearDownFn = tearDown - - return result, nil -} - -// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) - if err == nil { - return &result - } - - t.Fatalf("failed to launch server: %v", err) - return nil -} - -func createListenerOnFreePort() (net.Listener, int, error) { - ln, err := net.Listen("tcp", ":0") - if err != nil { - return nil, 0, err - } - - // get port - tcpAddr, ok := ln.Addr().(*net.TCPAddr) - if !ok { - ln.Close() - return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) - } - - return ln, tcpAddr.Port, nil -} diff --git a/cmd/kube-controller-manager/controller-manager.go b/cmd/kube-controller-manager/controller-manager.go deleted file mode 100644 index 4d8fb778e9440..0000000000000 --- a/cmd/kube-controller-manager/controller-manager.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The controller manager is responsible for monitoring replication -// controllers, and creating corresponding pods to achieve the desired -// state. It uses the API to listen for new controllers and to create/delete -// pods. -package main - -import ( - "math/rand" - "os" - "time" - - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugin - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/kubernetes/cmd/kube-controller-manager/app" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := app.NewControllerManagerCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/kube-proxy/BUILD b/cmd/kube-proxy/BUILD deleted file mode 100644 index 225f757901ad1..0000000000000 --- a/cmd/kube-proxy/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kube-proxy", - embed = [":go_default_library"], - pure = "on", - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["proxy.go"], - importpath = "k8s.io/kubernetes/cmd/kube-proxy", - deps = [ - "//cmd/kube-proxy/app:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/restclient:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-proxy/app:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kube-proxy/OWNERS b/cmd/kube-proxy/OWNERS deleted file mode 100644 index 37bfd526680ad..0000000000000 --- a/cmd/kube-proxy/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: -- sig-network-reviewers -approvers: -- sig-network-approvers -labels: -- sig/network diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD deleted file mode 100644 index fec2efa4c634b..0000000000000 --- a/cmd/kube-proxy/app/BUILD +++ /dev/null @@ -1,405 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "conntrack.go", - "init_others.go", - "init_windows.go", - "server.go", - "server_others.go", - "server_windows.go", - ], - importpath = "k8s.io/kubernetes/cmd/kube-proxy/app", - deps = [ - "//pkg/apis/core:go_default_library", - "//pkg/cluster/ports:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubelet/qos:go_default_library", - "//pkg/proxy:go_default_library", - "//pkg/proxy/apis:go_default_library", - "//pkg/proxy/apis/config:go_default_library", - "//pkg/proxy/apis/config/scheme:go_default_library", - "//pkg/proxy/apis/config/v1alpha1:go_default_library", - "//pkg/proxy/apis/config/validation:go_default_library", - "//pkg/proxy/config:go_default_library", - "//pkg/proxy/healthcheck:go_default_library", - "//pkg/proxy/iptables:go_default_library", - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/userspace:go_default_library", - "//pkg/proxy/util:go_default_library", - "//pkg/util/filesystem:go_default_library", - "//pkg/util/flag:go_default_library", - "//pkg/util/ipset:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/ipvs:go_default_library", - "//pkg/util/oom:go_default_library", - "//pkg/util/sysctl:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/selection:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/config:go_default_library", - "//staging/src/k8s.io/component-base/configz:go_default_library", - "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", - "//staging/src/k8s.io/mount-utils:go_default_library", - "//vendor/github.com/fsnotify/fsnotify:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:aix": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:android": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:darwin": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:dragonfly": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:freebsd": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:illumos": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:ios": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:js": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:nacl": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:netbsd": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:openbsd": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:plan9": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:solaris": [ - "//pkg/proxy/metrics:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/node:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", - "//staging/src/k8s.io/client-go/tools/cache:go_default_library", - "//staging/src/k8s.io/client-go/tools/watch:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "@io_bazel_rules_go//go/platform:windows": [ - "//pkg/proxy/winkernel:go_default_library", - "//pkg/proxy/winuserspace:go_default_library", - "//pkg/util/netsh:go_default_library", - "//pkg/util/node:go_default_library", - "//pkg/windows/service:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], - "//conditions:default": [], - }), -) - -go_test( - name = "go_default_test", - srcs = [ - "server_others_test.go", - "server_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//pkg/proxy/apis/config:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/component-base/config:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", - "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:aix": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:android": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:darwin": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:dragonfly": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:freebsd": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:illumos": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:ios": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:js": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:nacl": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:netbsd": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:openbsd": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:plan9": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "@io_bazel_rules_go//go/platform:solaris": [ - "//pkg/proxy/ipvs:go_default_library", - "//pkg/proxy/util/iptables:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], - "//conditions:default": [], - }), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kube-proxy/app/conntrack.go b/cmd/kube-proxy/app/conntrack.go deleted file mode 100644 index 3ebe9ff2ac27f..0000000000000 --- a/cmd/kube-proxy/app/conntrack.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "errors" - "io/ioutil" - "strconv" - "strings" - - "k8s.io/klog/v2" - "k8s.io/mount-utils" - - "k8s.io/kubernetes/pkg/util/sysctl" -) - -// Conntracker is an interface to the global sysctl. Descriptions of the various -// sysctl fields can be found here: -// -// https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt -type Conntracker interface { - // SetMax adjusts nf_conntrack_max. - SetMax(max int) error - // SetTCPEstablishedTimeout adjusts nf_conntrack_tcp_timeout_established. - SetTCPEstablishedTimeout(seconds int) error - // SetTCPCloseWaitTimeout nf_conntrack_tcp_timeout_close_wait. - SetTCPCloseWaitTimeout(seconds int) error -} - -type realConntracker struct{} - -var errReadOnlySysFS = errors.New("readOnlySysFS") - -func (rct realConntracker) SetMax(max int) error { - if err := rct.setIntSysCtl("nf_conntrack_max", max); err != nil { - return err - } - klog.Infof("Setting nf_conntrack_max to %d", max) - - // Linux does not support writing to /sys/module/nf_conntrack/parameters/hashsize - // when the writer process is not in the initial network namespace - // (https://github.com/torvalds/linux/blob/v4.10/net/netfilter/nf_conntrack_core.c#L1795-L1796). - // Usually that's fine. But in some configurations such as with github.com/kinvolk/kubeadm-nspawn, - // kube-proxy is in another netns. - // Therefore, check if writing in hashsize is necessary and skip the writing if not. - hashsize, err := readIntStringFile("/sys/module/nf_conntrack/parameters/hashsize") - if err != nil { - return err - } - if hashsize >= (max / 4) { - return nil - } - - // sysfs is expected to be mounted as 'rw'. However, it may be - // unexpectedly mounted as 'ro' by docker because of a known docker - // issue (https://github.com/docker/docker/issues/24000). Setting - // conntrack will fail when sysfs is readonly. When that happens, we - // don't set conntrack hashsize and return a special error - // errReadOnlySysFS here. The caller should deal with - // errReadOnlySysFS differently. - writable, err := isSysFSWritable() - if err != nil { - return err - } - if !writable { - return errReadOnlySysFS - } - // TODO: generify this and sysctl to a new sysfs.WriteInt() - klog.Infof("Setting conntrack hashsize to %d", max/4) - return writeIntStringFile("/sys/module/nf_conntrack/parameters/hashsize", max/4) -} - -func (rct realConntracker) SetTCPEstablishedTimeout(seconds int) error { - return rct.setIntSysCtl("nf_conntrack_tcp_timeout_established", seconds) -} - -func (rct realConntracker) SetTCPCloseWaitTimeout(seconds int) error { - return rct.setIntSysCtl("nf_conntrack_tcp_timeout_close_wait", seconds) -} - -func (realConntracker) setIntSysCtl(name string, value int) error { - entry := "net/netfilter/" + name - - sys := sysctl.New() - if val, _ := sys.GetSysctl(entry); val != value { - klog.Infof("Set sysctl '%v' to %v", entry, value) - if err := sys.SetSysctl(entry, value); err != nil { - return err - } - } - return nil -} - -// isSysFSWritable checks /proc/mounts to see whether sysfs is 'rw' or not. -func isSysFSWritable() (bool, error) { - const permWritable = "rw" - const sysfsDevice = "sysfs" - m := mount.New("" /* default mount path */) - mountPoints, err := m.List() - if err != nil { - klog.Errorf("failed to list mount points: %v", err) - return false, err - } - - for _, mountPoint := range mountPoints { - if mountPoint.Type != sysfsDevice { - continue - } - // Check whether sysfs is 'rw' - if len(mountPoint.Opts) > 0 && mountPoint.Opts[0] == permWritable { - return true, nil - } - klog.Errorf("sysfs is not writable: %+v (mount options are %v)", - mountPoint, mountPoint.Opts) - return false, errReadOnlySysFS - } - - return false, errors.New("no sysfs mounted") -} - -func readIntStringFile(filename string) (int, error) { - b, err := ioutil.ReadFile(filename) - if err != nil { - return -1, err - } - return strconv.Atoi(strings.TrimSpace(string(b))) -} - -func writeIntStringFile(filename string, value int) error { - return ioutil.WriteFile(filename, []byte(strconv.Itoa(value)), 0640) -} diff --git a/cmd/kube-proxy/app/init_others.go b/cmd/kube-proxy/app/init_others.go deleted file mode 100644 index 6c0e6b8c7d576..0000000000000 --- a/cmd/kube-proxy/app/init_others.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build !windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "github.com/spf13/pflag" -) - -func initForOS(service bool) error { - return nil -} - -func (o *Options) addOSFlags(fs *pflag.FlagSet) { -} diff --git a/cmd/kube-proxy/app/init_windows.go b/cmd/kube-proxy/app/init_windows.go deleted file mode 100644 index 5db0b703f40bd..0000000000000 --- a/cmd/kube-proxy/app/init_windows.go +++ /dev/null @@ -1,43 +0,0 @@ -// +build windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "k8s.io/kubernetes/pkg/windows/service" - - "github.com/spf13/pflag" -) - -const ( - serviceName = "kube-proxy" -) - -func initForOS(windowsService bool) error { - if windowsService { - return service.InitService(serviceName) - } - return nil -} - -func (o *Options) addOSFlags(fs *pflag.FlagSet) { - fs.BoolVar(&o.WindowsService, "windows-service", o.WindowsService, "Enable Windows Service Control Manager API integration") - fs.StringVar(&o.config.Winkernel.SourceVip, "source-vip", o.config.Winkernel.SourceVip, "The IP address of the source VIP for non-DSR.") - fs.StringVar(&o.config.Winkernel.NetworkName, "network-name", o.config.Winkernel.NetworkName, "The name of the cluster network.") - fs.BoolVar(&o.config.Winkernel.EnableDSR, "enable-dsr", o.config.Winkernel.EnableDSR, "If true make kube-proxy apply DSR policies for service VIP") -} diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go deleted file mode 100644 index 3f696152b834e..0000000000000 --- a/cmd/kube-proxy/app/server.go +++ /dev/null @@ -1,823 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to configure and run a -// Kubernetes app process. -package app - -import ( - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - goruntime "runtime" - "strings" - "time" - - "github.com/fsnotify/fsnotify" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - gerrors "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/selection" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/apiserver/pkg/server/mux" - "k8s.io/apiserver/pkg/server/routes" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/client-go/tools/record" - cliflag "k8s.io/component-base/cli/flag" - componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/configz" - "k8s.io/component-base/metrics/legacyregistry" - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - "k8s.io/klog/v2" - "k8s.io/kube-proxy/config/v1alpha1" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/cluster/ports" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/kubelet/qos" - "k8s.io/kubernetes/pkg/proxy" - "k8s.io/kubernetes/pkg/proxy/apis" - kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" - proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" - kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/config/v1alpha1" - "k8s.io/kubernetes/pkg/proxy/apis/config/validation" - "k8s.io/kubernetes/pkg/proxy/config" - "k8s.io/kubernetes/pkg/proxy/healthcheck" - "k8s.io/kubernetes/pkg/proxy/iptables" - "k8s.io/kubernetes/pkg/proxy/ipvs" - "k8s.io/kubernetes/pkg/proxy/userspace" - proxyutil "k8s.io/kubernetes/pkg/proxy/util" - "k8s.io/kubernetes/pkg/util/filesystem" - utilflag "k8s.io/kubernetes/pkg/util/flag" - utilipset "k8s.io/kubernetes/pkg/util/ipset" - utiliptables "k8s.io/kubernetes/pkg/util/iptables" - utilipvs "k8s.io/kubernetes/pkg/util/ipvs" - "k8s.io/kubernetes/pkg/util/oom" - "k8s.io/utils/exec" - utilpointer "k8s.io/utils/pointer" -) - -const ( - proxyModeUserspace = "userspace" - proxyModeIPTables = "iptables" - proxyModeIPVS = "ipvs" - proxyModeKernelspace = "kernelspace" -) - -// proxyRun defines the interface to run a specified ProxyServer -type proxyRun interface { - Run() error - CleanupAndExit() error -} - -// Options contains everything necessary to create and run a proxy server. -type Options struct { - // ConfigFile is the location of the proxy server's configuration file. - ConfigFile string - // WriteConfigTo is the path where the default configuration will be written. - WriteConfigTo string - // CleanupAndExit, when true, makes the proxy server clean up iptables and ipvs rules, then exit. - CleanupAndExit bool - // CleanupIPVS, when true, makes the proxy server clean up ipvs rules before running. - CleanupIPVS bool - // WindowsService should be set to true if kube-proxy is running as a service on Windows. - // Its corresponding flag only gets registered in Windows builds - WindowsService bool - // config is the proxy server's configuration object. - config *kubeproxyconfig.KubeProxyConfiguration - // watcher is used to watch on the update change of ConfigFile - watcher filesystem.FSWatcher - // proxyServer is the interface to run the proxy server - proxyServer proxyRun - // errCh is the channel that errors will be sent - errCh chan error - - // The fields below here are placeholders for flags that can't be directly mapped into - // config.KubeProxyConfiguration. - // - // TODO remove these fields once the deprecated flags are removed. - - // master is used to override the kubeconfig's URL to the apiserver. - master string - // healthzPort is the port to be used by the healthz server. - healthzPort int32 - // metricsPort is the port to be used by the metrics server. - metricsPort int32 - - // hostnameOverride, if set from the command line flag, takes precedence over the `HostnameOverride` value from the config file - hostnameOverride string -} - -// AddFlags adds flags to fs and binds them to options. -func (o *Options) AddFlags(fs *pflag.FlagSet) { - o.addOSFlags(fs) - - fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.") - fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the default configuration values to this file and exit.") - fs.StringVar(&o.config.ClientConnection.Kubeconfig, "kubeconfig", o.config.ClientConnection.Kubeconfig, "Path to kubeconfig file with authorization information (the master location can be overridden by the master flag).") - fs.StringVar(&o.config.ClusterCIDR, "cluster-cidr", o.config.ClusterCIDR, "The CIDR range of pods in the cluster. When configured, traffic sent to a Service cluster IP from outside this range will be masqueraded and traffic sent from pods to an external LoadBalancer IP will be directed to the respective cluster IP instead") - fs.StringVar(&o.config.ClientConnection.ContentType, "kube-api-content-type", o.config.ClientConnection.ContentType, "Content type of requests sent to apiserver.") - fs.StringVar(&o.master, "master", o.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - fs.StringVar(&o.hostnameOverride, "hostname-override", o.hostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") - fs.StringVar(&o.config.IPVS.Scheduler, "ipvs-scheduler", o.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs") - fs.StringVar(&o.config.ShowHiddenMetricsForVersion, "show-hidden-metrics-for-version", o.config.ShowHiddenMetricsForVersion, - "The previous version for which you want to show hidden metrics. "+ - "Only the previous minor version is meaningful, other values will not be allowed. "+ - "The format is ., e.g.: '1.16'. "+ - "The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, "+ - "rather than being surprised when they are permanently removed in the release after that.") - - fs.StringSliceVar(&o.config.IPVS.ExcludeCIDRs, "ipvs-exclude-cidrs", o.config.IPVS.ExcludeCIDRs, "A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.") - fs.StringSliceVar(&o.config.NodePortAddresses, "nodeport-addresses", o.config.NodePortAddresses, - "A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.") - - fs.BoolVar(&o.CleanupAndExit, "cleanup", o.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.") - fs.BoolVar(&o.CleanupIPVS, "cleanup-ipvs", o.CleanupIPVS, "If true and --cleanup is specified, kube-proxy will also flush IPVS rules, in addition to normal cleanup.") - fs.MarkDeprecated("cleanup-ipvs", "In a future release, running --cleanup will always flush IPVS rules") - - fs.Var(utilflag.IPVar{Val: &o.config.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to '0.0.0.0' for all IPv4 interfaces and '::' for all IPv6 interfaces)") - fs.Var(utilflag.IPPortVar{Val: &o.config.HealthzBindAddress}, "healthz-bind-address", "The IP address with port for the health check server to serve on (set to '0.0.0.0:10256' for all IPv4 interfaces and '[::]:10256' for all IPv6 interfaces). Set empty to disable.") - fs.Var(utilflag.IPPortVar{Val: &o.config.MetricsBindAddress}, "metrics-bind-address", "The IP address with port for the metrics server to serve on (set to '0.0.0.0:10249' for all IPv4 interfaces and '[::]:10249' for all IPv6 interfaces). Set empty to disable.") - fs.BoolVar(&o.config.BindAddressHardFail, "bind-address-hard-fail", o.config.BindAddressHardFail, "If true kube-proxy will treat failure to bind to a port as fatal and exit") - fs.Var(utilflag.PortRangeVar{Val: &o.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, single port or beginPort+offset, inclusive) that may be consumed in order to proxy service traffic. If (unspecified, 0, or 0-0) then ports will be randomly chosen.") - fs.Var(&o.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs' or 'kernelspace' (windows). If blank, use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") - fs.Var(cliflag.NewMapStringBool(&o.config.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ - "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) - - fs.Int32Var(&o.healthzPort, "healthz-port", o.healthzPort, "The port to bind the health check server. Use 0 to disable.") - fs.MarkDeprecated("healthz-port", "This flag is deprecated and will be removed in a future release. Please use --healthz-bind-address instead.") - fs.Int32Var(&o.metricsPort, "metrics-port", o.metricsPort, "The port to bind the metrics server. Use 0 to disable.") - fs.MarkDeprecated("metrics-port", "This flag is deprecated and will be removed in a future release. Please use --metrics-bind-address instead.") - fs.Int32Var(o.config.OOMScoreAdj, "oom-score-adj", utilpointer.Int32PtrDerefOr(o.config.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]") - fs.Int32Var(o.config.IPTables.MasqueradeBit, "iptables-masquerade-bit", utilpointer.Int32PtrDerefOr(o.config.IPTables.MasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within the range [0, 31].") - fs.Int32Var(o.config.Conntrack.MaxPerCore, "conntrack-max-per-core", *o.config.Conntrack.MaxPerCore, - "Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min).") - fs.Int32Var(o.config.Conntrack.Min, "conntrack-min", *o.config.Conntrack.Min, - "Minimum number of conntrack entries to allocate, regardless of conntrack-max-per-core (set conntrack-max-per-core=0 to leave the limit as-is).") - fs.Int32Var(&o.config.ClientConnection.Burst, "kube-api-burst", o.config.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver") - - fs.DurationVar(&o.config.IPTables.SyncPeriod.Duration, "iptables-sync-period", o.config.IPTables.SyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") - fs.DurationVar(&o.config.IPTables.MinSyncPeriod.Duration, "iptables-min-sync-period", o.config.IPTables.MinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').") - fs.DurationVar(&o.config.IPVS.SyncPeriod.Duration, "ipvs-sync-period", o.config.IPVS.SyncPeriod.Duration, "The maximum interval of how often ipvs rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") - fs.DurationVar(&o.config.IPVS.MinSyncPeriod.Duration, "ipvs-min-sync-period", o.config.IPVS.MinSyncPeriod.Duration, "The minimum interval of how often the ipvs rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').") - fs.DurationVar(&o.config.IPVS.TCPTimeout.Duration, "ipvs-tcp-timeout", o.config.IPVS.TCPTimeout.Duration, "The timeout for idle IPVS TCP connections, 0 to leave as-is. (e.g. '5s', '1m', '2h22m').") - fs.DurationVar(&o.config.IPVS.TCPFinTimeout.Duration, "ipvs-tcpfin-timeout", o.config.IPVS.TCPFinTimeout.Duration, "The timeout for IPVS TCP connections after receiving a FIN packet, 0 to leave as-is. (e.g. '5s', '1m', '2h22m').") - fs.DurationVar(&o.config.IPVS.UDPTimeout.Duration, "ipvs-udp-timeout", o.config.IPVS.UDPTimeout.Duration, "The timeout for IPVS UDP packets, 0 to leave as-is. (e.g. '5s', '1m', '2h22m').") - fs.DurationVar(&o.config.Conntrack.TCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", o.config.Conntrack.TCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)") - fs.DurationVar( - &o.config.Conntrack.TCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait", - o.config.Conntrack.TCPCloseWaitTimeout.Duration, - "NAT timeout for TCP connections in the CLOSE_WAIT state") - fs.DurationVar(&o.config.ConfigSyncPeriod.Duration, "config-sync-period", o.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed. Must be greater than 0.") - fs.DurationVar(&o.config.UDPIdleTimeout.Duration, "udp-timeout", o.config.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace") - - fs.BoolVar(&o.config.IPVS.StrictARP, "ipvs-strict-arp", o.config.IPVS.StrictARP, "Enable strict ARP by setting arp_ignore to 1 and arp_announce to 2") - fs.BoolVar(&o.config.IPTables.MasqueradeAll, "masquerade-all", o.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)") - fs.BoolVar(&o.config.EnableProfiling, "profiling", o.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.") - - fs.Float32Var(&o.config.ClientConnection.QPS, "kube-api-qps", o.config.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver") - fs.Var(&o.config.DetectLocalMode, "detect-local-mode", "Mode to use to detect local traffic") -} - -// NewOptions returns initialized Options -func NewOptions() *Options { - return &Options{ - config: new(kubeproxyconfig.KubeProxyConfiguration), - healthzPort: ports.ProxyHealthzPort, - metricsPort: ports.ProxyStatusPort, - CleanupIPVS: true, - errCh: make(chan error), - } -} - -// Complete completes all the required options. -func (o *Options) Complete() error { - if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 { - klog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup are deprecated. Please begin using a config file ASAP.") - o.config.HealthzBindAddress = addressFromDeprecatedFlags(o.config.HealthzBindAddress, o.healthzPort) - o.config.MetricsBindAddress = addressFromDeprecatedFlags(o.config.MetricsBindAddress, o.metricsPort) - } - - // Load the config file here in Complete, so that Validate validates the fully-resolved config. - if len(o.ConfigFile) > 0 { - c, err := o.loadConfigFromFile(o.ConfigFile) - if err != nil { - return err - } - o.config = c - - if err := o.initWatcher(); err != nil { - return err - } - } - - if err := o.processHostnameOverrideFlag(); err != nil { - return err - } - - return utilfeature.DefaultMutableFeatureGate.SetFromMap(o.config.FeatureGates) -} - -// Creates a new filesystem watcher and adds watches for the config file. -func (o *Options) initWatcher() error { - fswatcher := filesystem.NewFsnotifyWatcher() - err := fswatcher.Init(o.eventHandler, o.errorHandler) - if err != nil { - return err - } - err = fswatcher.AddWatch(o.ConfigFile) - if err != nil { - return err - } - o.watcher = fswatcher - return nil -} - -func (o *Options) eventHandler(ent fsnotify.Event) { - eventOpIs := func(Op fsnotify.Op) bool { - return ent.Op&Op == Op - } - if eventOpIs(fsnotify.Write) || eventOpIs(fsnotify.Rename) { - // error out when ConfigFile is updated - o.errCh <- fmt.Errorf("content of the proxy server's configuration file was updated") - return - } - o.errCh <- nil -} - -func (o *Options) errorHandler(err error) { - o.errCh <- err -} - -// processHostnameOverrideFlag processes hostname-override flag -func (o *Options) processHostnameOverrideFlag() error { - // Check if hostname-override flag is set and use value since configFile always overrides - if len(o.hostnameOverride) > 0 { - hostName := strings.TrimSpace(o.hostnameOverride) - if len(hostName) == 0 { - return fmt.Errorf("empty hostname-override is invalid") - } - o.config.HostnameOverride = strings.ToLower(hostName) - } - - return nil -} - -// Validate validates all the required options. -func (o *Options) Validate() error { - if errs := validation.Validate(o.config); len(errs) != 0 { - return errs.ToAggregate() - } - - return nil -} - -// Run runs the specified ProxyServer. -func (o *Options) Run() error { - defer close(o.errCh) - if len(o.WriteConfigTo) > 0 { - return o.writeConfigFile() - } - - proxyServer, err := NewProxyServer(o) - if err != nil { - return err - } - - if o.CleanupAndExit { - return proxyServer.CleanupAndExit() - } - - o.proxyServer = proxyServer - return o.runLoop() -} - -// runLoop will watch on the update change of the proxy server's configuration file. -// Return an error when updated -func (o *Options) runLoop() error { - if o.watcher != nil { - o.watcher.Run() - } - - // run the proxy in goroutine - go func() { - err := o.proxyServer.Run() - o.errCh <- err - }() - - for { - err := <-o.errCh - if err != nil { - return err - } - } -} - -func (o *Options) writeConfigFile() (err error) { - const mediaType = runtime.ContentTypeYAML - info, ok := runtime.SerializerInfoForMediaType(proxyconfigscheme.Codecs.SupportedMediaTypes(), mediaType) - if !ok { - return fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) - } - - encoder := proxyconfigscheme.Codecs.EncoderForVersion(info.Serializer, v1alpha1.SchemeGroupVersion) - - configFile, err := os.Create(o.WriteConfigTo) - if err != nil { - return err - } - - defer func() { - ferr := configFile.Close() - if ferr != nil && err == nil { - err = ferr - } - }() - - if err = encoder.Encode(o.config, configFile); err != nil { - return err - } - - klog.Infof("Wrote configuration to: %s\n", o.WriteConfigTo) - - return nil -} - -// addressFromDeprecatedFlags returns server address from flags -// passed on the command line based on the following rules: -// 1. If port is 0, disable the server (e.g. set address to empty). -// 2. Otherwise, set the port portion of the config accordingly. -func addressFromDeprecatedFlags(addr string, port int32) string { - if port == 0 { - return "" - } - return proxyutil.AppendPortIfNeeded(addr, port) -} - -// newLenientSchemeAndCodecs returns a scheme that has only v1alpha1 registered into -// it and a CodecFactory with strict decoding disabled. -func newLenientSchemeAndCodecs() (*runtime.Scheme, *serializer.CodecFactory, error) { - lenientScheme := runtime.NewScheme() - if err := kubeproxyconfig.AddToScheme(lenientScheme); err != nil { - return nil, nil, fmt.Errorf("failed to add kube-proxy config API to lenient scheme: %v", err) - } - if err := kubeproxyconfigv1alpha1.AddToScheme(lenientScheme); err != nil { - return nil, nil, fmt.Errorf("failed to add kube-proxy config v1alpha1 API to lenient scheme: %v", err) - } - lenientCodecs := serializer.NewCodecFactory(lenientScheme, serializer.DisableStrict) - return lenientScheme, &lenientCodecs, nil -} - -// loadConfigFromFile loads the contents of file and decodes it as a -// KubeProxyConfiguration object. -func (o *Options) loadConfigFromFile(file string) (*kubeproxyconfig.KubeProxyConfiguration, error) { - data, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - return o.loadConfig(data) -} - -// loadConfig decodes a serialized KubeProxyConfiguration to the internal type. -func (o *Options) loadConfig(data []byte) (*kubeproxyconfig.KubeProxyConfiguration, error) { - - configObj, gvk, err := proxyconfigscheme.Codecs.UniversalDecoder().Decode(data, nil, nil) - if err != nil { - // Try strict decoding first. If that fails decode with a lenient - // decoder, which has only v1alpha1 registered, and log a warning. - // The lenient path is to be dropped when support for v1alpha1 is dropped. - if !runtime.IsStrictDecodingError(err) { - return nil, gerrors.Wrap(err, "failed to decode") - } - - _, lenientCodecs, lenientErr := newLenientSchemeAndCodecs() - if lenientErr != nil { - return nil, lenientErr - } - - configObj, gvk, lenientErr = lenientCodecs.UniversalDecoder().Decode(data, nil, nil) - if lenientErr != nil { - // Lenient decoding failed with the current version, return the - // original strict error. - return nil, fmt.Errorf("failed lenient decoding: %v", err) - } - - // Continue with the v1alpha1 object that was decoded leniently, but emit a warning. - klog.Warningf("using lenient decoding as strict decoding failed: %v", err) - } - - proxyConfig, ok := configObj.(*kubeproxyconfig.KubeProxyConfiguration) - if !ok { - return nil, fmt.Errorf("got unexpected config type: %v", gvk) - } - return proxyConfig, nil -} - -// ApplyDefaults applies the default values to Options. -func (o *Options) ApplyDefaults(in *kubeproxyconfig.KubeProxyConfiguration) (*kubeproxyconfig.KubeProxyConfiguration, error) { - external, err := proxyconfigscheme.Scheme.ConvertToVersion(in, v1alpha1.SchemeGroupVersion) - if err != nil { - return nil, err - } - - proxyconfigscheme.Scheme.Default(external) - - internal, err := proxyconfigscheme.Scheme.ConvertToVersion(external, kubeproxyconfig.SchemeGroupVersion) - if err != nil { - return nil, err - } - - out := internal.(*kubeproxyconfig.KubeProxyConfiguration) - - return out, nil -} - -// NewProxyCommand creates a *cobra.Command object with default parameters -func NewProxyCommand() *cobra.Command { - opts := NewOptions() - - cmd := &cobra.Command{ - Use: "kube-proxy", - Long: `The Kubernetes network proxy runs on each node. This -reflects services as defined in the Kubernetes API on each node and can do simple -TCP, UDP, and SCTP stream forwarding or round robin TCP, UDP, and SCTP forwarding across a set of backends. -Service cluster IPs and ports are currently found through Docker-links-compatible -environment variables specifying ports opened by the service proxy. There is an optional -addon that provides cluster DNS for these cluster IPs. The user must create a service -with the apiserver API to configure the proxy.`, - Run: func(cmd *cobra.Command, args []string) { - verflag.PrintAndExitIfRequested() - cliflag.PrintFlags(cmd.Flags()) - - if err := initForOS(opts.WindowsService); err != nil { - klog.Fatalf("failed OS init: %v", err) - } - - if err := opts.Complete(); err != nil { - klog.Fatalf("failed complete: %v", err) - } - if err := opts.Validate(); err != nil { - klog.Fatalf("failed validate: %v", err) - } - - if err := opts.Run(); err != nil { - klog.Exit(err) - } - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - - var err error - opts.config, err = opts.ApplyDefaults(opts.config) - if err != nil { - klog.Fatalf("unable to create flag defaults: %v", err) - } - - opts.AddFlags(cmd.Flags()) - - // TODO handle error - cmd.MarkFlagFilename("config", "yaml", "yml", "json") - - return cmd -} - -// ProxyServer represents all the parameters required to start the Kubernetes proxy server. All -// fields are required. -type ProxyServer struct { - Client clientset.Interface - EventClient v1core.EventsGetter - IptInterface utiliptables.Interface - IpvsInterface utilipvs.Interface - IpsetInterface utilipset.Interface - execer exec.Interface - Proxier proxy.Provider - Broadcaster record.EventBroadcaster - Recorder record.EventRecorder - ConntrackConfiguration kubeproxyconfig.KubeProxyConntrackConfiguration - Conntracker Conntracker // if nil, ignored - ProxyMode string - NodeRef *v1.ObjectReference - CleanupIPVS bool - MetricsBindAddress string - BindAddressHardFail bool - EnableProfiling bool - UseEndpointSlices bool - OOMScoreAdj *int32 - ConfigSyncPeriod time.Duration - HealthzServer healthcheck.ProxierHealthUpdater -} - -// createClients creates a kube client and an event client from the given config and masterOverride. -// TODO remove masterOverride when CLI flags are removed. -func createClients(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, v1core.EventsGetter, error) { - var kubeConfig *rest.Config - var err error - - if len(config.Kubeconfig) == 0 && len(masterOverride) == 0 { - klog.Info("Neither kubeconfig file nor master URL was specified. Falling back to in-cluster config.") - kubeConfig, err = rest.InClusterConfig() - } else { - // This creates a client, first loading any specified kubeconfig - // file, and then overriding the Master flag, if non-empty. - kubeConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: config.Kubeconfig}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig() - } - if err != nil { - return nil, nil, err - } - - kubeConfig.AcceptContentTypes = config.AcceptContentTypes - kubeConfig.ContentType = config.ContentType - kubeConfig.QPS = config.QPS - kubeConfig.Burst = int(config.Burst) - - client, err := clientset.NewForConfig(kubeConfig) - if err != nil { - return nil, nil, err - } - - eventClient, err := clientset.NewForConfig(kubeConfig) - if err != nil { - return nil, nil, err - } - - return client, eventClient.CoreV1(), nil -} - -func serveHealthz(hz healthcheck.ProxierHealthUpdater, errCh chan error) { - if hz == nil { - return - } - - fn := func() { - err := hz.Run() - if err != nil { - klog.Errorf("healthz server failed: %v", err) - if errCh != nil { - errCh <- fmt.Errorf("healthz server failed: %v", err) - // if in hardfail mode, never retry again - blockCh := make(chan error) - <-blockCh - } - } else { - klog.Errorf("healthz server returned without error") - } - } - go wait.Until(fn, 5*time.Second, wait.NeverStop) -} - -func serveMetrics(bindAddress, proxyMode string, enableProfiling bool, errCh chan error) { - if len(bindAddress) == 0 { - return - } - - proxyMux := mux.NewPathRecorderMux("kube-proxy") - healthz.InstallHandler(proxyMux) - proxyMux.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Content-Type-Options", "nosniff") - fmt.Fprintf(w, "%s", proxyMode) - }) - - //lint:ignore SA1019 See the Metrics Stability Migration KEP - proxyMux.Handle("/metrics", legacyregistry.Handler()) - - if enableProfiling { - routes.Profiling{}.Install(proxyMux) - } - - configz.InstallHandler(proxyMux) - - fn := func() { - err := http.ListenAndServe(bindAddress, proxyMux) - if err != nil { - err = fmt.Errorf("starting metrics server failed: %v", err) - utilruntime.HandleError(err) - if errCh != nil { - errCh <- err - // if in hardfail mode, never retry again - blockCh := make(chan error) - <-blockCh - } - } - } - go wait.Until(fn, 5*time.Second, wait.NeverStop) -} - -// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). -// TODO: At the moment, Run() cannot return a nil error, otherwise it's caller will never exit. Update callers of Run to handle nil errors. -func (s *ProxyServer) Run() error { - // To help debugging, immediately log version - klog.Infof("Version: %+v", version.Get()) - - // TODO(vmarmol): Use container config for this. - var oomAdjuster *oom.OOMAdjuster - if s.OOMScoreAdj != nil { - oomAdjuster = oom.NewOOMAdjuster() - if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*s.OOMScoreAdj)); err != nil { - klog.V(2).Info(err) - } - } - - if s.Broadcaster != nil && s.EventClient != nil { - s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")}) - } - - // TODO(thockin): make it possible for healthz and metrics to be on the same port. - - var errCh chan error - if s.BindAddressHardFail { - errCh = make(chan error) - } - - // Start up a healthz server if requested - serveHealthz(s.HealthzServer, errCh) - - // Start up a metrics server if requested - serveMetrics(s.MetricsBindAddress, s.ProxyMode, s.EnableProfiling, errCh) - - // Tune conntrack, if requested - // Conntracker is always nil for windows - if s.Conntracker != nil { - max, err := getConntrackMax(s.ConntrackConfiguration) - if err != nil { - return err - } - if max > 0 { - err := s.Conntracker.SetMax(max) - if err != nil { - if err != errReadOnlySysFS { - return err - } - // errReadOnlySysFS is caused by a known docker issue (https://github.com/docker/docker/issues/24000), - // the only remediation we know is to restart the docker daemon. - // Here we'll send an node event with specific reason and message, the - // administrator should decide whether and how to handle this issue, - // whether to drain the node and restart docker. Occurs in other container runtimes - // as well. - // TODO(random-liu): Remove this when the docker bug is fixed. - const message = "CRI error: /sys is read-only: " + - "cannot modify conntrack limits, problems may arise later (If running Docker, see docker issue #24000)" - s.Recorder.Eventf(s.NodeRef, api.EventTypeWarning, err.Error(), message) - } - } - - if s.ConntrackConfiguration.TCPEstablishedTimeout != nil && s.ConntrackConfiguration.TCPEstablishedTimeout.Duration > 0 { - timeout := int(s.ConntrackConfiguration.TCPEstablishedTimeout.Duration / time.Second) - if err := s.Conntracker.SetTCPEstablishedTimeout(timeout); err != nil { - return err - } - } - - if s.ConntrackConfiguration.TCPCloseWaitTimeout != nil && s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration > 0 { - timeout := int(s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration / time.Second) - if err := s.Conntracker.SetTCPCloseWaitTimeout(timeout); err != nil { - return err - } - } - } - - noProxyName, err := labels.NewRequirement(apis.LabelServiceProxyName, selection.DoesNotExist, nil) - if err != nil { - return err - } - - noHeadlessEndpoints, err := labels.NewRequirement(v1.IsHeadlessService, selection.DoesNotExist, nil) - if err != nil { - return err - } - - labelSelector := labels.NewSelector() - labelSelector = labelSelector.Add(*noProxyName, *noHeadlessEndpoints) - - // Make informers that filter out objects that want a non-default service proxy. - informerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.ConfigSyncPeriod, - informers.WithTweakListOptions(func(options *metav1.ListOptions) { - options.LabelSelector = labelSelector.String() - })) - - // Create configs (i.e. Watches for Services and Endpoints or EndpointSlices) - // Note: RegisterHandler() calls need to happen before creation of Sources because sources - // only notify on changes, and the initial update (on process start) may be lost if no handlers - // are registered yet. - serviceConfig := config.NewServiceConfig(informerFactory.Core().V1().Services(), s.ConfigSyncPeriod) - serviceConfig.RegisterEventHandler(s.Proxier) - go serviceConfig.Run(wait.NeverStop) - - if s.UseEndpointSlices { - endpointSliceConfig := config.NewEndpointSliceConfig(informerFactory.Discovery().V1beta1().EndpointSlices(), s.ConfigSyncPeriod) - endpointSliceConfig.RegisterEventHandler(s.Proxier) - go endpointSliceConfig.Run(wait.NeverStop) - } else { - endpointsConfig := config.NewEndpointsConfig(informerFactory.Core().V1().Endpoints(), s.ConfigSyncPeriod) - endpointsConfig.RegisterEventHandler(s.Proxier) - go endpointsConfig.Run(wait.NeverStop) - } - - // This has to start after the calls to NewServiceConfig and NewEndpointsConfig because those - // functions must configure their shared informer event handlers first. - informerFactory.Start(wait.NeverStop) - - if utilfeature.DefaultFeatureGate.Enabled(features.ServiceTopology) { - // Make an informer that selects for our nodename. - currentNodeInformerFactory := informers.NewSharedInformerFactoryWithOptions(s.Client, s.ConfigSyncPeriod, - informers.WithTweakListOptions(func(options *metav1.ListOptions) { - options.FieldSelector = fields.OneTermEqualSelector("metadata.name", s.NodeRef.Name).String() - })) - nodeConfig := config.NewNodeConfig(currentNodeInformerFactory.Core().V1().Nodes(), s.ConfigSyncPeriod) - nodeConfig.RegisterEventHandler(s.Proxier) - go nodeConfig.Run(wait.NeverStop) - - // This has to start after the calls to NewNodeConfig because that must - // configure the shared informer event handler first. - currentNodeInformerFactory.Start(wait.NeverStop) - } - - // Birth Cry after the birth is successful - s.birthCry() - - go s.Proxier.SyncLoop() - - return <-errCh -} - -func (s *ProxyServer) birthCry() { - s.Recorder.Eventf(s.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.") -} - -func getConntrackMax(config kubeproxyconfig.KubeProxyConntrackConfiguration) (int, error) { - if config.MaxPerCore != nil && *config.MaxPerCore > 0 { - floor := 0 - if config.Min != nil { - floor = int(*config.Min) - } - scaled := int(*config.MaxPerCore) * goruntime.NumCPU() - if scaled > floor { - klog.V(3).Infof("getConntrackMax: using scaled conntrack-max-per-core") - return scaled, nil - } - klog.V(3).Infof("getConntrackMax: using conntrack-min") - return floor, nil - } - return 0, nil -} - -// CleanupAndExit remove iptables rules and ipset/ipvs rules in ipvs proxy mode -// and exit if success return nil -func (s *ProxyServer) CleanupAndExit() error { - // cleanup IPv6 and IPv4 iptables rules - ipts := []utiliptables.Interface{ - utiliptables.New(s.execer, utiliptables.ProtocolIPv4), - utiliptables.New(s.execer, utiliptables.ProtocolIPv6), - } - var encounteredError bool - for _, ipt := range ipts { - encounteredError = userspace.CleanupLeftovers(ipt) || encounteredError - encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError - encounteredError = ipvs.CleanupLeftovers(s.IpvsInterface, ipt, s.IpsetInterface, s.CleanupIPVS) || encounteredError - } - if encounteredError { - return errors.New("encountered an error while tearing down rules") - } - - return nil -} diff --git a/cmd/kube-proxy/app/server_others.go b/cmd/kube-proxy/app/server_others.go deleted file mode 100644 index b44dbb54b2f19..0000000000000 --- a/cmd/kube-proxy/app/server_others.go +++ /dev/null @@ -1,605 +0,0 @@ -// +build !windows - -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to configure and run a -// Kubernetes app process. -package app - -import ( - "context" - "errors" - "fmt" - "net" - "strings" - "time" - - "k8s.io/apimachinery/pkg/watch" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/cache" - - "k8s.io/apimachinery/pkg/fields" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - utilnet "k8s.io/apimachinery/pkg/util/net" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - utilfeature "k8s.io/apiserver/pkg/util/feature" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/record" - toolswatch "k8s.io/client-go/tools/watch" - "k8s.io/component-base/configz" - "k8s.io/component-base/metrics" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/proxy" - proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" - proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" - "k8s.io/kubernetes/pkg/proxy/healthcheck" - "k8s.io/kubernetes/pkg/proxy/iptables" - "k8s.io/kubernetes/pkg/proxy/ipvs" - proxymetrics "k8s.io/kubernetes/pkg/proxy/metrics" - "k8s.io/kubernetes/pkg/proxy/userspace" - proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" - utilipset "k8s.io/kubernetes/pkg/util/ipset" - utiliptables "k8s.io/kubernetes/pkg/util/iptables" - utilipvs "k8s.io/kubernetes/pkg/util/ipvs" - utilnode "k8s.io/kubernetes/pkg/util/node" - utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" - "k8s.io/utils/exec" - utilsnet "k8s.io/utils/net" - - "k8s.io/klog/v2" -) - -// timeoutForNodePodCIDR is the time to wait for allocators to assign a PodCIDR to the -// node after it is registered. -var timeoutForNodePodCIDR = 5 * time.Minute - -// NewProxyServer returns a new ProxyServer. -func NewProxyServer(o *Options) (*ProxyServer, error) { - return newProxyServer(o.config, o.CleanupAndExit, o.master) -} - -func newProxyServer( - config *proxyconfigapi.KubeProxyConfiguration, - cleanupAndExit bool, - master string) (*ProxyServer, error) { - - if config == nil { - return nil, errors.New("config is required") - } - - if c, err := configz.New(proxyconfigapi.GroupName); err == nil { - c.Set(config) - } else { - return nil, fmt.Errorf("unable to register configz: %s", err) - } - - var iptInterface utiliptables.Interface - var ipvsInterface utilipvs.Interface - var kernelHandler ipvs.KernelHandler - var ipsetInterface utilipset.Interface - - // Create a iptables utils. - execer := exec.New() - - kernelHandler = ipvs.NewLinuxKernelHandler() - ipsetInterface = utilipset.New(execer) - canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler) - if string(config.Mode) == proxyModeIPVS && err != nil { - klog.Errorf("Can't use the IPVS proxier: %v", err) - } - - if canUseIPVS { - ipvsInterface = utilipvs.New(execer) - } - - // We omit creation of pretty much everything if we run in cleanup mode - if cleanupAndExit { - return &ProxyServer{ - execer: execer, - IpvsInterface: ipvsInterface, - IpsetInterface: ipsetInterface, - }, nil - } - - if len(config.ShowHiddenMetricsForVersion) > 0 { - metrics.SetShowHidden() - } - - hostname, err := utilnode.GetHostname(config.HostnameOverride) - if err != nil { - return nil, err - } - - client, eventClient, err := createClients(config.ClientConnection, master) - if err != nil { - return nil, err - } - - nodeIP := detectNodeIP(client, hostname, config.BindAddress) - protocol := utiliptables.ProtocolIPv4 - if utilsnet.IsIPv6(nodeIP) { - klog.V(0).Infof("kube-proxy node IP is an IPv6 address (%s), assume IPv6 operation", nodeIP.String()) - protocol = utiliptables.ProtocolIPv6 - } else { - klog.V(0).Infof("kube-proxy node IP is an IPv4 address (%s), assume IPv4 operation", nodeIP.String()) - } - - iptInterface = utiliptables.New(execer, protocol) - - // Create event recorder - eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, v1.EventSource{Component: "kube-proxy", Host: hostname}) - - nodeRef := &v1.ObjectReference{ - Kind: "Node", - Name: hostname, - UID: types.UID(hostname), - Namespace: "", - } - - var healthzServer healthcheck.ProxierHealthUpdater - if len(config.HealthzBindAddress) > 0 { - healthzServer = healthcheck.NewProxierHealthServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef) - } - - var proxier proxy.Provider - var detectLocalMode proxyconfigapi.LocalMode - - proxyMode := getProxyMode(string(config.Mode), canUseIPVS, iptables.LinuxKernelCompatTester{}) - detectLocalMode, err = getDetectLocalMode(config) - if err != nil { - return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err) - } - - var nodeInfo *v1.Node - if detectLocalMode == proxyconfigapi.LocalModeNodeCIDR { - klog.Infof("Watching for node %s, awaiting podCIDR allocation", hostname) - nodeInfo, err = waitForPodCIDR(client, hostname) - if err != nil { - return nil, err - } - klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs) - } - - klog.V(2).Info("DetectLocalMode: '", string(detectLocalMode), "'") - - if proxyMode == proxyModeIPTables { - klog.V(0).Info("Using iptables Proxier.") - if config.IPTables.MasqueradeBit == nil { - // MasqueradeBit must be specified or defaulted. - return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") - } - - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - klog.V(0).Info("creating dualStackProxier for iptables.") - - // Create iptables handlers for both families, one is already created - // Always ordered as IPv4, IPv6 - var ipt [2]utiliptables.Interface - if iptInterface.IsIPv6() { - ipt[1] = iptInterface - ipt[0] = utiliptables.New(execer, utiliptables.ProtocolIPv4) - } else { - ipt[0] = iptInterface - ipt[1] = utiliptables.New(execer, utiliptables.ProtocolIPv6) - } - - // Always ordered to match []ipt - var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - - // TODO this has side effects that should only happen when Run() is invoked. - proxier, err = iptables.NewDualStackProxier( - ipt, - utilsysctl.New(), - execer, - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - localDetectors, - hostname, - nodeIPTuple(config.BindAddress), - recorder, - healthzServer, - config.NodePortAddresses, - ) - } else { // Create a single-stack proxier. - var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - - // TODO this has side effects that should only happen when Run() is invoked. - proxier, err = iptables.NewProxier( - iptInterface, - utilsysctl.New(), - execer, - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - localDetector, - hostname, - nodeIP, - recorder, - healthzServer, - config.NodePortAddresses, - ) - } - - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - proxymetrics.RegisterMetrics() - } else if proxyMode == proxyModeIPVS { - klog.V(0).Info("Using ipvs Proxier.") - if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) { - klog.V(0).Info("creating dualStackProxier for ipvs.") - - // Create iptables handlers for both families, one is already created - // Always ordered as IPv4, IPv6 - var ipt [2]utiliptables.Interface - if iptInterface.IsIPv6() { - ipt[1] = iptInterface - ipt[0] = utiliptables.New(execer, utiliptables.ProtocolIPv4) - } else { - ipt[0] = iptInterface - ipt[1] = utiliptables.New(execer, utiliptables.ProtocolIPv6) - } - - nodeIPs := nodeIPTuple(config.BindAddress) - - // Always ordered to match []ipt - var localDetectors [2]proxyutiliptables.LocalTrafficDetector - localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - - proxier, err = ipvs.NewDualStackProxier( - ipt, - ipvsInterface, - ipsetInterface, - utilsysctl.New(), - execer, - config.IPVS.SyncPeriod.Duration, - config.IPVS.MinSyncPeriod.Duration, - config.IPVS.ExcludeCIDRs, - config.IPVS.StrictARP, - config.IPVS.TCPTimeout.Duration, - config.IPVS.TCPFinTimeout.Duration, - config.IPVS.UDPTimeout.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - localDetectors, - hostname, - nodeIPs, - recorder, - healthzServer, - config.IPVS.Scheduler, - config.NodePortAddresses, - kernelHandler, - ) - } else { - var localDetector proxyutiliptables.LocalTrafficDetector - localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - - proxier, err = ipvs.NewProxier( - iptInterface, - ipvsInterface, - ipsetInterface, - utilsysctl.New(), - execer, - config.IPVS.SyncPeriod.Duration, - config.IPVS.MinSyncPeriod.Duration, - config.IPVS.ExcludeCIDRs, - config.IPVS.StrictARP, - config.IPVS.TCPTimeout.Duration, - config.IPVS.TCPFinTimeout.Duration, - config.IPVS.UDPTimeout.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - localDetector, - hostname, - nodeIP, - recorder, - healthzServer, - config.IPVS.Scheduler, - config.NodePortAddresses, - kernelHandler, - ) - } - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - proxymetrics.RegisterMetrics() - } else { - klog.V(0).Info("Using userspace Proxier.") - - // TODO this has side effects that should only happen when Run() is invoked. - proxier, err = userspace.NewProxier( - userspace.NewLoadBalancerRR(), - net.ParseIP(config.BindAddress), - iptInterface, - execer, - *utilnet.ParsePortRangeOrDie(config.PortRange), - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.UDPIdleTimeout.Duration, - config.NodePortAddresses, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - } - - return &ProxyServer{ - Client: client, - EventClient: eventClient, - IptInterface: iptInterface, - IpvsInterface: ipvsInterface, - IpsetInterface: ipsetInterface, - execer: execer, - Proxier: proxier, - Broadcaster: eventBroadcaster, - Recorder: recorder, - ConntrackConfiguration: config.Conntrack, - Conntracker: &realConntracker{}, - ProxyMode: proxyMode, - NodeRef: nodeRef, - MetricsBindAddress: config.MetricsBindAddress, - BindAddressHardFail: config.BindAddressHardFail, - EnableProfiling: config.EnableProfiling, - OOMScoreAdj: config.OOMScoreAdj, - ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, - HealthzServer: healthzServer, - UseEndpointSlices: utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying), - }, nil -} - -func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, error) { - // since allocators can assign the podCIDR after the node registers, we do a watch here to wait - // for podCIDR to be assigned, instead of assuming that the Get() on startup will have it. - ctx, cancelFunc := context.WithTimeout(context.TODO(), timeoutForNodePodCIDR) - defer cancelFunc() - - fieldSelector := fields.OneTermEqualSelector("metadata.name", nodeName).String() - lw := &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) { - options.FieldSelector = fieldSelector - return client.CoreV1().Nodes().List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { - options.FieldSelector = fieldSelector - return client.CoreV1().Nodes().Watch(ctx, options) - }, - } - condition := func(event watch.Event) (bool, error) { - if n, ok := event.Object.(*v1.Node); ok { - return n.Spec.PodCIDR != "" && len(n.Spec.PodCIDRs) > 0, nil - } - return false, fmt.Errorf("event object not of type Node") - } - - evt, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition) - if err != nil { - return nil, fmt.Errorf("timeout waiting for PodCIDR allocation to configure detect-local-mode %v: %v", proxyconfigapi.LocalModeNodeCIDR, err) - } - if n, ok := evt.Object.(*v1.Node); ok { - return n, nil - } - return nil, fmt.Errorf("event object not of type node") -} - -// detectNodeIP returns the nodeIP used by the proxier -// The order of precedence is: -// 1. config.bindAddress if bindAddress is not 0.0.0.0 or :: -// 2. the primary IP from the Node object, if set -// 3. if no IP is found it defaults to 127.0.0.1 and IPv4 -func detectNodeIP(client clientset.Interface, hostname, bindAddress string) net.IP { - nodeIP := net.ParseIP(bindAddress) - if nodeIP.IsUnspecified() { - nodeIP = utilnode.GetNodeIP(client, hostname) - } - if nodeIP == nil { - klog.V(0).Infof("can't determine this node's IP, assuming 127.0.0.1; if this is incorrect, please set the --bind-address flag") - nodeIP = net.ParseIP("127.0.0.1") - } - return nodeIP -} - -func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) { - mode := config.DetectLocalMode - switch mode { - case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR: - return mode, nil - default: - if strings.TrimSpace(mode.String()) != "" { - return mode, fmt.Errorf("unknown detect-local-mode: %v", mode) - } - klog.V(4).Info("Defaulting detect-local-mode to ", string(proxyconfigapi.LocalModeClusterCIDR)) - return proxyconfigapi.LocalModeClusterCIDR, nil - } -} - -func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) { - switch mode { - case proxyconfigapi.LocalModeClusterCIDR: - if len(strings.TrimSpace(config.ClusterCIDR)) == 0 { - klog.Warning("detect-local-mode set to ClusterCIDR, but no cluster CIDR defined") - break - } - return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt) - case proxyconfigapi.LocalModeNodeCIDR: - if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { - klog.Warning("detect-local-mode set to NodeCIDR, but no PodCIDR defined at node") - break - } - return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt) - } - klog.V(0).Info("detect-local-mode: ", string(mode), " , defaulting to no-op detect-local") - return proxyutiliptables.NewNoOpLocalDetector(), nil -} - -func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) { - var err error - localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()} - switch mode { - case proxyconfigapi.LocalModeClusterCIDR: - if len(strings.TrimSpace(config.ClusterCIDR)) == 0 { - klog.Warning("detect-local-mode set to ClusterCIDR, but no cluster CIDR defined") - break - } - - clusterCIDRs := cidrTuple(config.ClusterCIDR) - - if len(strings.TrimSpace(clusterCIDRs[0])) == 0 { - klog.Warning("detect-local-mode set to ClusterCIDR, but no IPv4 cluster CIDR defined, defaulting to no-op detect-local for IPv4") - } else { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(clusterCIDRs[0], ipt[0]) - if err != nil { // don't loose the original error - return localDetectors, err - } - } - - if len(strings.TrimSpace(clusterCIDRs[1])) == 0 { - klog.Warning("detect-local-mode set to ClusterCIDR, but no IPv6 cluster CIDR defined, , defaulting to no-op detect-local for IPv6") - } else { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(clusterCIDRs[1], ipt[1]) - } - return localDetectors, err - case proxyconfigapi.LocalModeNodeCIDR: - if nodeInfo == nil || len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 { - klog.Warning("No node info available to configure detect-local-mode NodeCIDR") - break - } - // localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR. - // so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1] - if utilsnet.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1]) - if err != nil { - return localDetectors, err - } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0]) - } - } else { - localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0]) - if err != nil { - return localDetectors, err - } - if len(nodeInfo.Spec.PodCIDRs) > 1 { - localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1]) - } - } - return localDetectors, err - default: - klog.Warningf("unknown detect-local-mode: %v", mode) - } - klog.Warning("detect-local-mode: ", string(mode), " , defaulting to no-op detect-local") - return localDetectors, nil -} - -// cidrTuple takes a comma separated list of CIDRs and return a tuple (ipv4cidr,ipv6cidr) -// The returned tuple is guaranteed to have the order (ipv4,ipv6) and if no cidr from a family is found an -// empty string "" is inserted. -func cidrTuple(cidrList string) [2]string { - cidrs := [2]string{"", ""} - foundIPv4 := false - foundIPv6 := false - - for _, cidr := range strings.Split(cidrList, ",") { - if utilsnet.IsIPv6CIDRString(cidr) && !foundIPv6 { - cidrs[1] = cidr - foundIPv6 = true - } else if !foundIPv4 { - cidrs[0] = cidr - foundIPv4 = true - } - if foundIPv6 && foundIPv4 { - break - } - } - - return cidrs -} - -// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6) -// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address -// will have "any" address (0.0.0.0 or ::) inserted. -func nodeIPTuple(bindAddress string) [2]net.IP { - nodes := [2]net.IP{net.IPv4zero, net.IPv6zero} - - adr := net.ParseIP(bindAddress) - if utilsnet.IsIPv6(adr) { - nodes[1] = adr - } else { - nodes[0] = adr - } - - return nodes -} - -func getProxyMode(proxyMode string, canUseIPVS bool, kcompat iptables.KernelCompatTester) string { - switch proxyMode { - case proxyModeUserspace: - return proxyModeUserspace - case proxyModeIPTables: - return tryIPTablesProxy(kcompat) - case proxyModeIPVS: - return tryIPVSProxy(canUseIPVS, kcompat) - } - klog.Warningf("Unknown proxy mode %q, assuming iptables proxy", proxyMode) - return tryIPTablesProxy(kcompat) -} - -func tryIPVSProxy(canUseIPVS bool, kcompat iptables.KernelCompatTester) string { - if canUseIPVS { - return proxyModeIPVS - } - - // Try to fallback to iptables before falling back to userspace - klog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier") - return tryIPTablesProxy(kcompat) -} - -func tryIPTablesProxy(kcompat iptables.KernelCompatTester) string { - // guaranteed false on error, error only necessary for debugging - useIPTablesProxy, err := iptables.CanUseIPTablesProxier(kcompat) - if err != nil { - utilruntime.HandleError(fmt.Errorf("can't determine whether to use iptables proxy, using userspace proxier: %v", err)) - return proxyModeUserspace - } - if useIPTablesProxy { - return proxyModeIPTables - } - // Fallback. - klog.V(1).Infof("Can't use iptables proxy, using userspace proxier") - return proxyModeUserspace -} diff --git a/cmd/kube-proxy/app/server_others_test.go b/cmd/kube-proxy/app/server_others_test.go deleted file mode 100644 index 8cdb354f82462..0000000000000 --- a/cmd/kube-proxy/app/server_others_test.go +++ /dev/null @@ -1,672 +0,0 @@ -// +build !windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "net" - "reflect" - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - clientsetfake "k8s.io/client-go/kubernetes/fake" - - proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" - "k8s.io/kubernetes/pkg/proxy/ipvs" - proxyutiliptables "k8s.io/kubernetes/pkg/proxy/util/iptables" - utiliptables "k8s.io/kubernetes/pkg/util/iptables" - utiliptablestest "k8s.io/kubernetes/pkg/util/iptables/testing" -) - -type fakeIPSetVersioner struct { - version string // what to return - err error // what to return -} - -func (fake *fakeIPSetVersioner) GetVersion() (string, error) { - return fake.version, fake.err -} - -type fakeKernelCompatTester struct { - ok bool -} - -func (fake *fakeKernelCompatTester) IsCompatible() error { - if !fake.ok { - return fmt.Errorf("error") - } - return nil -} - -// fakeKernelHandler implements KernelHandler. -type fakeKernelHandler struct { - modules []string - kernelVersion string -} - -func (fake *fakeKernelHandler) GetModules() ([]string, error) { - return fake.modules, nil -} - -func (fake *fakeKernelHandler) GetKernelVersion() (string, error) { - return fake.kernelVersion, nil -} - -func Test_getProxyMode(t *testing.T) { - var cases = []struct { - flag string - ipsetVersion string - kmods []string - kernelVersion string - kernelCompat bool - ipsetError error - expected string - scheduler string - }{ - { // flag says userspace - flag: "userspace", - expected: proxyModeUserspace, - }, - { // flag says iptables, kernel not compatible - flag: "iptables", - kernelCompat: false, - expected: proxyModeUserspace, - }, - { // flag says iptables, kernel is compatible - flag: "iptables", - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // detect, kernel not compatible - flag: "", - kernelCompat: false, - expected: proxyModeUserspace, - }, - { // detect, kernel is compatible - flag: "", - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, ipset version ok, kernel modules installed for linux kernel before 4.19 - flag: "ipvs", - kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"}, - kernelVersion: "4.18", - ipsetVersion: ipvs.MinIPSetCheckVersion, - expected: proxyModeIPVS, - scheduler: "rr", - }, - { // flag says ipvs, ipset version ok, kernel modules installed for linux kernel 4.19 - flag: "ipvs", - kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, - kernelVersion: "4.19", - ipsetVersion: ipvs.MinIPSetCheckVersion, - expected: proxyModeIPVS, - scheduler: "rr", - }, - { // flag says ipvs, ipset version too low, fallback on iptables mode - flag: "ipvs", - kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, - kernelVersion: "4.19", - ipsetVersion: "0.0", - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, bad ipset version, fallback on iptables mode - flag: "ipvs", - kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, - kernelVersion: "4.19", - ipsetVersion: "a.b.c", - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, required kernel modules are not installed, fallback on iptables mode - flag: "ipvs", - kmods: []string{"foo", "bar", "baz"}, - kernelVersion: "4.19", - ipsetVersion: ipvs.MinIPSetCheckVersion, - kernelCompat: true, - expected: proxyModeIPTables, - }, - { // flag says ipvs, ipset version ok, kernel modules installed for sed scheduler - flag: "ipvs", - kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_sed"}, - kernelVersion: "4.19", - ipsetVersion: ipvs.MinIPSetCheckVersion, - expected: proxyModeIPVS, - scheduler: "sed", - }, - { // flag says ipvs, kernel modules not installed for sed scheduler, fallback to iptables - flag: "ipvs", - kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"}, - kernelVersion: "4.19", - ipsetVersion: ipvs.MinIPSetCheckVersion, - expected: proxyModeIPTables, - kernelCompat: true, - scheduler: "sed", - }, - } - for i, c := range cases { - kcompater := &fakeKernelCompatTester{c.kernelCompat} - ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError} - khandler := &fakeKernelHandler{ - modules: c.kmods, - kernelVersion: c.kernelVersion, - } - canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver, cases[i].scheduler) - r := getProxyMode(c.flag, canUseIPVS, kcompater) - if r != c.expected { - t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r) - } - } -} - -func Test_getDetectLocalMode(t *testing.T) { - cases := []struct { - detectLocal string - expected proxyconfigapi.LocalMode - errExpected bool - }{ - { - detectLocal: "", - expected: proxyconfigapi.LocalModeClusterCIDR, - errExpected: false, - }, - { - detectLocal: string(proxyconfigapi.LocalModeClusterCIDR), - expected: proxyconfigapi.LocalModeClusterCIDR, - errExpected: false, - }, - { - detectLocal: "abcd", - expected: proxyconfigapi.LocalMode("abcd"), - errExpected: true, - }, - } - for i, c := range cases { - proxyConfig := &proxyconfigapi.KubeProxyConfiguration{DetectLocalMode: proxyconfigapi.LocalMode(c.detectLocal)} - r, err := getDetectLocalMode(proxyConfig) - if c.errExpected { - if err == nil { - t.Errorf("Expected error, but did not fail for mode %v", c.detectLocal) - } - continue - } - if err != nil { - t.Errorf("Got error parsing mode: %v", err) - continue - } - if r != c.expected { - t.Errorf("Case[%d] Expected %q got %q", i, c.expected, r) - } - } -} - -func Test_detectNodeIP(t *testing.T) { - cases := []struct { - name string - nodeInfo *v1.Node - hostname string - bindAddress string - expectedIP net.IP - }{ - { - name: "Bind address IPv4 unicast address and no Node object", - nodeInfo: makeNodeWithAddresses("", "", ""), - hostname: "fakeHost", - bindAddress: "10.0.0.1", - expectedIP: net.ParseIP("10.0.0.1"), - }, - { - name: "Bind address IPv6 unicast address and no Node object", - nodeInfo: makeNodeWithAddresses("", "", ""), - hostname: "fakeHost", - bindAddress: "fd00:4321::2", - expectedIP: net.ParseIP("fd00:4321::2"), - }, - { - name: "No Valid IP found", - nodeInfo: makeNodeWithAddresses("", "", ""), - hostname: "fakeHost", - bindAddress: "", - expectedIP: net.ParseIP("127.0.0.1"), - }, - // Disabled because the GetNodeIP method has a backoff retry mechanism - // and the test takes more than 30 seconds - // ok k8s.io/kubernetes/cmd/kube-proxy/app 34.136s - // { - // name: "No Valid IP found and unspecified bind address", - // nodeInfo: makeNodeWithAddresses("", "", ""), - // hostname: "fakeHost", - // bindAddress: "0.0.0.0", - // expectedIP: net.ParseIP("127.0.0.1"), - // }, - { - name: "Bind address 0.0.0.0 and node with IPv4 InternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"), - hostname: "fakeHost", - bindAddress: "0.0.0.0", - expectedIP: net.ParseIP("192.168.1.1"), - }, - { - name: "Bind address :: and node with IPv4 InternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "192.168.1.1", "90.90.90.90"), - hostname: "fakeHost", - bindAddress: "::", - expectedIP: net.ParseIP("192.168.1.1"), - }, - { - name: "Bind address 0.0.0.0 and node with IPv6 InternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"), - hostname: "fakeHost", - bindAddress: "0.0.0.0", - expectedIP: net.ParseIP("fd00:1234::1"), - }, - { - name: "Bind address :: and node with IPv6 InternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "fd00:1234::1", "2001:db8::2"), - hostname: "fakeHost", - bindAddress: "::", - expectedIP: net.ParseIP("fd00:1234::1"), - }, - { - name: "Bind address 0.0.0.0 and node with only IPv4 ExternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "", "90.90.90.90"), - hostname: "fakeHost", - bindAddress: "0.0.0.0", - expectedIP: net.ParseIP("90.90.90.90"), - }, - { - name: "Bind address :: and node with only IPv4 ExternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "", "90.90.90.90"), - hostname: "fakeHost", - bindAddress: "::", - expectedIP: net.ParseIP("90.90.90.90"), - }, - { - name: "Bind address 0.0.0.0 and node with only IPv6 ExternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "", "2001:db8::2"), - hostname: "fakeHost", - bindAddress: "0.0.0.0", - expectedIP: net.ParseIP("2001:db8::2"), - }, - { - name: "Bind address :: and node with only IPv6 ExternalIP set", - nodeInfo: makeNodeWithAddresses("fakeHost", "", "2001:db8::2"), - hostname: "fakeHost", - bindAddress: "::", - expectedIP: net.ParseIP("2001:db8::2"), - }, - } - for _, c := range cases { - client := clientsetfake.NewSimpleClientset(c.nodeInfo) - ip := detectNodeIP(client, c.hostname, c.bindAddress) - if !ip.Equal(c.expectedIP) { - t.Errorf("Case[%s] Expected IP %q got %q", c.name, c.expectedIP, ip) - } - } -} - -func Test_getLocalDetector(t *testing.T) { - cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt utiliptables.Interface - expected proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool - }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/14", utiliptablestest.NewFake())), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/64", utiliptablestest.NewIPv6Fake())), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - errExpected: false, - }, - // LocalModeNodeCIDR - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0"), - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101"), - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewIPv6Fake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: utiliptablestest.NewFake(), - expected: nil, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: true, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, - }, - // unknown mode, nodeInfo would be nil for these cases - { - mode: proxyconfigapi.LocalMode("abcd"), - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: utiliptablestest.NewFake(), - expected: proxyutiliptables.NewNoOpLocalDetector(), - errExpected: false, - }, - } - for i, c := range cases { - r, err := getLocalDetector(c.mode, c.config, c.ipt, c.nodeInfo) - if c.errExpected { - if err == nil { - t.Errorf("Case[%d] Expected error, but succeeded with %v", i, r) - } - continue - } - if err != nil { - t.Errorf("Case[%d] Error resolving detect-local: %v", i, err) - continue - } - if !reflect.DeepEqual(r, c.expected) { - t.Errorf("Case[%d] Unexpected detect-local implementation, expected: %q, got: %q", i, c.expected, r) - } - } -} - -func Test_getDualStackLocalDetectorTuple(t *testing.T) { - cases := []struct { - mode proxyconfigapi.LocalMode - config *proxyconfigapi.KubeProxyConfiguration - ipt [2]utiliptables.Interface - expected [2]proxyutiliptables.LocalTrafficDetector - nodeInfo *v1.Node - errExpected bool - }{ - // LocalModeClusterCIDR, nodeInfo would be nil for these cases - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: resolveDualStackLocalDetectors(t)( - proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/14", utiliptablestest.NewFake()))( - proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/64", utiliptablestest.NewIPv6Fake())), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64,10.0.0.0/14"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: resolveDualStackLocalDetectors(t)( - proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/14", utiliptablestest.NewFake()))( - proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/64", utiliptablestest.NewIPv6Fake())), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{ - resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/14", utiliptablestest.NewFake())), - proxyutiliptables.NewNoOpLocalDetector()}, - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{ - proxyutiliptables.NewNoOpLocalDetector(), - resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/64", utiliptablestest.NewIPv6Fake()))}, - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeClusterCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - errExpected: false, - }, - // LocalModeNodeCIDR - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14,2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: resolveDualStackLocalDetectors(t)( - proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( - proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24", "2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64,10.0.0.0/14"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: resolveDualStackLocalDetectors(t)( - proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake()))( - proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake())), - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96", "10.0.0.0/24"), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "10.0.0.0/14"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{ - resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("10.0.0.0/24", utiliptablestest.NewFake())), - proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs("10.0.0.0/24"), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: "2002::1234:abcd:ffff:c0a8:101/64"}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{ - proxyutiliptables.NewNoOpLocalDetector(), - resolveLocalDetector(t)(proxyutiliptables.NewDetectLocalByCIDR("2002::1234:abcd:ffff:c0a8:101/96", utiliptablestest.NewIPv6Fake()))}, - nodeInfo: makeNodeWithPodCIDRs("2002::1234:abcd:ffff:c0a8:101/96"), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: makeNodeWithPodCIDRs(), - errExpected: false, - }, - { - mode: proxyconfigapi.LocalModeNodeCIDR, - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - nodeInfo: nil, - errExpected: false, - }, - // unknown mode, nodeInfo would be nil for these cases - { - mode: proxyconfigapi.LocalMode("abcd"), - config: &proxyconfigapi.KubeProxyConfiguration{ClusterCIDR: ""}, - ipt: [2]utiliptables.Interface{utiliptablestest.NewFake(), utiliptablestest.NewIPv6Fake()}, - expected: [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}, - errExpected: false, - }, - } - for i, c := range cases { - r, err := getDualStackLocalDetectorTuple(c.mode, c.config, c.ipt, c.nodeInfo) - if c.errExpected { - if err == nil { - t.Errorf("Case[%d] expected error, but succeeded with %q", i, r) - } - continue - } - if err != nil { - t.Errorf("Case[%d] Error resolving detect-local: %v", i, err) - continue - } - if !reflect.DeepEqual(r, c.expected) { - t.Errorf("Case[%d] Unexpected detect-local implementation, expected: %q, got: %q", i, c.expected, r) - } - } -} - -func makeNodeWithAddresses(name, internal, external string) *v1.Node { - if name == "" { - return &v1.Node{} - } - - node := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{}, - }, - } - - if internal != "" { - node.Status.Addresses = append(node.Status.Addresses, - v1.NodeAddress{Type: v1.NodeInternalIP, Address: internal}, - ) - } - - if external != "" { - node.Status.Addresses = append(node.Status.Addresses, - v1.NodeAddress{Type: v1.NodeExternalIP, Address: external}, - ) - } - - return node -} - -func makeNodeWithPodCIDRs(cidrs ...string) *v1.Node { - if len(cidrs) == 0 { - return &v1.Node{} - } - return &v1.Node{ - Spec: v1.NodeSpec{ - PodCIDR: cidrs[0], - PodCIDRs: cidrs, - }, - } -} - -func resolveLocalDetector(t *testing.T) func(proxyutiliptables.LocalTrafficDetector, error) proxyutiliptables.LocalTrafficDetector { - return func(localDetector proxyutiliptables.LocalTrafficDetector, err error) proxyutiliptables.LocalTrafficDetector { - t.Helper() - if err != nil { - t.Fatalf("Error resolving detect-local: %v", err) - } - return localDetector - } -} - -func resolveDualStackLocalDetectors(t *testing.T) func(localDetector proxyutiliptables.LocalTrafficDetector, err1 error) func(proxyutiliptables.LocalTrafficDetector, error) [2]proxyutiliptables.LocalTrafficDetector { - return func(localDetector proxyutiliptables.LocalTrafficDetector, err error) func(proxyutiliptables.LocalTrafficDetector, error) [2]proxyutiliptables.LocalTrafficDetector { - t.Helper() - if err != nil { - t.Fatalf("Error resolving dual stack detect-local: %v", err) - } - return func(otherLocalDetector proxyutiliptables.LocalTrafficDetector, err1 error) [2]proxyutiliptables.LocalTrafficDetector { - t.Helper() - if err1 != nil { - t.Fatalf("Error resolving dual stack detect-local: %v", err) - } - return [2]proxyutiliptables.LocalTrafficDetector{localDetector, otherLocalDetector} - } - } -} diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go deleted file mode 100644 index 0bd5350eef12a..0000000000000 --- a/cmd/kube-proxy/app/server_test.go +++ /dev/null @@ -1,627 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "runtime" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - - "github.com/stretchr/testify/assert" - - utilpointer "k8s.io/utils/pointer" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - componentbaseconfig "k8s.io/component-base/config" - kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" -) - -func TestGetConntrackMax(t *testing.T) { - ncores := runtime.NumCPU() - testCases := []struct { - min int32 - maxPerCore int32 - expected int - err string - }{ - { - expected: 0, - }, - { - maxPerCore: 67890, // use this if Max is 0 - min: 1, // avoid 0 default - expected: 67890 * ncores, - }, - { - maxPerCore: 1, // ensure that Min is considered - min: 123456, - expected: 123456, - }, - { - maxPerCore: 0, // leave system setting - min: 123456, - expected: 0, - }, - } - - for i, tc := range testCases { - cfg := kubeproxyconfig.KubeProxyConntrackConfiguration{ - Min: utilpointer.Int32Ptr(tc.min), - MaxPerCore: utilpointer.Int32Ptr(tc.maxPerCore), - } - x, e := getConntrackMax(cfg) - if e != nil { - if tc.err == "" { - t.Errorf("[%d] unexpected error: %v", i, e) - } else if !strings.Contains(e.Error(), tc.err) { - t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e) - } - } else if x != tc.expected { - t.Errorf("[%d] expected %d, got %d", i, tc.expected, x) - } - } -} - -// TestLoadConfig tests proper operation of loadConfig() -func TestLoadConfig(t *testing.T) { - - yamlTemplate := `apiVersion: kubeproxy.config.k8s.io/v1alpha1 -bindAddress: %s -clientConnection: - acceptContentTypes: "abc" - burst: 100 - contentType: content-type - kubeconfig: "/path/to/kubeconfig" - qps: 7 -clusterCIDR: "%s" -configSyncPeriod: 15s -conntrack: - maxPerCore: 2 - min: 1 - tcpCloseWaitTimeout: 10s - tcpEstablishedTimeout: 20s -healthzBindAddress: "%s" -hostnameOverride: "foo" -iptables: - masqueradeAll: true - masqueradeBit: 17 - minSyncPeriod: 10s - syncPeriod: 60s -ipvs: - minSyncPeriod: 10s - syncPeriod: 60s - excludeCIDRs: - - "10.20.30.40/16" - - "fd00:1::0/64" -kind: KubeProxyConfiguration -metricsBindAddress: "%s" -mode: "%s" -oomScoreAdj: 17 -portRange: "2-7" -udpIdleTimeout: 123ms -detectLocalMode: "ClusterCIDR" -nodePortAddresses: - - "10.20.30.40/16" - - "fd00:1::0/64" -` - - testCases := []struct { - name string - mode string - bindAddress string - clusterCIDR string - healthzBindAddress string - metricsBindAddress string - extraConfig string - }{ - { - name: "iptables mode, IPv4 all-zeros bind address", - mode: "iptables", - bindAddress: "0.0.0.0", - clusterCIDR: "1.2.3.0/24", - healthzBindAddress: "1.2.3.4:12345", - metricsBindAddress: "2.3.4.5:23456", - }, - { - name: "iptables mode, non-zeros IPv4 config", - mode: "iptables", - bindAddress: "9.8.7.6", - clusterCIDR: "1.2.3.0/24", - healthzBindAddress: "1.2.3.4:12345", - metricsBindAddress: "2.3.4.5:23456", - }, - { - // Test for 'bindAddress: "::"' (IPv6 all-zeros) in kube-proxy - // config file. The user will need to put quotes around '::' since - // 'bindAddress: ::' is invalid yaml syntax. - name: "iptables mode, IPv6 \"::\" bind address", - mode: "iptables", - bindAddress: "\"::\"", - clusterCIDR: "fd00:1::0/64", - healthzBindAddress: "[fd00:1::5]:12345", - metricsBindAddress: "[fd00:2::5]:23456", - }, - { - // Test for 'bindAddress: "[::]"' (IPv6 all-zeros in brackets) - // in kube-proxy config file. The user will need to use - // surrounding quotes here since 'bindAddress: [::]' is invalid - // yaml syntax. - name: "iptables mode, IPv6 \"[::]\" bind address", - mode: "iptables", - bindAddress: "\"[::]\"", - clusterCIDR: "fd00:1::0/64", - healthzBindAddress: "[fd00:1::5]:12345", - metricsBindAddress: "[fd00:2::5]:23456", - }, - { - // Test for 'bindAddress: ::0' (another form of IPv6 all-zeros). - // No surrounding quotes are required around '::0'. - name: "iptables mode, IPv6 ::0 bind address", - mode: "iptables", - bindAddress: "::0", - clusterCIDR: "fd00:1::0/64", - healthzBindAddress: "[fd00:1::5]:12345", - metricsBindAddress: "[fd00:2::5]:23456", - }, - { - name: "ipvs mode, IPv6 config", - mode: "ipvs", - bindAddress: "2001:db8::1", - clusterCIDR: "fd00:1::0/64", - healthzBindAddress: "[fd00:1::5]:12345", - metricsBindAddress: "[fd00:2::5]:23456", - }, - { - // Test for unknown field within config. - // For v1alpha1 a lenient path is implemented and will throw a - // strict decoding warning instead of failing to load - name: "unknown field", - mode: "iptables", - bindAddress: "9.8.7.6", - clusterCIDR: "1.2.3.0/24", - healthzBindAddress: "1.2.3.4:12345", - metricsBindAddress: "2.3.4.5:23456", - extraConfig: "foo: bar", - }, - { - // Test for duplicate field within config. - // For v1alpha1 a lenient path is implemented and will throw a - // strict decoding warning instead of failing to load - name: "duplicate field", - mode: "iptables", - bindAddress: "9.8.7.6", - clusterCIDR: "1.2.3.0/24", - healthzBindAddress: "1.2.3.4:12345", - metricsBindAddress: "2.3.4.5:23456", - extraConfig: "bindAddress: 9.8.7.6", - }, - } - - for _, tc := range testCases { - expBindAddr := tc.bindAddress - if tc.bindAddress[0] == '"' { - // Surrounding double quotes will get stripped by the yaml parser. - expBindAddr = expBindAddr[1 : len(tc.bindAddress)-1] - } - expected := &kubeproxyconfig.KubeProxyConfiguration{ - BindAddress: expBindAddr, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - AcceptContentTypes: "abc", - Burst: 100, - ContentType: "content-type", - Kubeconfig: "/path/to/kubeconfig", - QPS: 7, - }, - ClusterCIDR: tc.clusterCIDR, - ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second}, - Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ - MaxPerCore: utilpointer.Int32Ptr(2), - Min: utilpointer.Int32Ptr(1), - TCPCloseWaitTimeout: &metav1.Duration{Duration: 10 * time.Second}, - TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second}, - }, - FeatureGates: map[string]bool{}, - HealthzBindAddress: tc.healthzBindAddress, - HostnameOverride: "foo", - IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{ - MasqueradeAll: true, - MasqueradeBit: utilpointer.Int32Ptr(17), - MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second}, - SyncPeriod: metav1.Duration{Duration: 60 * time.Second}, - }, - IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ - MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second}, - SyncPeriod: metav1.Duration{Duration: 60 * time.Second}, - ExcludeCIDRs: []string{"10.20.30.40/16", "fd00:1::0/64"}, - }, - MetricsBindAddress: tc.metricsBindAddress, - Mode: kubeproxyconfig.ProxyMode(tc.mode), - OOMScoreAdj: utilpointer.Int32Ptr(17), - PortRange: "2-7", - UDPIdleTimeout: metav1.Duration{Duration: 123 * time.Millisecond}, - NodePortAddresses: []string{"10.20.30.40/16", "fd00:1::0/64"}, - DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR, - } - - options := NewOptions() - - baseYAML := fmt.Sprintf( - yamlTemplate, tc.bindAddress, tc.clusterCIDR, - tc.healthzBindAddress, tc.metricsBindAddress, tc.mode) - - // Append additional configuration to the base yaml template - yaml := fmt.Sprintf("%s\n%s", baseYAML, tc.extraConfig) - - config, err := options.loadConfig([]byte(yaml)) - - assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err) - - if !reflect.DeepEqual(expected, config) { - t.Fatalf("unexpected config for %s, diff = %s", tc.name, cmp.Diff(config, expected)) - } - } -} - -// TestLoadConfigFailures tests failure modes for loadConfig() -func TestLoadConfigFailures(t *testing.T) { - // TODO(phenixblue): Uncomment below template when v1alpha2+ of kube-proxy config is - // released with strict decoding. These associated tests will fail with - // the lenient codec and only one config API version. - /* - yamlTemplate := `bindAddress: 0.0.0.0 - clusterCIDR: "1.2.3.0/24" - configSyncPeriod: 15s - kind: KubeProxyConfiguration` - */ - - testCases := []struct { - name string - config string - expErr string - checkFn func(err error) bool - }{ - { - name: "Decode error test", - config: "Twas bryllyg, and ye slythy toves", - expErr: "could not find expected ':'", - }, - { - name: "Bad config type test", - config: "kind: KubeSchedulerConfiguration", - expErr: "no kind", - }, - { - name: "Missing quotes around :: bindAddress", - config: "bindAddress: ::", - expErr: "mapping values are not allowed in this context", - }, - // TODO(phenixblue): Uncomment below tests when v1alpha2+ of kube-proxy config is - // released with strict decoding. These tests will fail with the - // lenient codec and only one config API version. - /* - { - name: "Duplicate fields", - config: fmt.Sprintf("%s\nbindAddress: 1.2.3.4", yamlTemplate), - checkFn: kuberuntime.IsStrictDecodingError, - }, - { - name: "Unknown field", - config: fmt.Sprintf("%s\nfoo: bar", yamlTemplate), - checkFn: kuberuntime.IsStrictDecodingError, - }, - */ - } - - version := "apiVersion: kubeproxy.config.k8s.io/v1alpha1" - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - options := NewOptions() - config := fmt.Sprintf("%s\n%s", version, tc.config) - _, err := options.loadConfig([]byte(config)) - - if assert.Error(t, err, tc.name) { - if tc.expErr != "" { - assert.Contains(t, err.Error(), tc.expErr) - } - if tc.checkFn != nil { - assert.True(t, tc.checkFn(err), tc.name) - } - } - }) - } -} - -// TestProcessHostnameOverrideFlag tests processing hostname-override arg -func TestProcessHostnameOverrideFlag(t *testing.T) { - testCases := []struct { - name string - hostnameOverrideFlag string - expectedHostname string - expectError bool - }{ - { - name: "Hostname from config file", - hostnameOverrideFlag: "", - expectedHostname: "foo", - expectError: false, - }, - { - name: "Hostname from flag", - hostnameOverrideFlag: " bar ", - expectedHostname: "bar", - expectError: false, - }, - { - name: "Hostname is space", - hostnameOverrideFlag: " ", - expectError: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - options := NewOptions() - options.config = &kubeproxyconfig.KubeProxyConfiguration{ - HostnameOverride: "foo", - } - - options.hostnameOverride = tc.hostnameOverrideFlag - - err := options.processHostnameOverrideFlag() - if tc.expectError { - if err == nil { - t.Fatalf("should error for this case %s", tc.name) - } - } else { - assert.NoError(t, err, "unexpected error %v", err) - if tc.expectedHostname != options.config.HostnameOverride { - t.Fatalf("expected hostname: %s, but got: %s", tc.expectedHostname, options.config.HostnameOverride) - } - } - }) - } -} - -func TestConfigChange(t *testing.T) { - setUp := func() (*os.File, string, error) { - tempDir, err := ioutil.TempDir("", "kubeproxy-config-change") - if err != nil { - return nil, "", fmt.Errorf("unable to create temporary directory: %v", err) - } - fullPath := filepath.Join(tempDir, "kube-proxy-config") - file, err := os.Create(fullPath) - if err != nil { - return nil, "", fmt.Errorf("unexpected error when creating temp file: %v", err) - } - - _, err = file.WriteString(`apiVersion: kubeproxy.config.k8s.io/v1alpha1 -bindAddress: 0.0.0.0 -bindAddressHardFail: false -clientConnection: - acceptContentTypes: "" - burst: 10 - contentType: application/vnd.kubernetes.protobuf - kubeconfig: /var/lib/kube-proxy/kubeconfig.conf - qps: 5 -clusterCIDR: 10.244.0.0/16 -configSyncPeriod: 15m0s -conntrack: - maxPerCore: 32768 - min: 131072 - tcpCloseWaitTimeout: 1h0m0s - tcpEstablishedTimeout: 24h0m0s -enableProfiling: false -healthzBindAddress: 0.0.0.0:10256 -hostnameOverride: "" -iptables: - masqueradeAll: false - masqueradeBit: 14 - minSyncPeriod: 0s - syncPeriod: 30s -ipvs: - excludeCIDRs: null - minSyncPeriod: 0s - scheduler: "" - syncPeriod: 30s -kind: KubeProxyConfiguration -metricsBindAddress: 127.0.0.1:10249 -mode: "" -nodePortAddresses: null -oomScoreAdj: -999 -portRange: "" -detectLocalMode: "ClusterCIDR" -udpIdleTimeout: 250ms`) - if err != nil { - return nil, "", fmt.Errorf("unexpected error when writing content to temp kube-proxy config file: %v", err) - } - - return file, tempDir, nil - } - - tearDown := func(file *os.File, tempDir string) { - file.Close() - os.RemoveAll(tempDir) - } - - testCases := []struct { - name string - proxyServer proxyRun - append bool - expectedErr string - }{ - { - name: "update config file", - proxyServer: new(fakeProxyServerLongRun), - append: true, - expectedErr: "content of the proxy server's configuration file was updated", - }, - { - name: "fake error", - proxyServer: new(fakeProxyServerError), - expectedErr: "mocking error from ProxyServer.Run()", - }, - } - - for _, tc := range testCases { - file, tempDir, err := setUp() - if err != nil { - t.Fatalf("unexpected error when setting up environment: %v", err) - } - - opt := NewOptions() - opt.ConfigFile = file.Name() - err = opt.Complete() - if err != nil { - t.Fatal(err) - } - opt.proxyServer = tc.proxyServer - - errCh := make(chan error, 1) - go func() { - errCh <- opt.runLoop() - }() - - if tc.append { - file.WriteString("append fake content") - } - - select { - case err := <-errCh: - if err != nil { - if !strings.Contains(err.Error(), tc.expectedErr) { - t.Errorf("[%s] Expected error containing %v, got %v", tc.name, tc.expectedErr, err) - } - } - case <-time.After(10 * time.Second): - t.Errorf("[%s] Timeout: unable to get any events or internal timeout.", tc.name) - } - tearDown(file, tempDir) - } -} - -type fakeProxyServerLongRun struct{} - -// Run runs the specified ProxyServer. -func (s *fakeProxyServerLongRun) Run() error { - for { - time.Sleep(2 * time.Second) - } -} - -// CleanupAndExit runs in the specified ProxyServer. -func (s *fakeProxyServerLongRun) CleanupAndExit() error { - return nil -} - -type fakeProxyServerError struct{} - -// Run runs the specified ProxyServer. -func (s *fakeProxyServerError) Run() error { - for { - time.Sleep(2 * time.Second) - return fmt.Errorf("mocking error from ProxyServer.Run()") - } -} - -// CleanupAndExit runs in the specified ProxyServer. -func (s *fakeProxyServerError) CleanupAndExit() error { - return errors.New("mocking error from ProxyServer.CleanupAndExit()") -} - -func TestAddressFromDeprecatedFlags(t *testing.T) { - testCases := []struct { - name string - healthzPort int32 - healthzBindAddress string - metricsPort int32 - metricsBindAddress string - expHealthz string - expMetrics string - }{ - { - name: "IPv4 bind address", - healthzBindAddress: "1.2.3.4", - healthzPort: 12345, - metricsBindAddress: "2.3.4.5", - metricsPort: 23456, - expHealthz: "1.2.3.4:12345", - expMetrics: "2.3.4.5:23456", - }, - { - name: "IPv4 bind address has port", - healthzBindAddress: "1.2.3.4:12345", - healthzPort: 23456, - metricsBindAddress: "2.3.4.5:12345", - metricsPort: 23456, - expHealthz: "1.2.3.4:12345", - expMetrics: "2.3.4.5:12345", - }, - { - name: "IPv6 bind address", - healthzBindAddress: "fd00:1::5", - healthzPort: 12345, - metricsBindAddress: "fd00:1::6", - metricsPort: 23456, - expHealthz: "[fd00:1::5]:12345", - expMetrics: "[fd00:1::6]:23456", - }, - { - name: "IPv6 bind address has port", - healthzBindAddress: "[fd00:1::5]:12345", - healthzPort: 56789, - metricsBindAddress: "[fd00:1::6]:56789", - metricsPort: 12345, - expHealthz: "[fd00:1::5]:12345", - expMetrics: "[fd00:1::6]:56789", - }, - { - name: "Invalid IPv6 Config", - healthzBindAddress: "[fd00:1::5]", - healthzPort: 12345, - metricsBindAddress: "[fd00:1::6]", - metricsPort: 56789, - expHealthz: "[fd00:1::5]", - expMetrics: "[fd00:1::6]", - }, - } - - for i := range testCases { - gotHealthz := addressFromDeprecatedFlags(testCases[i].healthzBindAddress, testCases[i].healthzPort) - gotMetrics := addressFromDeprecatedFlags(testCases[i].metricsBindAddress, testCases[i].metricsPort) - - errFn := func(name, except, got string) { - t.Errorf("case %s: expected %v, got %v", name, except, got) - } - - if gotHealthz != testCases[i].expHealthz { - errFn(testCases[i].name, testCases[i].expHealthz, gotHealthz) - } - - if gotMetrics != testCases[i].expMetrics { - errFn(testCases[i].name, testCases[i].expMetrics, gotMetrics) - } - - } -} diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go deleted file mode 100644 index 759f04b088649..0000000000000 --- a/cmd/kube-proxy/app/server_windows.go +++ /dev/null @@ -1,222 +0,0 @@ -// +build windows - -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to configure and run a -// Kubernetes app process. -package app - -import ( - "errors" - "fmt" - "net" - - // Enable pprof HTTP handlers. - _ "net/http/pprof" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - utilnet "k8s.io/apimachinery/pkg/util/net" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/tools/record" - "k8s.io/component-base/configz" - "k8s.io/component-base/metrics" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/proxy" - proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/config" - proxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/config/scheme" - "k8s.io/kubernetes/pkg/proxy/healthcheck" - "k8s.io/kubernetes/pkg/proxy/winkernel" - "k8s.io/kubernetes/pkg/proxy/winuserspace" - utilnetsh "k8s.io/kubernetes/pkg/util/netsh" - utilnode "k8s.io/kubernetes/pkg/util/node" - "k8s.io/utils/exec" - utilsnet "k8s.io/utils/net" -) - -// NewProxyServer returns a new ProxyServer. -func NewProxyServer(o *Options) (*ProxyServer, error) { - return newProxyServer(o.config, o.CleanupAndExit, o.master) -} - -func newProxyServer(config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, master string) (*ProxyServer, error) { - if config == nil { - return nil, errors.New("config is required") - } - - if c, err := configz.New(proxyconfigapi.GroupName); err == nil { - c.Set(config) - } else { - return nil, fmt.Errorf("unable to register configz: %s", err) - } - - // We omit creation of pretty much everything if we run in cleanup mode - if cleanupAndExit { - return &ProxyServer{}, nil - } - - if len(config.ShowHiddenMetricsForVersion) > 0 { - metrics.SetShowHidden() - } - - client, eventClient, err := createClients(config.ClientConnection, master) - if err != nil { - return nil, err - } - - // Create event recorder - hostname, err := utilnode.GetHostname(config.HostnameOverride) - if err != nil { - return nil, err - } - eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder(proxyconfigscheme.Scheme, v1.EventSource{Component: "kube-proxy", Host: hostname}) - - nodeRef := &v1.ObjectReference{ - Kind: "Node", - Name: hostname, - UID: types.UID(hostname), - Namespace: "", - } - - var healthzServer healthcheck.ProxierHealthUpdater - if len(config.HealthzBindAddress) > 0 { - healthzServer = healthcheck.NewProxierHealthServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef) - } - - var proxier proxy.Provider - - proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{}) - if proxyMode == proxyModeKernelspace { - klog.V(0).Info("Using Kernelspace Proxier.") - isIPv6DualStackEnabled := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) - if isIPv6DualStackEnabled { - klog.V(0).Info("creating dualStackProxier for Windows kernel.") - - proxier, err = winkernel.NewDualStackProxier( - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - config.ClusterCIDR, - hostname, - nodeIPTuple(config.BindAddress), - recorder, - healthzServer, - config.Winkernel, - ) - } else { - - proxier, err = winkernel.NewProxier( - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - config.ClusterCIDR, - hostname, - utilnode.GetNodeIP(client, hostname), - recorder, - healthzServer, - config.Winkernel, - ) - - } - - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - } else { - klog.V(0).Info("Using userspace Proxier.") - execer := exec.New() - var netshInterface utilnetsh.Interface - netshInterface = utilnetsh.New(execer) - - proxier, err = winuserspace.NewProxier( - winuserspace.NewLoadBalancerRR(), - net.ParseIP(config.BindAddress), - netshInterface, - *utilnet.ParsePortRangeOrDie(config.PortRange), - // TODO @pires replace below with default values, if applicable - config.IPTables.SyncPeriod.Duration, - config.UDPIdleTimeout.Duration, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - } - - return &ProxyServer{ - Client: client, - EventClient: eventClient, - Proxier: proxier, - Broadcaster: eventBroadcaster, - Recorder: recorder, - ProxyMode: proxyMode, - NodeRef: nodeRef, - MetricsBindAddress: config.MetricsBindAddress, - BindAddressHardFail: config.BindAddressHardFail, - EnableProfiling: config.EnableProfiling, - OOMScoreAdj: config.OOMScoreAdj, - ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, - HealthzServer: healthzServer, - UseEndpointSlices: utilfeature.DefaultFeatureGate.Enabled(features.WindowsEndpointSliceProxying), - }, nil -} - -func getProxyMode(proxyMode string, kcompat winkernel.KernelCompatTester) string { - if proxyMode == proxyModeUserspace { - return proxyModeUserspace - } else if proxyMode == proxyModeKernelspace { - return tryWinKernelSpaceProxy(kcompat) - } - return proxyModeUserspace -} - -func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string { - // Check for Windows Kernel Version if we can support Kernel Space proxy - // Check for Windows Version - - // guaranteed false on error, error only necessary for debugging - useWinKernelProxy, err := winkernel.CanUseWinKernelProxier(kcompat) - if err != nil { - klog.Errorf("Can't determine whether to use windows kernel proxy, using userspace proxier: %v", err) - return proxyModeUserspace - } - if useWinKernelProxy { - return proxyModeKernelspace - } - // Fallback. - klog.V(1).Infof("Can't use winkernel proxy, using userspace proxier") - return proxyModeUserspace -} - -// nodeIPTuple takes an addresses and return a tuple (ipv4,ipv6) -// The returned tuple is guaranteed to have the order (ipv4,ipv6). The address NOT of the passed address -// will have "any" address (0.0.0.0 or ::) inserted. -func nodeIPTuple(bindAddress string) [2]net.IP { - nodes := [2]net.IP{net.IPv4zero, net.IPv6zero} - - adr := net.ParseIP(bindAddress) - if utilsnet.IsIPv6(adr) { - nodes[1] = adr - } else { - nodes[0] = adr - } - - return nodes -} diff --git a/cmd/kube-proxy/proxy.go b/cmd/kube-proxy/proxy.go deleted file mode 100644 index 8e3002c2cc087..0000000000000 --- a/cmd/kube-proxy/proxy.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - goflag "flag" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/kubernetes/cmd/kube-proxy/app" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := app.NewProxyCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/kube-scheduler/BUILD b/cmd/kube-scheduler/BUILD deleted file mode 100644 index 01a3b17d80c9b..0000000000000 --- a/cmd/kube-scheduler/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kube-scheduler", - embed = [":go_default_library"], - pure = "on", - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["scheduler.go"], - importpath = "k8s.io/kubernetes/cmd/kube-scheduler", - deps = [ - "//cmd/kube-scheduler/app:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/clientgo:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-scheduler/app:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kube-scheduler/OWNERS b/cmd/kube-scheduler/OWNERS deleted file mode 100644 index 485eb202d9889..0000000000000 --- a/cmd/kube-scheduler/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- sig-scheduling-maintainers -reviewers: -- sig-scheduling -labels: -- sig/scheduling diff --git a/cmd/kube-scheduler/app/BUILD b/cmd/kube-scheduler/app/BUILD deleted file mode 100644 index 546efbf9c2170..0000000000000 --- a/cmd/kube-scheduler/app/BUILD +++ /dev/null @@ -1,75 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["server.go"], - importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app", - deps = [ - "//cmd/kube-scheduler/app/config:go_default_library", - "//cmd/kube-scheduler/app/options:go_default_library", - "//pkg/scheduler:go_default_library", - "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/framework/runtime:go_default_library", - "//pkg/scheduler/metrics/resources:go_default_library", - "//pkg/scheduler/profile:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/endpoints/filters:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/filters:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/mux:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/routes:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/tools/events:go_default_library", - "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/cli/globalflag:go_default_library", - "//staging/src/k8s.io/component-base/configz:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", - "//staging/src/k8s.io/component-base/term:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-scheduler/app/config:all-srcs", - "//cmd/kube-scheduler/app/options:all-srcs", - "//cmd/kube-scheduler/app/testing:all-srcs", - ], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["server_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kube-scheduler/app/options:go_default_library", - "//pkg/scheduler/apis/config:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) diff --git a/cmd/kube-scheduler/app/config/BUILD b/cmd/kube-scheduler/app/config/BUILD deleted file mode 100644 index cdd87b7b8c1a4..0000000000000 --- a/cmd/kube-scheduler/app/config/BUILD +++ /dev/null @@ -1,41 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["config.go"], - importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/config", - visibility = ["//visibility:public"], - deps = [ - "//pkg/scheduler/apis/config:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", - "//staging/src/k8s.io/client-go/informers:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/events:go_default_library", - "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = ["config_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", - ], -) diff --git a/cmd/kube-scheduler/app/config/config.go b/cmd/kube-scheduler/app/config/config.go deleted file mode 100644 index 9ee3dcb0fc931..0000000000000 --- a/cmd/kube-scheduler/app/config/config.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - apiserver "k8s.io/apiserver/pkg/server" - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/events" - "k8s.io/client-go/tools/leaderelection" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" -) - -// Config has all the context to run a Scheduler -type Config struct { - // ComponentConfig is the scheduler server's configuration object. - ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration - - // LoopbackClientConfig is a config for a privileged loopback connection - LoopbackClientConfig *restclient.Config - - InsecureServing *apiserver.DeprecatedInsecureServingInfo // nil will disable serving on an insecure port - InsecureMetricsServing *apiserver.DeprecatedInsecureServingInfo // non-nil if metrics should be served independently - Authentication apiserver.AuthenticationInfo - Authorization apiserver.AuthorizationInfo - SecureServing *apiserver.SecureServingInfo - - Client clientset.Interface - InformerFactory informers.SharedInformerFactory - - //lint:ignore SA1019 this deprecated field still needs to be used for now. It will be removed once the migration is done. - EventBroadcaster events.EventBroadcasterAdapter - - // LeaderElection is optional. - LeaderElection *leaderelection.LeaderElectionConfig -} - -type completedConfig struct { - *Config -} - -// CompletedConfig same as Config, just to swap private object. -type CompletedConfig struct { - // Embed a private pointer that cannot be instantiated outside of this package. - *completedConfig -} - -// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() CompletedConfig { - cc := completedConfig{c} - - if c.InsecureServing != nil { - c.InsecureServing.Name = "healthz" - } - if c.InsecureMetricsServing != nil { - c.InsecureMetricsServing.Name = "metrics" - } - - apiserver.AuthorizeClientBearerToken(c.LoopbackClientConfig, &c.Authentication, &c.Authorization) - - return CompletedConfig{&cc} -} diff --git a/cmd/kube-scheduler/app/config/config_test.go b/cmd/kube-scheduler/app/config/config_test.go deleted file mode 100644 index 4d206ab0a618f..0000000000000 --- a/cmd/kube-scheduler/app/config/config_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - apiserver "k8s.io/apiserver/pkg/server" -) - -func TestConfigComplete(t *testing.T) { - scenarios := []struct { - name string - want *Config - config *Config - }{ - { - name: "SetInsecureServingName", - want: &Config{ - InsecureServing: &apiserver.DeprecatedInsecureServingInfo{ - Name: "healthz", - }, - }, - config: &Config{ - InsecureServing: &apiserver.DeprecatedInsecureServingInfo{}, - }, - }, - { - name: "SetMetricsInsecureServingName", - want: &Config{ - InsecureMetricsServing: &apiserver.DeprecatedInsecureServingInfo{ - Name: "metrics", - }, - }, - config: &Config{ - InsecureMetricsServing: &apiserver.DeprecatedInsecureServingInfo{}, - }, - }, - } - - for _, scenario := range scenarios { - t.Run(scenario.name, func(t *testing.T) { - cc := scenario.config.Complete() - - returnValue := cc.completedConfig.Config - - if diff := cmp.Diff(scenario.want, returnValue); diff != "" { - t.Errorf("Complete(): Unexpected return value (-want, +got): %s", diff) - } - }) - } -} diff --git a/cmd/kube-scheduler/app/options/BUILD b/cmd/kube-scheduler/app/options/BUILD deleted file mode 100644 index 72577489c51a2..0000000000000 --- a/cmd/kube-scheduler/app/options/BUILD +++ /dev/null @@ -1,84 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "configfile.go", - "deprecated.go", - "insecure_serving.go", - "options.go", - ], - importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/options", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kube-scheduler/app/config:go_default_library", - "//pkg/scheduler:go_default_library", - "//pkg/scheduler/algorithmprovider:go_default_library", - "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/apis/config/scheme:go_default_library", - "//pkg/scheduler/apis/config/v1beta1:go_default_library", - "//pkg/scheduler/apis/config/validation:go_default_library", - "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/tools/events:go_default_library", - "//staging/src/k8s.io/client-go/tools/leaderelection:go_default_library", - "//staging/src/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library", - "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/config:go_default_library", - "//staging/src/k8s.io/component-base/config/options:go_default_library", - "//staging/src/k8s.io/component-base/config/v1alpha1:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//staging/src/k8s.io/kube-scheduler/config/v1beta1:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = [ - "deprecated_test.go", - "insecure_serving_test.go", - "options_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kube-scheduler/app/config:go_default_library", - "//pkg/scheduler/apis/config:go_default_library", - "//pkg/scheduler/framework/plugins/interpodaffinity:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/component-base/config:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", - "//vendor/github.com/stretchr/testify/assert:go_default_library", - ], -) diff --git a/cmd/kube-scheduler/app/options/configfile.go b/cmd/kube-scheduler/app/options/configfile.go deleted file mode 100644 index 5cd6bf724127a..0000000000000 --- a/cmd/kube-scheduler/app/options/configfile.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "io/ioutil" - "k8s.io/klog/v2" - "os" - - "k8s.io/apimachinery/pkg/runtime" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" - kubeschedulerconfigv1beta1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta1" -) - -func loadConfigFromFile(file string) (*kubeschedulerconfig.KubeSchedulerConfiguration, error) { - data, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - return loadConfig(data) -} - -func loadConfig(data []byte) (*kubeschedulerconfig.KubeSchedulerConfiguration, error) { - // The UniversalDecoder runs defaulting and returns the internal type by default. - obj, gvk, err := kubeschedulerscheme.Codecs.UniversalDecoder().Decode(data, nil, nil) - if err != nil { - return nil, err - } - if cfgObj, ok := obj.(*kubeschedulerconfig.KubeSchedulerConfiguration); ok { - return cfgObj, nil - } - return nil, fmt.Errorf("couldn't decode as KubeSchedulerConfiguration, got %s: ", gvk) -} - -// LogOrWriteConfig logs the completed component config and writes it into the given file name as YAML, if either is enabled -func LogOrWriteConfig(fileName string, cfg *kubeschedulerconfig.KubeSchedulerConfiguration, completedProfiles []kubeschedulerconfig.KubeSchedulerProfile) error { - if !(klog.V(2).Enabled() || len(fileName) > 0) { - return nil - } - cfg.Profiles = completedProfiles - - const mediaType = runtime.ContentTypeYAML - info, ok := runtime.SerializerInfoForMediaType(kubeschedulerscheme.Codecs.SupportedMediaTypes(), mediaType) - if !ok { - return fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) - } - - encoder := kubeschedulerscheme.Codecs.EncoderForVersion(info.Serializer, kubeschedulerconfigv1beta1.SchemeGroupVersion) - if klog.V(2).Enabled() { - bytes, err := runtime.Encode(encoder, cfg) - if err != nil { - return err - } - configString := string(bytes) - klog.Infof("Using component config:\n%+v\n", configString) - } - - if len(fileName) > 0 { - configFile, err := os.Create(fileName) - if err != nil { - return err - } - defer configFile.Close() - if err := encoder.Encode(cfg, configFile); err != nil { - return err - } - klog.Infof("Wrote configuration to: %s\n", fileName) - os.Exit(0) - } - return nil -} diff --git a/cmd/kube-scheduler/app/options/deprecated.go b/cmd/kube-scheduler/app/options/deprecated.go deleted file mode 100644 index 089c510f825e6..0000000000000 --- a/cmd/kube-scheduler/app/options/deprecated.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/kubernetes/pkg/scheduler/algorithmprovider" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" -) - -// DeprecatedOptions contains deprecated options and their flags. -// TODO remove these fields once the deprecated flags are removed. -type DeprecatedOptions struct { - // The fields below here are placeholders for flags that can't be directly - // mapped into componentconfig.KubeSchedulerConfiguration. - PolicyConfigFile string - PolicyConfigMapName string - PolicyConfigMapNamespace string - UseLegacyPolicyConfig bool - AlgorithmProvider string - HardPodAffinitySymmetricWeight int32 - SchedulerName string -} - -// AddFlags adds flags for the deprecated options. -func (o *DeprecatedOptions) AddFlags(fs *pflag.FlagSet, cfg *kubeschedulerconfig.KubeSchedulerConfiguration) { - if o == nil { - return - } - - fs.StringVar(&o.AlgorithmProvider, "algorithm-provider", o.AlgorithmProvider, "DEPRECATED: the scheduling algorithm provider to use, this sets the default plugins for component config profiles. Choose one of: "+algorithmprovider.ListAlgorithmProviders()) - fs.StringVar(&o.PolicyConfigFile, "policy-config-file", o.PolicyConfigFile, "DEPRECATED: file with scheduler policy configuration. This file is used if policy ConfigMap is not provided or --use-legacy-policy-config=true. Note: The scheduler will fail if this is combined with Plugin configs") - usage := fmt.Sprintf("DEPRECATED: name of the ConfigMap object that contains scheduler's policy configuration. It must exist in the system namespace before scheduler initialization if --use-legacy-policy-config=false. The config must be provided as the value of an element in 'Data' map with the key='%v'. Note: The scheduler will fail if this is combined with Plugin configs", kubeschedulerconfig.SchedulerPolicyConfigMapKey) - fs.StringVar(&o.PolicyConfigMapName, "policy-configmap", o.PolicyConfigMapName, usage) - fs.StringVar(&o.PolicyConfigMapNamespace, "policy-configmap-namespace", o.PolicyConfigMapNamespace, "DEPRECATED: the namespace where policy ConfigMap is located. The kube-system namespace will be used if this is not provided or is empty. Note: The scheduler will fail if this is combined with Plugin configs") - fs.BoolVar(&o.UseLegacyPolicyConfig, "use-legacy-policy-config", o.UseLegacyPolicyConfig, "DEPRECATED: when set to true, scheduler will ignore policy ConfigMap and uses policy config file. Note: The scheduler will fail if this is combined with Plugin configs") - - fs.BoolVar(&cfg.EnableProfiling, "profiling", cfg.EnableProfiling, "DEPRECATED: enable profiling via web interface host:port/debug/pprof/") - fs.BoolVar(&cfg.EnableContentionProfiling, "contention-profiling", cfg.EnableContentionProfiling, "DEPRECATED: enable lock contention profiling, if profiling is enabled") - fs.StringVar(&cfg.ClientConnection.Kubeconfig, "kubeconfig", cfg.ClientConnection.Kubeconfig, "DEPRECATED: path to kubeconfig file with authorization and master location information.") - fs.StringVar(&cfg.ClientConnection.ContentType, "kube-api-content-type", cfg.ClientConnection.ContentType, "DEPRECATED: content type of requests sent to apiserver.") - fs.Float32Var(&cfg.ClientConnection.QPS, "kube-api-qps", cfg.ClientConnection.QPS, "DEPRECATED: QPS to use while talking with kubernetes apiserver") - fs.Int32Var(&cfg.ClientConnection.Burst, "kube-api-burst", cfg.ClientConnection.Burst, "DEPRECATED: burst to use while talking with kubernetes apiserver") - fs.StringVar(&cfg.LeaderElection.ResourceNamespace, "lock-object-namespace", cfg.LeaderElection.ResourceNamespace, "DEPRECATED: define the namespace of the lock object. Will be removed in favor of leader-elect-resource-namespace.") - fs.StringVar(&cfg.LeaderElection.ResourceName, "lock-object-name", cfg.LeaderElection.ResourceName, "DEPRECATED: define the name of the lock object. Will be removed in favor of leader-elect-resource-name") - - fs.Int32Var(&o.HardPodAffinitySymmetricWeight, "hard-pod-affinity-symmetric-weight", o.HardPodAffinitySymmetricWeight, - "DEPRECATED: RequiredDuringScheduling affinity is not symmetric, but there is an implicit PreferredDuringScheduling affinity rule corresponding "+ - "to every RequiredDuringScheduling affinity rule. --hard-pod-affinity-symmetric-weight represents the weight of implicit PreferredDuringScheduling affinity rule. Must be in the range 0-100."+ - "This option was moved to the policy configuration file") - fs.StringVar(&o.SchedulerName, "scheduler-name", o.SchedulerName, "DEPRECATED: name of the scheduler, used to select which pods will be processed by this scheduler, based on pod's \"spec.schedulerName\".") - // MarkDeprecated hides the flag from the help. We don't want that: - // fs.MarkDeprecated("hard-pod-affinity-symmetric-weight", "This option was moved to the policy configuration file") -} - -// Validate validates the deprecated scheduler options. -func (o *DeprecatedOptions) Validate() []error { - var errs []error - - if o.UseLegacyPolicyConfig && len(o.PolicyConfigFile) == 0 { - errs = append(errs, field.Required(field.NewPath("policyConfigFile"), "required when --use-legacy-policy-config is true")) - } - - if err := validation.ValidateHardPodAffinityWeight(field.NewPath("hardPodAffinitySymmetricWeight"), o.HardPodAffinitySymmetricWeight); err != nil { - errs = append(errs, err) - } - - return errs -} - -// ApplyAlgorithmSourceTo sets cfg.AlgorithmSource from flags passed on the command line in the following precedence order: -// -// 1. --use-legacy-policy-config to use a policy file. -// 2. --policy-configmap to use a policy config map value. -// 3. --algorithm-provider to use a named algorithm provider. -func (o *DeprecatedOptions) ApplyAlgorithmSourceTo(cfg *kubeschedulerconfig.KubeSchedulerConfiguration) { - if o == nil { - return - } - - switch { - case o.UseLegacyPolicyConfig || (len(o.PolicyConfigFile) > 0 && o.PolicyConfigMapName == ""): - cfg.AlgorithmSource = kubeschedulerconfig.SchedulerAlgorithmSource{ - Policy: &kubeschedulerconfig.SchedulerPolicySource{ - File: &kubeschedulerconfig.SchedulerPolicyFileSource{ - Path: o.PolicyConfigFile, - }, - }, - } - case len(o.PolicyConfigMapName) > 0: - cfg.AlgorithmSource = kubeschedulerconfig.SchedulerAlgorithmSource{ - Policy: &kubeschedulerconfig.SchedulerPolicySource{ - ConfigMap: &kubeschedulerconfig.SchedulerPolicyConfigMapSource{ - Name: o.PolicyConfigMapName, - Namespace: o.PolicyConfigMapNamespace, - }, - }, - } - case len(o.AlgorithmProvider) > 0: - cfg.AlgorithmSource = kubeschedulerconfig.SchedulerAlgorithmSource{ - Provider: &o.AlgorithmProvider, - } - } -} - -// ApplyTo sets a default profile plugin config if no config file is specified -// It also calls ApplyAlgorithmSourceTo to set Policy settings in AlgorithmSource, if applicable. -// Deprecated flags have an effect iff no config file was provided, in which -// case this function expects a default KubeSchedulerConfiguration instance, -// which has a single profile. -func (o *DeprecatedOptions) ApplyTo(cfg *kubeschedulerconfig.KubeSchedulerConfiguration) { - if o == nil { - return - } - // The following deprecated options affect the only existing profile that is - // added by default. - profile := &cfg.Profiles[0] - if len(o.SchedulerName) > 0 { - profile.SchedulerName = o.SchedulerName - } - plCfg := kubeschedulerconfig.PluginConfig{ - Name: interpodaffinity.Name, - Args: &kubeschedulerconfig.InterPodAffinityArgs{ - HardPodAffinityWeight: o.HardPodAffinitySymmetricWeight, - }, - } - - profile.PluginConfig = append(profile.PluginConfig, plCfg) - o.ApplyAlgorithmSourceTo(cfg) -} diff --git a/cmd/kube-scheduler/app/options/deprecated_test.go b/cmd/kube-scheduler/app/options/deprecated_test.go deleted file mode 100644 index 17830535a5ca1..0000000000000 --- a/cmd/kube-scheduler/app/options/deprecated_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "testing" -) - -func TestValidateDeprecatedKubeSchedulerConfiguration(t *testing.T) { - scenarios := map[string]struct { - expectedToFail bool - config *DeprecatedOptions - }{ - "good": { - config: &DeprecatedOptions{ - PolicyConfigFile: "/some/file", - UseLegacyPolicyConfig: true, - AlgorithmProvider: "", - }, - }, - "bad-policy-config-file-null": { - expectedToFail: true, - config: &DeprecatedOptions{ - PolicyConfigFile: "", - UseLegacyPolicyConfig: true, - AlgorithmProvider: "", - }, - }, - "good affinity weight": { - config: &DeprecatedOptions{ - HardPodAffinitySymmetricWeight: 50, - }, - }, - "bad affinity weight": { - expectedToFail: true, - config: &DeprecatedOptions{ - HardPodAffinitySymmetricWeight: -1, - }, - }, - } - - for name, scenario := range scenarios { - errs := scenario.config.Validate() - if len(errs) == 0 && scenario.expectedToFail { - t.Errorf("Unexpected success for scenario: %s", name) - } - if len(errs) > 0 && !scenario.expectedToFail { - t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) - } - } -} diff --git a/cmd/kube-scheduler/app/options/insecure_serving.go b/cmd/kube-scheduler/app/options/insecure_serving.go deleted file mode 100644 index 93535b0ce52d1..0000000000000 --- a/cmd/kube-scheduler/app/options/insecure_serving.go +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "net" - "strconv" - - "github.com/spf13/pflag" - - apiserveroptions "k8s.io/apiserver/pkg/server/options" - schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" -) - -// CombinedInsecureServingOptions sets up to two insecure listeners for healthz and metrics. The flags -// override the ComponentConfig and DeprecatedInsecureServingOptions values for both. -type CombinedInsecureServingOptions struct { - Healthz *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback - Metrics *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback - - BindPort int // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig - BindAddress string // overrides the structs above on ApplyTo, ignored on ApplyToFromLoadedConfig -} - -// AddFlags adds flags for the insecure serving options. -func (o *CombinedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) { - if o == nil { - return - } - - fs.StringVar(&o.BindAddress, "address", o.BindAddress, "DEPRECATED: the IP address on which to listen for the --port port (set to 0.0.0.0 or :: for listening in all interfaces and IP families). See --bind-address instead.") - // MarkDeprecated hides the flag from the help. We don't want that: - // fs.MarkDeprecated("address", "see --bind-address instead.") - fs.IntVar(&o.BindPort, "port", o.BindPort, "DEPRECATED: the port on which to serve HTTP insecurely without authentication and authorization. If 0, don't serve plain HTTP at all. See --secure-port instead.") - // MarkDeprecated hides the flag from the help. We don't want that: - // fs.MarkDeprecated("port", "see --secure-port instead.") -} - -func (o *CombinedInsecureServingOptions) applyTo(c *schedulerappconfig.Config, componentConfig *kubeschedulerconfig.KubeSchedulerConfiguration) error { - if err := updateAddressFromDeprecatedInsecureServingOptions(&componentConfig.HealthzBindAddress, o.Healthz); err != nil { - return err - } - if err := updateAddressFromDeprecatedInsecureServingOptions(&componentConfig.MetricsBindAddress, o.Metrics); err != nil { - return err - } - - if err := o.Healthz.ApplyTo(&c.InsecureServing, &c.LoopbackClientConfig); err != nil { - return err - } - if o.Metrics != nil && (c.ComponentConfig.MetricsBindAddress != c.ComponentConfig.HealthzBindAddress || o.Healthz == nil) { - if err := o.Metrics.ApplyTo(&c.InsecureMetricsServing, &c.LoopbackClientConfig); err != nil { - return err - } - } - - return nil -} - -// ApplyTo applies the insecure serving options to the given scheduler app configuration, and updates the componentConfig. -func (o *CombinedInsecureServingOptions) ApplyTo(c *schedulerappconfig.Config, componentConfig *kubeschedulerconfig.KubeSchedulerConfiguration) error { - if o == nil { - componentConfig.HealthzBindAddress = "" - componentConfig.MetricsBindAddress = "" - return nil - } - - if o.Healthz != nil { - o.Healthz.BindPort = o.BindPort - o.Healthz.BindAddress = net.ParseIP(o.BindAddress) - } - if o.Metrics != nil { - o.Metrics.BindPort = o.BindPort - o.Metrics.BindAddress = net.ParseIP(o.BindAddress) - } - - return o.applyTo(c, componentConfig) -} - -// ApplyToFromLoadedConfig updates the insecure serving options from the component config and then applies it to the given scheduler app configuration. -func (o *CombinedInsecureServingOptions) ApplyToFromLoadedConfig(c *schedulerappconfig.Config, componentConfig *kubeschedulerconfig.KubeSchedulerConfiguration) error { - if o == nil { - return nil - } - - if err := updateDeprecatedInsecureServingOptionsFromAddress(o.Healthz, componentConfig.HealthzBindAddress); err != nil { - return fmt.Errorf("invalid healthz address: %v", err) - } - if err := updateDeprecatedInsecureServingOptionsFromAddress(o.Metrics, componentConfig.MetricsBindAddress); err != nil { - return fmt.Errorf("invalid metrics address: %v", err) - } - - return o.applyTo(c, componentConfig) -} - -func updateAddressFromDeprecatedInsecureServingOptions(addr *string, is *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback) error { - if is == nil { - *addr = "" - } else { - if is.Listener != nil { - *addr = is.Listener.Addr().String() - } else if is.BindPort == 0 { - *addr = "" - } else { - *addr = net.JoinHostPort(is.BindAddress.String(), strconv.Itoa(is.BindPort)) - } - } - - return nil -} - -func updateDeprecatedInsecureServingOptionsFromAddress(is *apiserveroptions.DeprecatedInsecureServingOptionsWithLoopback, addr string) error { - if is == nil { - return nil - } - if len(addr) == 0 { - is.BindPort = 0 - return nil - } - - host, portInt, err := splitHostIntPort(addr) - if err != nil { - return fmt.Errorf("invalid address %q", addr) - } - - is.BindAddress = net.ParseIP(host) - is.BindPort = portInt - - return nil -} - -// Validate validates the insecure serving options. -func (o *CombinedInsecureServingOptions) Validate() []error { - if o == nil { - return nil - } - - errors := []error{} - - if o.BindPort < 0 || o.BindPort > 65535 { - errors = append(errors, fmt.Errorf("--port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", o.BindPort)) - } - - if len(o.BindAddress) > 0 && net.ParseIP(o.BindAddress) == nil { - errors = append(errors, fmt.Errorf("--address %v is an invalid IP address", o.BindAddress)) - } - - return errors -} diff --git a/cmd/kube-scheduler/app/options/insecure_serving_test.go b/cmd/kube-scheduler/app/options/insecure_serving_test.go deleted file mode 100644 index 199495fe6fe81..0000000000000 --- a/cmd/kube-scheduler/app/options/insecure_serving_test.go +++ /dev/null @@ -1,276 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "net" - "strconv" - "testing" - - "k8s.io/apimachinery/pkg/util/rand" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" -) - -func TestOptions_ApplyTo(t *testing.T) { - tests := []struct { - name string - options Options - configLoaded bool - expectHealthzBindAddress, expectMetricsBindAddress string - expectInsecureServingAddress, expectInsecureMetricsServingAddress string - expectInsecureServingPort, expectInsecureMetricsServingPort int - wantErr bool - }{ - { - name: "no config, zero port", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindPort: 0, - }, - }, - configLoaded: false, - }, - { - name: "config loaded, non-nil healthz", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindPort: 0, - }, - }, - configLoaded: true, - - expectHealthzBindAddress: "1.2.3.4:1234", - expectInsecureServingPort: 1234, - expectInsecureServingAddress: "1.2.3.4", - }, - { - name: "config loaded, non-nil metrics", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindPort: 0, - }, - }, - configLoaded: true, - - expectMetricsBindAddress: "1.2.3.4:1234", - expectInsecureMetricsServingPort: 1234, - expectInsecureMetricsServingAddress: "1.2.3.4", - }, - { - name: "config loaded, all set, zero BindPort", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindPort: 0, - }, - }, - configLoaded: true, - - expectHealthzBindAddress: "1.2.3.4:1234", - expectInsecureServingPort: 1234, - expectInsecureServingAddress: "1.2.3.4", - - expectMetricsBindAddress: "1.2.3.4:1234", - }, - { - name: "config loaded, all set, different addresses", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1235", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindPort: 0, - }, - }, - configLoaded: true, - - expectHealthzBindAddress: "1.2.3.4:1234", - expectInsecureServingPort: 1234, - expectInsecureServingAddress: "1.2.3.4", - - expectMetricsBindAddress: "1.2.3.4:1235", - expectInsecureMetricsServingPort: 1235, - expectInsecureMetricsServingAddress: "1.2.3.4", - }, - { - name: "no config, all set, port passed", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindPort: 1236, - BindAddress: "1.2.3.4", - }, - }, - configLoaded: false, - - expectHealthzBindAddress: "1.2.3.4:1236", - expectInsecureServingPort: 1236, - expectInsecureServingAddress: "1.2.3.4", - - expectMetricsBindAddress: "1.2.3.4:1236", - }, - { - name: "no config, all set, address passed", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindAddress: "2.3.4.5", - BindPort: 1234, - }, - }, - configLoaded: false, - - expectHealthzBindAddress: "2.3.4.5:1234", - expectInsecureServingPort: 1234, - expectInsecureServingAddress: "2.3.4.5", - - expectMetricsBindAddress: "2.3.4.5:1234", - }, - { - name: "no config, all set, zero port passed", - options: Options{ - ComponentConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - HealthzBindAddress: "1.2.3.4:1234", - MetricsBindAddress: "1.2.3.4:1234", - }, - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{}).WithLoopback(), - BindAddress: "2.3.4.5", - BindPort: 0, - }, - }, - configLoaded: false, - }, - } - for i, tt := range tests { - t.Run(fmt.Sprintf("%d-%s", i, tt.name), func(t *testing.T) { - c := schedulerappconfig.Config{ - ComponentConfig: tt.options.ComponentConfig, - } - - if tt.options.CombinedInsecureServing != nil { - if tt.options.CombinedInsecureServing.Healthz != nil { - tt.options.CombinedInsecureServing.Healthz.ListenFunc = createMockListener - } - if tt.options.CombinedInsecureServing.Metrics != nil { - tt.options.CombinedInsecureServing.Metrics.ListenFunc = createMockListener - } - } - - if tt.configLoaded { - if err := tt.options.CombinedInsecureServing.ApplyToFromLoadedConfig(&c, &c.ComponentConfig); (err != nil) != tt.wantErr { - t.Fatalf("%d - Options.ApplyTo() error = %v, wantErr %v", i, err, tt.wantErr) - } - } else { - if err := tt.options.CombinedInsecureServing.ApplyTo(&c, &c.ComponentConfig); (err != nil) != tt.wantErr { - t.Fatalf("%d - Options.ApplyTo() error = %v, wantErr %v", i, err, tt.wantErr) - } - } - if got, expect := c.ComponentConfig.HealthzBindAddress, tt.expectHealthzBindAddress; got != expect { - t.Errorf("%d - expected HealthzBindAddress %q, got %q", i, expect, got) - } - if got, expect := c.ComponentConfig.MetricsBindAddress, tt.expectMetricsBindAddress; got != expect { - t.Errorf("%d - expected MetricsBindAddress %q, got %q", i, expect, got) - } - if got, expect := c.InsecureServing != nil, tt.expectInsecureServingPort != 0; got != expect { - t.Errorf("%d - expected InsecureServing != nil to be %v, got %v", i, expect, got) - } else if c.InsecureServing != nil { - if got, expect := c.InsecureServing.Listener.(*mockListener).address, tt.expectInsecureServingAddress; got != expect { - t.Errorf("%d - expected healthz address %q, got %q", i, expect, got) - } - if got, expect := c.InsecureServing.Listener.(*mockListener).port, tt.expectInsecureServingPort; got != expect { - t.Errorf("%d - expected healthz port %v, got %v", i, expect, got) - } - } - if got, expect := c.InsecureMetricsServing != nil, tt.expectInsecureMetricsServingPort != 0; got != expect { - t.Errorf("%d - expected Metrics != nil to be %v, got %v", i, expect, got) - } else if c.InsecureMetricsServing != nil { - if got, expect := c.InsecureMetricsServing.Listener.(*mockListener).address, tt.expectInsecureMetricsServingAddress; got != expect { - t.Errorf("%d - expected metrics address %q, got %q", i, expect, got) - } - if got, expect := c.InsecureMetricsServing.Listener.(*mockListener).port, tt.expectInsecureMetricsServingPort; got != expect { - t.Errorf("%d - expected metrics port %v, got %v", i, expect, got) - } - } - }) - } -} - -type mockListener struct { - address string - port int -} - -func createMockListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) { - host, portInt, err := splitHostIntPort(addr) - if err != nil { - return nil, 0, err - } - if portInt == 0 { - portInt = rand.IntnRange(1, 32767) - } - return &mockListener{host, portInt}, portInt, nil -} - -func (l *mockListener) Accept() (net.Conn, error) { return nil, nil } -func (l *mockListener) Close() error { return nil } -func (l *mockListener) Addr() net.Addr { - return mockAddr(net.JoinHostPort(l.address, strconv.Itoa(l.port))) -} - -type mockAddr string - -func (a mockAddr) Network() string { return "tcp" } -func (a mockAddr) String() string { return string(a) } diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go deleted file mode 100644 index ebed9e4c5212c..0000000000000 --- a/cmd/kube-scheduler/app/options/options.go +++ /dev/null @@ -1,369 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "net" - "os" - "strconv" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/uuid" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - utilfeature "k8s.io/apiserver/pkg/util/feature" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/client-go/tools/events" - "k8s.io/client-go/tools/leaderelection" - "k8s.io/client-go/tools/leaderelection/resourcelock" - "k8s.io/client-go/tools/record" - cliflag "k8s.io/component-base/cli/flag" - componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/config/options" - configv1alpha1 "k8s.io/component-base/config/v1alpha1" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - "k8s.io/klog/v2" - kubeschedulerconfigv1beta1 "k8s.io/kube-scheduler/config/v1beta1" - schedulerappconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" - "k8s.io/kubernetes/pkg/scheduler" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - kubeschedulerscheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" - "k8s.io/kubernetes/pkg/scheduler/apis/config/validation" -) - -// Options has all the params needed to run a Scheduler -type Options struct { - // The default values. These are overridden if ConfigFile is set or by values in InsecureServing. - ComponentConfig kubeschedulerconfig.KubeSchedulerConfiguration - - SecureServing *apiserveroptions.SecureServingOptionsWithLoopback - CombinedInsecureServing *CombinedInsecureServingOptions - Authentication *apiserveroptions.DelegatingAuthenticationOptions - Authorization *apiserveroptions.DelegatingAuthorizationOptions - Metrics *metrics.Options - Logs *logs.Options - Deprecated *DeprecatedOptions - - // ConfigFile is the location of the scheduler server's configuration file. - ConfigFile string - - // WriteConfigTo is the path where the default configuration will be written. - WriteConfigTo string - - Master string -} - -// NewOptions returns default scheduler app options. -func NewOptions() (*Options, error) { - cfg, err := newDefaultComponentConfig() - if err != nil { - return nil, err - } - - hhost, hport, err := splitHostIntPort(cfg.HealthzBindAddress) - if err != nil { - return nil, err - } - - o := &Options{ - ComponentConfig: *cfg, - SecureServing: apiserveroptions.NewSecureServingOptions().WithLoopback(), - CombinedInsecureServing: &CombinedInsecureServingOptions{ - Healthz: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindNetwork: "tcp", - }).WithLoopback(), - Metrics: (&apiserveroptions.DeprecatedInsecureServingOptions{ - BindNetwork: "tcp", - }).WithLoopback(), - BindPort: hport, - BindAddress: hhost, - }, - Authentication: apiserveroptions.NewDelegatingAuthenticationOptions(), - Authorization: apiserveroptions.NewDelegatingAuthorizationOptions(), - Deprecated: &DeprecatedOptions{ - UseLegacyPolicyConfig: false, - PolicyConfigMapNamespace: metav1.NamespaceSystem, - SchedulerName: corev1.DefaultSchedulerName, - HardPodAffinitySymmetricWeight: 1, - }, - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), - } - - o.Authentication.TolerateInClusterLookupFailure = true - o.Authentication.RemoteKubeConfigFileOptional = true - o.Authorization.RemoteKubeConfigFileOptional = true - o.Authorization.AlwaysAllowPaths = []string{"/healthz"} - - // Set the PairName but leave certificate directory blank to generate in-memory by default - o.SecureServing.ServerCert.CertDirectory = "" - o.SecureServing.ServerCert.PairName = "kube-scheduler" - o.SecureServing.BindPort = kubeschedulerconfig.DefaultKubeSchedulerPort - - return o, nil -} - -func splitHostIntPort(s string) (string, int, error) { - host, port, err := net.SplitHostPort(s) - if err != nil { - return "", 0, err - } - portInt, err := strconv.Atoi(port) - if err != nil { - return "", 0, err - } - return host, portInt, err -} - -func newDefaultComponentConfig() (*kubeschedulerconfig.KubeSchedulerConfiguration, error) { - versionedCfg := kubeschedulerconfigv1beta1.KubeSchedulerConfiguration{} - versionedCfg.DebuggingConfiguration = *configv1alpha1.NewRecommendedDebuggingConfiguration() - - kubeschedulerscheme.Scheme.Default(&versionedCfg) - cfg := kubeschedulerconfig.KubeSchedulerConfiguration{} - if err := kubeschedulerscheme.Scheme.Convert(&versionedCfg, &cfg, nil); err != nil { - return nil, err - } - return &cfg, nil -} - -// Flags returns flags for a specific scheduler by section name -func (o *Options) Flags() (nfs cliflag.NamedFlagSets) { - fs := nfs.FlagSet("misc") - fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, `The path to the configuration file. The following flags can overwrite fields in this file: - --address - --port - --use-legacy-policy-config - --policy-configmap - --policy-config-file - --algorithm-provider`) - fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.") - fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - - o.SecureServing.AddFlags(nfs.FlagSet("secure serving")) - o.CombinedInsecureServing.AddFlags(nfs.FlagSet("insecure serving")) - o.Authentication.AddFlags(nfs.FlagSet("authentication")) - o.Authorization.AddFlags(nfs.FlagSet("authorization")) - o.Deprecated.AddFlags(nfs.FlagSet("deprecated"), &o.ComponentConfig) - - options.BindLeaderElectionFlags(&o.ComponentConfig.LeaderElection, nfs.FlagSet("leader election")) - utilfeature.DefaultMutableFeatureGate.AddFlag(nfs.FlagSet("feature gate")) - o.Metrics.AddFlags(nfs.FlagSet("metrics")) - o.Logs.AddFlags(nfs.FlagSet("logs")) - - return nfs -} - -// ApplyTo applies the scheduler options to the given scheduler app configuration. -func (o *Options) ApplyTo(c *schedulerappconfig.Config) error { - if len(o.ConfigFile) == 0 { - c.ComponentConfig = o.ComponentConfig - - // apply deprecated flags if no config file is loaded (this is the old behaviour). - o.Deprecated.ApplyTo(&c.ComponentConfig) - if err := o.CombinedInsecureServing.ApplyTo(c, &c.ComponentConfig); err != nil { - return err - } - } else { - cfg, err := loadConfigFromFile(o.ConfigFile) - if err != nil { - return err - } - if err := validation.ValidateKubeSchedulerConfiguration(cfg).ToAggregate(); err != nil { - return err - } - - c.ComponentConfig = *cfg - - // apply any deprecated Policy flags, if applicable - o.Deprecated.ApplyAlgorithmSourceTo(&c.ComponentConfig) - - // if the user has set CC profiles and is trying to use a Policy config, error out - // these configs are no longer merged and they should not be used simultaneously - if !emptySchedulerProfileConfig(c.ComponentConfig.Profiles) && c.ComponentConfig.AlgorithmSource.Policy != nil { - return fmt.Errorf("cannot set a Plugin config and Policy config") - } - - // use the loaded config file only, with the exception of --address and --port. - if err := o.CombinedInsecureServing.ApplyToFromLoadedConfig(c, &c.ComponentConfig); err != nil { - return err - } - } - - if err := o.SecureServing.ApplyTo(&c.SecureServing, &c.LoopbackClientConfig); err != nil { - return err - } - if o.SecureServing != nil && (o.SecureServing.BindPort != 0 || o.SecureServing.Listener != nil) { - if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil { - return err - } - if err := o.Authorization.ApplyTo(&c.Authorization); err != nil { - return err - } - } - o.Metrics.Apply() - o.Logs.Apply() - return nil -} - -// emptySchedulerProfileConfig returns true if the list of profiles passed to it contains only -// the "default-scheduler" profile with no plugins or pluginconfigs registered -// (this is the default empty profile initialized by defaults.go) -func emptySchedulerProfileConfig(profiles []kubeschedulerconfig.KubeSchedulerProfile) bool { - return len(profiles) == 1 && - len(profiles[0].PluginConfig) == 0 && - profiles[0].Plugins == nil -} - -// Validate validates all the required options. -func (o *Options) Validate() []error { - var errs []error - - if err := validation.ValidateKubeSchedulerConfiguration(&o.ComponentConfig).ToAggregate(); err != nil { - errs = append(errs, err.Errors()...) - } - errs = append(errs, o.SecureServing.Validate()...) - errs = append(errs, o.CombinedInsecureServing.Validate()...) - errs = append(errs, o.Authentication.Validate()...) - errs = append(errs, o.Authorization.Validate()...) - errs = append(errs, o.Deprecated.Validate()...) - errs = append(errs, o.Metrics.Validate()...) - errs = append(errs, o.Logs.Validate()...) - - return errs -} - -// Config return a scheduler config object -func (o *Options) Config() (*schedulerappconfig.Config, error) { - if o.SecureServing != nil { - if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { - return nil, fmt.Errorf("error creating self-signed certificates: %v", err) - } - } - - c := &schedulerappconfig.Config{} - if err := o.ApplyTo(c); err != nil { - return nil, err - } - - // Prepare kube clients. - client, leaderElectionClient, eventClient, err := createClients(c.ComponentConfig.ClientConnection, o.Master, c.ComponentConfig.LeaderElection.RenewDeadline.Duration) - if err != nil { - return nil, err - } - - c.EventBroadcaster = events.NewEventBroadcasterAdapter(eventClient) - - // Set up leader election if enabled. - var leaderElectionConfig *leaderelection.LeaderElectionConfig - if c.ComponentConfig.LeaderElection.LeaderElect { - // Use the scheduler name in the first profile to record leader election. - coreRecorder := c.EventBroadcaster.DeprecatedNewLegacyRecorder(c.ComponentConfig.Profiles[0].SchedulerName) - leaderElectionConfig, err = makeLeaderElectionConfig(c.ComponentConfig.LeaderElection, leaderElectionClient, coreRecorder) - if err != nil { - return nil, err - } - } - - c.Client = client - c.InformerFactory = scheduler.NewInformerFactory(client, 0) - c.LeaderElection = leaderElectionConfig - - return c, nil -} - -// makeLeaderElectionConfig builds a leader election configuration. It will -// create a new resource lock associated with the configuration. -func makeLeaderElectionConfig(config componentbaseconfig.LeaderElectionConfiguration, client clientset.Interface, recorder record.EventRecorder) (*leaderelection.LeaderElectionConfig, error) { - hostname, err := os.Hostname() - if err != nil { - return nil, fmt.Errorf("unable to get hostname: %v", err) - } - // add a uniquifier so that two processes on the same host don't accidentally both become active - id := hostname + "_" + string(uuid.NewUUID()) - - rl, err := resourcelock.New(config.ResourceLock, - config.ResourceNamespace, - config.ResourceName, - client.CoreV1(), - client.CoordinationV1(), - resourcelock.ResourceLockConfig{ - Identity: id, - EventRecorder: recorder, - }) - if err != nil { - return nil, fmt.Errorf("couldn't create resource lock: %v", err) - } - - return &leaderelection.LeaderElectionConfig{ - Lock: rl, - LeaseDuration: config.LeaseDuration.Duration, - RenewDeadline: config.RenewDeadline.Duration, - RetryPeriod: config.RetryPeriod.Duration, - WatchDog: leaderelection.NewLeaderHealthzAdaptor(time.Second * 20), - Name: "kube-scheduler", - }, nil -} - -// createClients creates a kube client and an event client from the given config and masterOverride. -// TODO remove masterOverride when CLI flags are removed. -func createClients(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string, timeout time.Duration) (clientset.Interface, clientset.Interface, clientset.Interface, error) { - if len(config.Kubeconfig) == 0 && len(masterOverride) == 0 { - klog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") - } - - // This creates a client, first loading any specified kubeconfig - // file, and then overriding the Master flag, if non-empty. - kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: config.Kubeconfig}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig() - if err != nil { - return nil, nil, nil, err - } - - kubeConfig.DisableCompression = true - kubeConfig.AcceptContentTypes = config.AcceptContentTypes - kubeConfig.ContentType = config.ContentType - kubeConfig.QPS = config.QPS - kubeConfig.Burst = int(config.Burst) - - client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "scheduler")) - if err != nil { - return nil, nil, nil, err - } - - // shallow copy, do not modify the kubeConfig.Timeout. - restConfig := *kubeConfig - restConfig.Timeout = timeout - leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(&restConfig, "leader-election")) - if err != nil { - return nil, nil, nil, err - } - - eventClient, err := clientset.NewForConfig(kubeConfig) - if err != nil { - return nil, nil, nil, err - } - - return client, leaderElectionClient, eventClient, nil -} diff --git a/cmd/kube-scheduler/app/options/options_test.go b/cmd/kube-scheduler/app/options/options_test.go deleted file mode 100644 index abe32ff848317..0000000000000 --- a/cmd/kube-scheduler/app/options/options_test.go +++ /dev/null @@ -1,820 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - apiserveroptions "k8s.io/apiserver/pkg/server/options" - componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/logs" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity" -) - -func TestSchedulerOptions(t *testing.T) { - // temp dir - tmpDir, err := ioutil.TempDir("", "scheduler-options") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - // record the username requests were made with - username := "" - // https server - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - username, _, _ = req.BasicAuth() - if username == "" { - username = "none, tls" - } - w.WriteHeader(200) - w.Write([]byte(`ok`)) - })) - defer server.Close() - // http server - insecureserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - username, _, _ = req.BasicAuth() - if username == "" { - username = "none, http" - } - w.WriteHeader(200) - w.Write([]byte(`ok`)) - })) - defer insecureserver.Close() - - // config file and kubeconfig - configFile := filepath.Join(tmpDir, "scheduler.yaml") - configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig") - if err := ioutil.WriteFile(configFile, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -leaderElection: - leaderElect: true`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(configKubeconfig, []byte(fmt.Sprintf(` -apiVersion: v1 -kind: Config -clusters: -- cluster: - insecure-skip-tls-verify: true - server: %s - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -users: -- name: default - user: - username: config -`, server.URL)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - oldConfigFile := filepath.Join(tmpDir, "scheduler_old.yaml") - if err := ioutil.WriteFile(oldConfigFile, []byte(fmt.Sprintf(` -apiVersion: componentconfig/v1alpha1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -leaderElection: - leaderElect: true`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - unknownVersionConfig := filepath.Join(tmpDir, "scheduler_invalid_wrong_api_version.yaml") - if err := ioutil.WriteFile(unknownVersionConfig, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/unknown -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -leaderElection: - leaderElect: true`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - noVersionConfig := filepath.Join(tmpDir, "scheduler_invalid_no_version.yaml") - if err := ioutil.WriteFile(noVersionConfig, []byte(fmt.Sprintf(` -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -leaderElection: - leaderElect: true`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - unknownFieldConfig := filepath.Join(tmpDir, "scheduler_invalid_unknown_field.yaml") - if err := ioutil.WriteFile(unknownFieldConfig, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -leaderElection: - leaderElect: true -foo: bar`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - duplicateFieldConfig := filepath.Join(tmpDir, "scheduler_invalid_duplicate_fields.yaml") - if err := ioutil.WriteFile(duplicateFieldConfig, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -leaderElection: - leaderElect: true - leaderElect: false`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // flag-specified kubeconfig - flagKubeconfig := filepath.Join(tmpDir, "flag.kubeconfig") - if err := ioutil.WriteFile(flagKubeconfig, []byte(fmt.Sprintf(` -apiVersion: v1 -kind: Config -clusters: -- cluster: - insecure-skip-tls-verify: true - server: %s - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -users: -- name: default - user: - username: flag -`, server.URL)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // plugin config - pluginConfigFile := filepath.Join(tmpDir, "plugin.yaml") - if err := ioutil.WriteFile(pluginConfigFile, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -profiles: -- plugins: - reserve: - enabled: - - name: foo - - name: bar - disabled: - - name: baz - preBind: - enabled: - - name: foo - disabled: - - name: baz - pluginConfig: - - name: InterPodAffinity - args: - hardPodAffinityWeight: 2 - - name: foo - args: - bar: baz -`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // multiple profiles config - multiProfilesConfig := filepath.Join(tmpDir, "multi-profiles.yaml") - if err := ioutil.WriteFile(multiProfilesConfig, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -profiles: -- schedulerName: "foo-profile" - plugins: - reserve: - enabled: - - name: foo -- schedulerName: "bar-profile" - plugins: - preBind: - disabled: - - name: baz - pluginConfig: - - name: foo -`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // Insulate this test from picking up in-cluster config when run inside a pod - // We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing - originalHost := os.Getenv("KUBERNETES_SERVICE_HOST") - if len(originalHost) > 0 { - os.Setenv("KUBERNETES_SERVICE_HOST", "") - defer os.Setenv("KUBERNETES_SERVICE_HOST", originalHost) - } - - defaultSource := "DefaultProvider" - defaultPodInitialBackoffSeconds := int64(1) - defaultPodMaxBackoffSeconds := int64(10) - defaultPercentageOfNodesToScore := int32(0) - - testcases := []struct { - name string - options *Options - expectedUsername string - expectedError string - expectedConfig kubeschedulerconfig.KubeSchedulerConfiguration - checkErrFn func(err error) bool - }{ - { - name: "config file", - options: &Options{ - ConfigFile: configFile, - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, err := newDefaultComponentConfig() - if err != nil { - t.Fatal(err) - } - return *cfg - }(), - SecureServing: (&apiserveroptions.SecureServingOptions{ - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/a/b/c", - PairName: "kube-scheduler", - }, - HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), - Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, - RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ - UsernameHeaders: []string{"x-remote-user"}, - GroupHeaders: []string{"x-remote-group"}, - ExtraHeaderPrefixes: []string{"x-remote-extra-"}, - }, - RemoteKubeConfigFileOptional: true, - }, - Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ - AllowCacheTTL: 10 * time.Second, - DenyCacheTTL: 10 * time.Second, - RemoteKubeConfigFileOptional: true, - AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* - }, - Logs: logs.NewOptions(), - }, - expectedUsername: "config", - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - HealthzBindAddress: "0.0.0.0:10251", - MetricsBindAddress: "0.0.0.0:10251", - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: configKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - {SchedulerName: "default-scheduler"}, - }, - }, - }, - { - name: "config file in componentconfig/v1alpha1", - options: &Options{ - ConfigFile: oldConfigFile, - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, err := newDefaultComponentConfig() - if err != nil { - t.Fatal(err) - } - return *cfg - }(), - Logs: logs.NewOptions(), - }, - expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"componentconfig/v1alpha1\"", - }, - - { - name: "unknown version kubescheduler.config.k8s.io/unknown", - options: &Options{ - ConfigFile: unknownVersionConfig, - Logs: logs.NewOptions(), - }, - expectedError: "no kind \"KubeSchedulerConfiguration\" is registered for version \"kubescheduler.config.k8s.io/unknown\"", - }, - { - name: "config file with no version", - options: &Options{ - ConfigFile: noVersionConfig, - Logs: logs.NewOptions(), - }, - expectedError: "Object 'apiVersion' is missing", - }, - { - name: "kubeconfig flag", - options: &Options{ - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, _ := newDefaultComponentConfig() - cfg.ClientConnection.Kubeconfig = flagKubeconfig - return *cfg - }(), - SecureServing: (&apiserveroptions.SecureServingOptions{ - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/a/b/c", - PairName: "kube-scheduler", - }, - HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), - Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - ClientCert: apiserveroptions.ClientCertAuthenticationOptions{}, - RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ - UsernameHeaders: []string{"x-remote-user"}, - GroupHeaders: []string{"x-remote-group"}, - ExtraHeaderPrefixes: []string{"x-remote-extra-"}, - }, - RemoteKubeConfigFileOptional: true, - }, - Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ - AllowCacheTTL: 10 * time.Second, - DenyCacheTTL: 10 * time.Second, - RemoteKubeConfigFileOptional: true, - AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* - }, - Logs: logs.NewOptions(), - }, - expectedUsername: "flag", - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - HealthzBindAddress: "", // defaults empty when not running from config file - MetricsBindAddress: "", // defaults empty when not running from config file - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: flagKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - {SchedulerName: "default-scheduler"}, - }, - }, - }, - { - name: "overridden master", - options: &Options{ - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, _ := newDefaultComponentConfig() - cfg.ClientConnection.Kubeconfig = flagKubeconfig - return *cfg - }(), - Master: insecureserver.URL, - SecureServing: (&apiserveroptions.SecureServingOptions{ - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/a/b/c", - PairName: "kube-scheduler", - }, - HTTP2MaxStreamsPerConnection: 47, - }).WithLoopback(), - Authentication: &apiserveroptions.DelegatingAuthenticationOptions{ - CacheTTL: 10 * time.Second, - RequestHeader: apiserveroptions.RequestHeaderAuthenticationOptions{ - UsernameHeaders: []string{"x-remote-user"}, - GroupHeaders: []string{"x-remote-group"}, - ExtraHeaderPrefixes: []string{"x-remote-extra-"}, - }, - RemoteKubeConfigFileOptional: true, - }, - Authorization: &apiserveroptions.DelegatingAuthorizationOptions{ - AllowCacheTTL: 10 * time.Second, - DenyCacheTTL: 10 * time.Second, - RemoteKubeConfigFileOptional: true, - AlwaysAllowPaths: []string{"/healthz"}, // note: this does not match /healthz/ or /healthz/* - }, - Logs: logs.NewOptions(), - }, - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - HealthzBindAddress: "", // defaults empty when not running from config file - MetricsBindAddress: "", // defaults empty when not running from config file - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: flagKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - {SchedulerName: "default-scheduler"}, - }, - }, - expectedUsername: "none, http", - }, - { - name: "plugin config", - options: &Options{ - ConfigFile: pluginConfigFile, - Logs: logs.NewOptions(), - }, - expectedUsername: "config", - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - HealthzBindAddress: "0.0.0.0:10251", - MetricsBindAddress: "0.0.0.0:10251", - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: configKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - { - SchedulerName: "default-scheduler", - Plugins: &kubeschedulerconfig.Plugins{ - Reserve: &kubeschedulerconfig.PluginSet{ - Enabled: []kubeschedulerconfig.Plugin{ - {Name: "foo"}, - {Name: "bar"}, - }, - Disabled: []kubeschedulerconfig.Plugin{ - {Name: "baz"}, - }, - }, - PreBind: &kubeschedulerconfig.PluginSet{ - Enabled: []kubeschedulerconfig.Plugin{ - {Name: "foo"}, - }, - Disabled: []kubeschedulerconfig.Plugin{ - {Name: "baz"}, - }, - }, - }, - PluginConfig: []kubeschedulerconfig.PluginConfig{ - { - Name: "InterPodAffinity", - Args: &kubeschedulerconfig.InterPodAffinityArgs{ - HardPodAffinityWeight: 2, - }, - }, - { - Name: "foo", - Args: &runtime.Unknown{ - Raw: []byte(`{"bar":"baz"}`), - ContentType: "application/json", - }, - }, - }, - }, - }, - }, - }, - { - name: "multiple profiles", - options: &Options{ - ConfigFile: multiProfilesConfig, - Logs: logs.NewOptions(), - }, - expectedUsername: "config", - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - HealthzBindAddress: "0.0.0.0:10251", - MetricsBindAddress: "0.0.0.0:10251", - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: configKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - { - SchedulerName: "foo-profile", - Plugins: &kubeschedulerconfig.Plugins{ - Reserve: &kubeschedulerconfig.PluginSet{ - Enabled: []kubeschedulerconfig.Plugin{ - {Name: "foo"}, - }, - }, - }, - }, - { - SchedulerName: "bar-profile", - Plugins: &kubeschedulerconfig.Plugins{ - PreBind: &kubeschedulerconfig.PluginSet{ - Disabled: []kubeschedulerconfig.Plugin{ - {Name: "baz"}, - }, - }, - }, - PluginConfig: []kubeschedulerconfig.PluginConfig{ - { - Name: "foo", - }, - }, - }, - }, - }, - }, - { - name: "no config", - options: &Options{ - Logs: logs.NewOptions(), - }, - expectedError: "no configuration has been provided", - }, - { - name: "Deprecated HardPodAffinitySymmetricWeight", - options: &Options{ - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, _ := newDefaultComponentConfig() - cfg.ClientConnection.Kubeconfig = flagKubeconfig - return *cfg - }(), - Deprecated: &DeprecatedOptions{ - HardPodAffinitySymmetricWeight: 5, - }, - Logs: logs.NewOptions(), - }, - expectedUsername: "flag", - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: flagKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - { - SchedulerName: "default-scheduler", - PluginConfig: []kubeschedulerconfig.PluginConfig{ - { - Name: "InterPodAffinity", - Args: &kubeschedulerconfig.InterPodAffinityArgs{HardPodAffinityWeight: 5}, - }, - }, - }, - }, - }, - }, - { - name: "Deprecated SchedulerName flag", - options: &Options{ - ComponentConfig: func() kubeschedulerconfig.KubeSchedulerConfiguration { - cfg, _ := newDefaultComponentConfig() - cfg.ClientConnection.Kubeconfig = flagKubeconfig - return *cfg - }(), - Deprecated: &DeprecatedOptions{ - SchedulerName: "my-nice-scheduler", - HardPodAffinitySymmetricWeight: 1, - }, - Logs: logs.NewOptions(), - }, - expectedUsername: "flag", - expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{ - Parallelism: 16, - AlgorithmSource: kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource}, - DebuggingConfiguration: componentbaseconfig.DebuggingConfiguration{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: true, - LeaseDuration: metav1.Duration{Duration: 15 * time.Second}, - RenewDeadline: metav1.Duration{Duration: 10 * time.Second}, - RetryPeriod: metav1.Duration{Duration: 2 * time.Second}, - ResourceLock: "leases", - ResourceNamespace: "kube-system", - ResourceName: "kube-scheduler", - }, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: flagKubeconfig, - QPS: 50, - Burst: 100, - ContentType: "application/vnd.kubernetes.protobuf", - }, - PercentageOfNodesToScore: defaultPercentageOfNodesToScore, - PodInitialBackoffSeconds: defaultPodInitialBackoffSeconds, - PodMaxBackoffSeconds: defaultPodMaxBackoffSeconds, - Profiles: []kubeschedulerconfig.KubeSchedulerProfile{ - { - SchedulerName: "my-nice-scheduler", - PluginConfig: []kubeschedulerconfig.PluginConfig{ - { - Name: interpodaffinity.Name, - Args: &kubeschedulerconfig.InterPodAffinityArgs{ - HardPodAffinityWeight: 1, - }, - }, - }, - }, - }, - }, - }, - { - name: "Attempting to set Component Config Profiles and Policy config", - options: &Options{ - ConfigFile: pluginConfigFile, - Deprecated: &DeprecatedOptions{ - PolicyConfigMapName: "bar", - }, - }, - expectedError: "cannot set a Plugin config and Policy config", - }, - { - name: "unknown field", - options: &Options{ - ConfigFile: unknownFieldConfig, - Logs: logs.NewOptions(), - }, - expectedError: "found unknown field: foo", - checkErrFn: runtime.IsStrictDecodingError, - }, - { - name: "duplicate fields", - options: &Options{ - ConfigFile: duplicateFieldConfig, - Logs: logs.NewOptions(), - }, - expectedError: `key "leaderElect" already set`, - checkErrFn: runtime.IsStrictDecodingError, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - // create the config - config, err := tc.options.Config() - - // handle errors - if err != nil { - if tc.expectedError != "" || tc.checkErrFn != nil { - if tc.expectedError != "" { - assert.Contains(t, err.Error(), tc.expectedError) - } - if tc.checkErrFn != nil { - assert.True(t, tc.checkErrFn(err), "got error: %v", err) - } - return - } - assert.NoError(t, err) - return - } - - if diff := cmp.Diff(tc.expectedConfig, config.ComponentConfig); diff != "" { - t.Errorf("incorrect config (-want,+got):\n%s", diff) - } - - // ensure we have a client - if config.Client == nil { - t.Error("unexpected nil client") - return - } - - // test the client talks to the endpoint we expect with the credentials we expect - username = "" - _, err = config.Client.Discovery().RESTClient().Get().AbsPath("/").DoRaw(context.TODO()) - if err != nil { - t.Error(err) - return - } - if username != tc.expectedUsername { - t.Errorf("expected server call with user %q, got %q", tc.expectedUsername, username) - } - }) - } -} diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go deleted file mode 100644 index 8bc747688a3f7..0000000000000 --- a/cmd/kube-scheduler/app/server.go +++ /dev/null @@ -1,347 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app implements a Server object for running the scheduler. -package app - -import ( - "context" - "fmt" - "net/http" - "os" - goruntime "runtime" - - "github.com/spf13/cobra" - - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authorization/authorizer" - genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" - apirequest "k8s.io/apiserver/pkg/endpoints/request" - genericfilters "k8s.io/apiserver/pkg/server/filters" - "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/apiserver/pkg/server/mux" - "k8s.io/apiserver/pkg/server/routes" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/events" - "k8s.io/client-go/tools/leaderelection" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/cli/globalflag" - "k8s.io/component-base/configz" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics/legacyregistry" - "k8s.io/component-base/term" - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - "k8s.io/klog/v2" - schedulerserverconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" - "k8s.io/kubernetes/cmd/kube-scheduler/app/options" - "k8s.io/kubernetes/pkg/scheduler" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" - "k8s.io/kubernetes/pkg/scheduler/framework/runtime" - "k8s.io/kubernetes/pkg/scheduler/metrics/resources" - "k8s.io/kubernetes/pkg/scheduler/profile" -) - -// Option configures a framework.Registry. -type Option func(runtime.Registry) error - -// NewSchedulerCommand creates a *cobra.Command object with default parameters and registryOptions -func NewSchedulerCommand(registryOptions ...Option) *cobra.Command { - opts, err := options.NewOptions() - if err != nil { - klog.Fatalf("unable to initialize command options: %v", err) - } - - cmd := &cobra.Command{ - Use: "kube-scheduler", - Long: `The Kubernetes scheduler is a control plane process which assigns -Pods to Nodes. The scheduler determines which Nodes are valid placements for -each Pod in the scheduling queue according to constraints and available -resources. The scheduler then ranks each valid Node and binds the Pod to a -suitable Node. Multiple different schedulers may be used within a cluster; -kube-scheduler is the reference implementation. -See [scheduling](https://kubernetes.io/docs/concepts/scheduling-eviction/) -for more information about scheduling and the kube-scheduler component.`, - Run: func(cmd *cobra.Command, args []string) { - if err := runCommand(cmd, opts, registryOptions...); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - fs := cmd.Flags() - namedFlagSets := opts.Flags() - verflag.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name()) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - - usageFmt := "Usage:\n %s\n" - cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) - cmd.SetUsageFunc(func(cmd *cobra.Command) error { - fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) - return nil - }) - cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { - fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) - }) - cmd.MarkFlagFilename("config", "yaml", "yml", "json") - - return cmd -} - -// runCommand runs the scheduler. -func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Option) error { - verflag.PrintAndExitIfRequested() - cliflag.PrintFlags(cmd.Flags()) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cc, sched, err := Setup(ctx, opts, registryOptions...) - if err != nil { - return err - } - - return Run(ctx, cc, sched) -} - -// Run executes the scheduler based on the given configuration. It only returns on error or when context is done. -func Run(ctx context.Context, cc *schedulerserverconfig.CompletedConfig, sched *scheduler.Scheduler) error { - // To help debugging, immediately log version - klog.V(1).Infof("Starting Kubernetes Scheduler version %+v", version.Get()) - - // Configz registration. - if cz, err := configz.New("componentconfig"); err == nil { - cz.Set(cc.ComponentConfig) - } else { - return fmt.Errorf("unable to register configz: %s", err) - } - - // Prepare the event broadcaster. - cc.EventBroadcaster.StartRecordingToSink(ctx.Done()) - - // Setup healthz checks. - var checks []healthz.HealthChecker - if cc.ComponentConfig.LeaderElection.LeaderElect { - checks = append(checks, cc.LeaderElection.WatchDog) - } - - waitingForLeader := make(chan struct{}) - isLeader := func() bool { - select { - case _, ok := <-waitingForLeader: - // if channel is closed, we are leading - return !ok - default: - // channel is open, we are waiting for a leader - return false - } - } - - // Start up the healthz server. - if cc.InsecureServing != nil { - separateMetrics := cc.InsecureMetricsServing != nil - handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, separateMetrics, checks...), nil, nil) - if err := cc.InsecureServing.Serve(handler, 0, ctx.Done()); err != nil { - return fmt.Errorf("failed to start healthz server: %v", err) - } - } - if cc.InsecureMetricsServing != nil { - handler := buildHandlerChain(newMetricsHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader), nil, nil) - if err := cc.InsecureMetricsServing.Serve(handler, 0, ctx.Done()); err != nil { - return fmt.Errorf("failed to start metrics server: %v", err) - } - } - if cc.SecureServing != nil { - handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, cc.InformerFactory, isLeader, false, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer) - // TODO: handle stoppedCh returned by c.SecureServing.Serve - if _, err := cc.SecureServing.Serve(handler, 0, ctx.Done()); err != nil { - // fail early for secure handlers, removing the old error loop from above - return fmt.Errorf("failed to start secure server: %v", err) - } - } - - // Start all informers. - cc.InformerFactory.Start(ctx.Done()) - - // Wait for all caches to sync before scheduling. - cc.InformerFactory.WaitForCacheSync(ctx.Done()) - - // If leader election is enabled, runCommand via LeaderElector until done and exit. - if cc.LeaderElection != nil { - cc.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{ - OnStartedLeading: func(ctx context.Context) { - close(waitingForLeader) - sched.Run(ctx) - }, - OnStoppedLeading: func() { - klog.Fatalf("leaderelection lost") - }, - } - leaderElector, err := leaderelection.NewLeaderElector(*cc.LeaderElection) - if err != nil { - return fmt.Errorf("couldn't create leader elector: %v", err) - } - - leaderElector.Run(ctx) - - return fmt.Errorf("lost lease") - } - - // Leader election is disabled, so runCommand inline until done. - close(waitingForLeader) - sched.Run(ctx) - return fmt.Errorf("finished without leader elect") -} - -// buildHandlerChain wraps the given handler with the standard filters. -func buildHandlerChain(handler http.Handler, authn authenticator.Request, authz authorizer.Authorizer) http.Handler { - requestInfoResolver := &apirequest.RequestInfoFactory{} - failedHandler := genericapifilters.Unauthorized(scheme.Codecs) - - handler = genericapifilters.WithAuthorization(handler, authz, scheme.Codecs) - handler = genericapifilters.WithAuthentication(handler, authn, failedHandler, nil) - handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver) - handler = genericapifilters.WithCacheControl(handler) - handler = genericfilters.WithPanicRecovery(handler, requestInfoResolver) - - return handler -} - -func installMetricHandler(pathRecorderMux *mux.PathRecorderMux, informers informers.SharedInformerFactory, isLeader func() bool) { - configz.InstallHandler(pathRecorderMux) - pathRecorderMux.Handle("/metrics", legacyregistry.HandlerWithReset()) - - resourceMetricsHandler := resources.Handler(informers.Core().V1().Pods().Lister()) - pathRecorderMux.HandleFunc("/metrics/resources", func(w http.ResponseWriter, req *http.Request) { - if !isLeader() { - return - } - resourceMetricsHandler.ServeHTTP(w, req) - }) -} - -// newMetricsHandler builds a metrics server from the config. -func newMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool) http.Handler { - pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") - installMetricHandler(pathRecorderMux, informers, isLeader) - if config.EnableProfiling { - routes.Profiling{}.Install(pathRecorderMux) - if config.EnableContentionProfiling { - goruntime.SetBlockProfileRate(1) - } - routes.DebugFlags{}.Install(pathRecorderMux, "v", routes.StringFlagPutHandler(logs.GlogSetter)) - } - return pathRecorderMux -} - -// newHealthzHandler creates a healthz server from the config, and will also -// embed the metrics handler if the healthz and metrics address configurations -// are the same. -func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, informers informers.SharedInformerFactory, isLeader func() bool, separateMetrics bool, checks ...healthz.HealthChecker) http.Handler { - pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") - healthz.InstallHandler(pathRecorderMux, checks...) - if !separateMetrics { - installMetricHandler(pathRecorderMux, informers, isLeader) - } - if config.EnableProfiling { - routes.Profiling{}.Install(pathRecorderMux) - if config.EnableContentionProfiling { - goruntime.SetBlockProfileRate(1) - } - routes.DebugFlags{}.Install(pathRecorderMux, "v", routes.StringFlagPutHandler(logs.GlogSetter)) - } - return pathRecorderMux -} - -func getRecorderFactory(cc *schedulerserverconfig.CompletedConfig) profile.RecorderFactory { - return func(name string) events.EventRecorder { - return cc.EventBroadcaster.NewRecorder(name) - } -} - -// WithPlugin creates an Option based on plugin name and factory. Please don't remove this function: it is used to register out-of-tree plugins, -// hence there are no references to it from the kubernetes scheduler code base. -func WithPlugin(name string, factory runtime.PluginFactory) Option { - return func(registry runtime.Registry) error { - return registry.Register(name, factory) - } -} - -// Setup creates a completed config and a scheduler based on the command args and options -func Setup(ctx context.Context, opts *options.Options, outOfTreeRegistryOptions ...Option) (*schedulerserverconfig.CompletedConfig, *scheduler.Scheduler, error) { - if errs := opts.Validate(); len(errs) > 0 { - return nil, nil, utilerrors.NewAggregate(errs) - } - - c, err := opts.Config() - if err != nil { - return nil, nil, err - } - - // Get the completed config - cc := c.Complete() - - outOfTreeRegistry := make(runtime.Registry) - for _, option := range outOfTreeRegistryOptions { - if err := option(outOfTreeRegistry); err != nil { - return nil, nil, err - } - } - - recorderFactory := getRecorderFactory(&cc) - completedProfiles := make([]kubeschedulerconfig.KubeSchedulerProfile, 0) - // Create the scheduler. - sched, err := scheduler.New(cc.Client, - cc.InformerFactory, - recorderFactory, - ctx.Done(), - scheduler.WithProfiles(cc.ComponentConfig.Profiles...), - scheduler.WithAlgorithmSource(cc.ComponentConfig.AlgorithmSource), - scheduler.WithPercentageOfNodesToScore(cc.ComponentConfig.PercentageOfNodesToScore), - scheduler.WithFrameworkOutOfTreeRegistry(outOfTreeRegistry), - scheduler.WithPodMaxBackoffSeconds(cc.ComponentConfig.PodMaxBackoffSeconds), - scheduler.WithPodInitialBackoffSeconds(cc.ComponentConfig.PodInitialBackoffSeconds), - scheduler.WithExtenders(cc.ComponentConfig.Extenders...), - scheduler.WithParallelism(cc.ComponentConfig.Parallelism), - scheduler.WithBuildFrameworkCapturer(func(profile kubeschedulerconfig.KubeSchedulerProfile) { - // Profiles are processed during Framework instantiation to set default plugins and configurations. Capturing them for logging - completedProfiles = append(completedProfiles, profile) - }), - ) - if err != nil { - return nil, nil, err - } - if err := options.LogOrWriteConfig(opts.WriteConfigTo, &cc.ComponentConfig, completedProfiles); err != nil { - return nil, nil, err - } - - return &cc, sched, nil -} diff --git a/cmd/kube-scheduler/app/server_test.go b/cmd/kube-scheduler/app/server_test.go deleted file mode 100644 index 82e7a23585bbc..0000000000000 --- a/cmd/kube-scheduler/app/server_test.go +++ /dev/null @@ -1,415 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "context" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/spf13/pflag" - "k8s.io/kubernetes/cmd/kube-scheduler/app/options" - kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config" -) - -func TestSetup(t *testing.T) { - // temp dir - tmpDir, err := ioutil.TempDir("", "scheduler-options") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - // https server - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(200) - w.Write([]byte(`ok`)) - })) - defer server.Close() - - configKubeconfig := filepath.Join(tmpDir, "config.kubeconfig") - if err := ioutil.WriteFile(configKubeconfig, []byte(fmt.Sprintf(` -apiVersion: v1 -kind: Config -clusters: -- cluster: - insecure-skip-tls-verify: true - server: %s - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -users: -- name: default - user: - username: config -`, server.URL)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // plugin config - pluginConfigFile := filepath.Join(tmpDir, "plugin.yaml") - if err := ioutil.WriteFile(pluginConfigFile, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -profiles: -- plugins: - preFilter: - enabled: - - name: NodeResourcesFit - - name: NodePorts - disabled: - - name: "*" - filter: - enabled: - - name: NodeResourcesFit - - name: NodePorts - disabled: - - name: "*" - preScore: - enabled: - - name: InterPodAffinity - - name: TaintToleration - disabled: - - name: "*" - score: - enabled: - - name: InterPodAffinity - - name: TaintToleration - disabled: - - name: "*" -`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // multiple profiles config - multiProfilesConfig := filepath.Join(tmpDir, "multi-profiles.yaml") - if err := ioutil.WriteFile(multiProfilesConfig, []byte(fmt.Sprintf(` -apiVersion: kubescheduler.config.k8s.io/v1beta1 -kind: KubeSchedulerConfiguration -clientConnection: - kubeconfig: "%s" -profiles: -- schedulerName: "profile-default-plugins" -- schedulerName: "profile-disable-all-filter-and-score-plugins" - plugins: - preFilter: - disabled: - - name: "*" - filter: - disabled: - - name: "*" - postFilter: - disabled: - - name: "*" - preScore: - disabled: - - name: "*" - score: - disabled: - - name: "*" -`, configKubeconfig)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - // policy config file - policyConfigFile := filepath.Join(tmpDir, "policy-config.yaml") - if err := ioutil.WriteFile(policyConfigFile, []byte(`{ - "kind": "Policy", - "apiVersion": "v1", - "predicates": [ - {"name": "MatchInterPodAffinity"} - ],"priorities": [ - {"name": "InterPodAffinityPriority", "weight": 2} - ]}`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - defaultPlugins := map[string][]kubeschedulerconfig.Plugin{ - "QueueSortPlugin": { - {Name: "PrioritySort"}, - }, - "PreFilterPlugin": { - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - {Name: "PodTopologySpread"}, - {Name: "InterPodAffinity"}, - {Name: "VolumeBinding"}, - }, - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "NodeName"}, - {Name: "TaintToleration"}, - {Name: "NodeAffinity"}, - {Name: "NodePorts"}, - {Name: "NodeResourcesFit"}, - {Name: "VolumeRestrictions"}, - {Name: "EBSLimits"}, - {Name: "GCEPDLimits"}, - {Name: "NodeVolumeLimits"}, - {Name: "AzureDiskLimits"}, - {Name: "VolumeBinding"}, - {Name: "VolumeZone"}, - {Name: "PodTopologySpread"}, - {Name: "InterPodAffinity"}, - }, - "PostFilterPlugin": { - {Name: "DefaultPreemption"}, - }, - "PreScorePlugin": { - {Name: "InterPodAffinity"}, - {Name: "PodTopologySpread"}, - {Name: "TaintToleration"}, - {Name: "NodeAffinity"}, - }, - "ScorePlugin": { - {Name: "NodeResourcesBalancedAllocation", Weight: 1}, - {Name: "ImageLocality", Weight: 1}, - {Name: "InterPodAffinity", Weight: 1}, - {Name: "NodeResourcesLeastAllocated", Weight: 1}, - {Name: "NodeAffinity", Weight: 1}, - {Name: "NodePreferAvoidPods", Weight: 10000}, - {Name: "PodTopologySpread", Weight: 2}, - {Name: "TaintToleration", Weight: 1}, - }, - "BindPlugin": {{Name: "DefaultBinder"}}, - "ReservePlugin": {{Name: "VolumeBinding"}}, - "PreBindPlugin": {{Name: "VolumeBinding"}}, - } - - testcases := []struct { - name string - flags []string - wantPlugins map[string]map[string][]kubeschedulerconfig.Plugin - }{ - { - name: "default config", - flags: []string{ - "--kubeconfig", configKubeconfig, - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "default-scheduler": defaultPlugins, - }, - }, - { - name: "plugin config with single profile", - flags: []string{ - "--config", pluginConfigFile, - "--kubeconfig", configKubeconfig, - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "default-scheduler": { - "BindPlugin": {{Name: "DefaultBinder"}}, - "FilterPlugin": {{Name: "NodeResourcesFit"}, {Name: "NodePorts"}}, - "PreFilterPlugin": {{Name: "NodeResourcesFit"}, {Name: "NodePorts"}}, - "PostFilterPlugin": {{Name: "DefaultPreemption"}}, - "PreScorePlugin": {{Name: "InterPodAffinity"}, {Name: "TaintToleration"}}, - "QueueSortPlugin": {{Name: "PrioritySort"}}, - "ScorePlugin": {{Name: "InterPodAffinity", Weight: 1}, {Name: "TaintToleration", Weight: 1}}, - "ReservePlugin": {{Name: "VolumeBinding"}}, - "PreBindPlugin": {{Name: "VolumeBinding"}}, - }, - }, - }, - { - name: "plugin config with multiple profiles", - flags: []string{ - "--config", multiProfilesConfig, - "--kubeconfig", configKubeconfig, - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "profile-default-plugins": defaultPlugins, - "profile-disable-all-filter-and-score-plugins": { - "BindPlugin": {{Name: "DefaultBinder"}}, - "QueueSortPlugin": {{Name: "PrioritySort"}}, - "ReservePlugin": {{Name: "VolumeBinding"}}, - "PreBindPlugin": {{Name: "VolumeBinding"}}, - }, - }, - }, - { - name: "Deprecated SchedulerName flag", - flags: []string{ - "--kubeconfig", configKubeconfig, - "--scheduler-name", "my-scheduler", - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "my-scheduler": defaultPlugins, - }, - }, - { - name: "default algorithm provider", - flags: []string{ - "--kubeconfig", configKubeconfig, - "--algorithm-provider", "DefaultProvider", - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "default-scheduler": defaultPlugins, - }, - }, - { - name: "cluster autoscaler provider", - flags: []string{ - "--kubeconfig", configKubeconfig, - "--algorithm-provider", "ClusterAutoscalerProvider", - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "default-scheduler": { - "QueueSortPlugin": { - {Name: "PrioritySort"}, - }, - "PreFilterPlugin": { - {Name: "NodeResourcesFit"}, - {Name: "NodePorts"}, - {Name: "PodTopologySpread"}, - {Name: "InterPodAffinity"}, - {Name: "VolumeBinding"}, - }, - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "NodeName"}, - {Name: "TaintToleration"}, - {Name: "NodeAffinity"}, - {Name: "NodePorts"}, - {Name: "NodeResourcesFit"}, - {Name: "VolumeRestrictions"}, - {Name: "EBSLimits"}, - {Name: "GCEPDLimits"}, - {Name: "NodeVolumeLimits"}, - {Name: "AzureDiskLimits"}, - {Name: "VolumeBinding"}, - {Name: "VolumeZone"}, - {Name: "PodTopologySpread"}, - {Name: "InterPodAffinity"}, - }, - "PostFilterPlugin": { - {Name: "DefaultPreemption"}, - }, - "PreScorePlugin": { - {Name: "InterPodAffinity"}, - {Name: "PodTopologySpread"}, - {Name: "TaintToleration"}, - {Name: "NodeAffinity"}, - }, - "ScorePlugin": { - {Name: "NodeResourcesBalancedAllocation", Weight: 1}, - {Name: "ImageLocality", Weight: 1}, - {Name: "InterPodAffinity", Weight: 1}, - {Name: "NodeResourcesMostAllocated", Weight: 1}, - {Name: "NodeAffinity", Weight: 1}, - {Name: "NodePreferAvoidPods", Weight: 10000}, - {Name: "PodTopologySpread", Weight: 2}, - {Name: "TaintToleration", Weight: 1}, - }, - "BindPlugin": {{Name: "DefaultBinder"}}, - "ReservePlugin": {{Name: "VolumeBinding"}}, - "PreBindPlugin": {{Name: "VolumeBinding"}}, - }, - }, - }, - { - name: "policy config file", - flags: []string{ - "--kubeconfig", configKubeconfig, - "--policy-config-file", policyConfigFile, - }, - wantPlugins: map[string]map[string][]kubeschedulerconfig.Plugin{ - "default-scheduler": { - "QueueSortPlugin": {{Name: "PrioritySort"}}, - "PreFilterPlugin": { - {Name: "InterPodAffinity"}, - }, - "FilterPlugin": { - {Name: "NodeUnschedulable"}, - {Name: "TaintToleration"}, - {Name: "InterPodAffinity"}, - }, - "PostFilterPlugin": {{Name: "DefaultPreemption"}}, - "PreScorePlugin": { - {Name: "InterPodAffinity"}, - }, - "ScorePlugin": { - {Name: "InterPodAffinity", Weight: 2}, - }, - "BindPlugin": {{Name: "DefaultBinder"}}, - }, - }, - }, - } - - makeListener := func(t *testing.T) net.Listener { - t.Helper() - l, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatal(err) - } - return l - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - opts, err := options.NewOptions() - if err != nil { - t.Fatal(err) - } - - // use listeners instead of static ports so parallel test runs don't conflict - opts.SecureServing.Listener = makeListener(t) - defer opts.SecureServing.Listener.Close() - opts.CombinedInsecureServing.Metrics.Listener = makeListener(t) - defer opts.CombinedInsecureServing.Metrics.Listener.Close() - opts.CombinedInsecureServing.Healthz.Listener = makeListener(t) - defer opts.CombinedInsecureServing.Healthz.Listener.Close() - - for _, f := range opts.Flags().FlagSets { - fs.AddFlagSet(f) - } - if err := fs.Parse(tc.flags); err != nil { - t.Fatal(err) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - _, sched, err := Setup(ctx, opts) - if err != nil { - t.Fatal(err) - } - - gotPlugins := make(map[string]map[string][]kubeschedulerconfig.Plugin) - for n, p := range sched.Profiles { - gotPlugins[n] = p.ListPlugins() - } - - if diff := cmp.Diff(tc.wantPlugins, gotPlugins); diff != "" { - t.Errorf("unexpected plugins diff (-want, +got): %s", diff) - } - }) - } -} diff --git a/cmd/kube-scheduler/app/testing/BUILD b/cmd/kube-scheduler/app/testing/BUILD deleted file mode 100644 index 8e083a83a0a63..0000000000000 --- a/cmd/kube-scheduler/app/testing/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["testserver.go"], - importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app/testing", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kube-scheduler/app:go_default_library", - "//cmd/kube-scheduler/app/config:go_default_library", - "//cmd/kube-scheduler/app/options:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/component-base/configz:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kube-scheduler/app/testing/testserver.go b/cmd/kube-scheduler/app/testing/testserver.go deleted file mode 100644 index 080f93ccdd4b1..0000000000000 --- a/cmd/kube-scheduler/app/testing/testserver.go +++ /dev/null @@ -1,187 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testing - -import ( - "context" - "fmt" - "io/ioutil" - "net" - "os" - "time" - - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/component-base/configz" - "k8s.io/kubernetes/cmd/kube-scheduler/app" - kubeschedulerconfig "k8s.io/kubernetes/cmd/kube-scheduler/app/config" - "k8s.io/kubernetes/cmd/kube-scheduler/app/options" -) - -// TearDownFunc is to be called to tear down a test server. -type TearDownFunc func() - -// TestServer return values supplied by kube-test-ApiServer -type TestServer struct { - LoopbackClientConfig *restclient.Config // Rest client config using the magic token - Options *options.Options - Config *kubeschedulerconfig.Config - TearDownFn TearDownFunc // TearDown function - TmpDir string // Temp Dir used, by the apiserver -} - -// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie -type Logger interface { - Errorf(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) - Logf(format string, args ...interface{}) -} - -// StartTestServer starts a kube-scheduler. A rest client config and a tear-down func, -// and location of the tmpdir are returned. -// -// Note: we return a tear-down func instead of a stop channel because the later will leak temporary -// files that because Golang testing's call to os.Exit will not give a stop channel go routine -// enough time to remove temporary files. -func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { - ctx, cancel := context.WithCancel(context.Background()) - tearDown := func() { - cancel() - if len(result.TmpDir) != 0 { - os.RemoveAll(result.TmpDir) - } - configz.Delete("componentconfig") - } - defer func() { - if result.TearDownFn == nil { - tearDown() - } - }() - - result.TmpDir, err = ioutil.TempDir("", "kube-scheduler") - if err != nil { - return result, fmt.Errorf("failed to create temp dir: %v", err) - } - - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - - opts, err := options.NewOptions() - if err != nil { - return TestServer{}, err - } - namedFlagSets := opts.Flags() - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - - fs.Parse(customFlags) - - if opts.SecureServing.BindPort != 0 { - opts.SecureServing.Listener, opts.SecureServing.BindPort, err = createListenerOnFreePort() - if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) - } - opts.SecureServing.ServerCert.CertDirectory = result.TmpDir - - t.Logf("kube-scheduler will listen securely on port %d...", opts.SecureServing.BindPort) - } - - if opts.CombinedInsecureServing.BindPort != 0 { - listener, port, err := createListenerOnFreePort() - if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) - } - opts.CombinedInsecureServing.BindPort = port - opts.CombinedInsecureServing.Healthz.Listener = listener - opts.CombinedInsecureServing.Metrics.Listener = listener - t.Logf("kube-scheduler will listen insecurely on port %d...", opts.CombinedInsecureServing.BindPort) - } - - cc, sched, err := app.Setup(ctx, opts) - if err != nil { - return result, fmt.Errorf("failed to create config from options: %v", err) - } - - errCh := make(chan error) - go func(ctx context.Context) { - if err := app.Run(ctx, cc, sched); err != nil { - errCh <- err - } - }(ctx) - - t.Logf("Waiting for /healthz to be ok...") - client, err := kubernetes.NewForConfig(cc.LoopbackClientConfig) - if err != nil { - return result, fmt.Errorf("failed to create a client: %v", err) - } - err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { - select { - case err := <-errCh: - return false, err - default: - } - - result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()) - status := 0 - result.StatusCode(&status) - if status == 200 { - return true, nil - } - return false, nil - }) - if err != nil { - return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) - } - - // from here the caller must call tearDown - result.LoopbackClientConfig = cc.LoopbackClientConfig - result.Options = opts - result.Config = cc.Config - result.TearDownFn = tearDown - - return result, nil -} - -// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed. -func StartTestServerOrDie(t Logger, flags []string) *TestServer { - result, err := StartTestServer(t, flags) - if err == nil { - return &result - } - - t.Fatalf("failed to launch server: %v", err) - return nil -} - -func createListenerOnFreePort() (net.Listener, int, error) { - ln, err := net.Listen("tcp", ":0") - if err != nil { - return nil, 0, err - } - - // get port - tcpAddr, ok := ln.Addr().(*net.TCPAddr) - if !ok { - ln.Close() - return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String()) - } - - return ln, tcpAddr.Port, nil -} diff --git a/cmd/kube-scheduler/scheduler.go b/cmd/kube-scheduler/scheduler.go deleted file mode 100644 index eff69614f06cf..0000000000000 --- a/cmd/kube-scheduler/scheduler.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/clientgo" - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/kubernetes/cmd/kube-scheduler/app" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := app.NewSchedulerCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - // utilflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/kubeadm/.import-restrictions b/cmd/kubeadm/.import-restrictions deleted file mode 100644 index ca9dc3e498a33..0000000000000 --- a/cmd/kubeadm/.import-restrictions +++ /dev/null @@ -1,18 +0,0 @@ -rules: - - selectorRegexp: k8s[.]io/kubelet/ - allowedPrefixes: - - k8s.io/kubelet/config/v1beta1 - - selectorRegexp: k8s[.]io/kube-openapi/ - allowedPrefixes: - - k8s.io/kube-openapi/pkg/util/proto - - selectorRegexp: k8s[.]io/kube-proxy/ - allowedPrefixes: - - k8s.io/kube-proxy/config/v1alpha1 - - selectorRegexp: k8s[.]io/kubernetes - allowedPrefixes: - - k8s.io/kubernetes/cmd/kubeadm - - selectorRegexp: gopkg[.]in - allowedPrefixes: - - gopkg.in/inf.v0 - - gopkg.in/square/go-jose.v2 - - gopkg.in/yaml.v2 diff --git a/cmd/kubeadm/BUILD b/cmd/kubeadm/BUILD deleted file mode 100644 index ae6e741f54e95..0000000000000 --- a/cmd/kubeadm/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kubeadm", - embed = [":go_default_library"], - pure = "on", - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["kubeadm.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm", - deps = [ - "//cmd/kubeadm/app:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app:all-srcs", - "//cmd/kubeadm/test:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/OWNERS b/cmd/kubeadm/OWNERS deleted file mode 100644 index 0d9854c81bd11..0000000000000 --- a/cmd/kubeadm/OWNERS +++ /dev/null @@ -1,26 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- luxas -- timothysc -- fabriziopandini -- neolit123 -- rosti -- ereslibre -reviewers: -- luxas -- timothysc -- fabriziopandini -- neolit123 -- kad -- detiber -- dixudx -- rosti -- yagonobre -- ereslibre -- SataQiu -- yastij -# Might want to add @xiangpengzhao, @stealthybox, @liztio and @chuckha back in the future -labels: -- area/kubeadm -- sig/cluster-lifecycle diff --git a/cmd/kubeadm/app/BUILD b/cmd/kubeadm/app/BUILD deleted file mode 100644 index 288378c65637a..0000000000000 --- a/cmd/kubeadm/app/BUILD +++ /dev/null @@ -1,58 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["kubeadm.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app", - deps = [ - "//cmd/kubeadm/app/cmd:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/apis/kubeadm:all-srcs", - "//cmd/kubeadm/app/apis/output:all-srcs", - "//cmd/kubeadm/app/cmd:all-srcs", - "//cmd/kubeadm/app/componentconfigs:all-srcs", - "//cmd/kubeadm/app/constants:all-srcs", - "//cmd/kubeadm/app/discovery:all-srcs", - "//cmd/kubeadm/app/features:all-srcs", - "//cmd/kubeadm/app/images:all-srcs", - "//cmd/kubeadm/app/phases/addons/dns:all-srcs", - "//cmd/kubeadm/app/phases/addons/proxy:all-srcs", - "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:all-srcs", - "//cmd/kubeadm/app/phases/bootstraptoken/node:all-srcs", - "//cmd/kubeadm/app/phases/certs:all-srcs", - "//cmd/kubeadm/app/phases/controlplane:all-srcs", - "//cmd/kubeadm/app/phases/copycerts:all-srcs", - "//cmd/kubeadm/app/phases/etcd:all-srcs", - "//cmd/kubeadm/app/phases/kubeconfig:all-srcs", - "//cmd/kubeadm/app/phases/kubelet:all-srcs", - "//cmd/kubeadm/app/phases/markcontrolplane:all-srcs", - "//cmd/kubeadm/app/phases/patchnode:all-srcs", - "//cmd/kubeadm/app/phases/selfhosting:all-srcs", - "//cmd/kubeadm/app/phases/upgrade:all-srcs", - "//cmd/kubeadm/app/phases/uploadconfig:all-srcs", - "//cmd/kubeadm/app/preflight:all-srcs", - "//cmd/kubeadm/app/util:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/BUILD b/cmd/kubeadm/app/apis/kubeadm/BUILD deleted file mode 100644 index ccdd7227c6e0b..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/BUILD +++ /dev/null @@ -1,67 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "apiendpoint.go", - "bootstraptokenhelpers.go", - "bootstraptokenstring.go", - "doc.go", - "register.go", - "types.go", - "zz_generated.deepcopy.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm", - deps = [ - "//cmd/kubeadm/app/features:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/util/secrets:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/apis/kubeadm/fuzzer:all-srcs", - "//cmd/kubeadm/app/apis/kubeadm/scheme:all-srcs", - "//cmd/kubeadm/app/apis/kubeadm/v1beta1:all-srcs", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:all-srcs", - "//cmd/kubeadm/app/apis/kubeadm/validation:all-srcs", - ], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = [ - "apiendpoint_test.go", - "bootstraptokenhelpers_test.go", - "bootstraptokenstring_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/apiendpoint.go b/cmd/kubeadm/app/apis/kubeadm/apiendpoint.go deleted file mode 100644 index 99492e1173690..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/apiendpoint.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "net" - "strconv" - - "github.com/pkg/errors" -) - -// APIEndpointFromString returns an APIEndpoint struct based on a "host:port" raw string. -func APIEndpointFromString(apiEndpoint string) (APIEndpoint, error) { - apiEndpointHost, apiEndpointPortStr, err := net.SplitHostPort(apiEndpoint) - if err != nil { - return APIEndpoint{}, errors.Wrapf(err, "invalid advertise address endpoint: %s", apiEndpoint) - } - apiEndpointPort, err := net.LookupPort("tcp", apiEndpointPortStr) - if err != nil { - return APIEndpoint{}, errors.Wrapf(err, "invalid advertise address endpoint port: %s", apiEndpointPortStr) - } - return APIEndpoint{ - AdvertiseAddress: apiEndpointHost, - BindPort: int32(apiEndpointPort), - }, nil -} - -func (endpoint *APIEndpoint) String() string { - return net.JoinHostPort(endpoint.AdvertiseAddress, strconv.FormatInt(int64(endpoint.BindPort), 10)) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/apiendpoint_test.go b/cmd/kubeadm/app/apis/kubeadm/apiendpoint_test.go deleted file mode 100644 index c171caa512c1b..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/apiendpoint_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "reflect" - "testing" -) - -func TestAPIEndpointFromString(t *testing.T) { - var tests = []struct { - apiEndpoint string - expectedEndpoint APIEndpoint - expectedErr bool - }{ - {apiEndpoint: "1.2.3.4:1234", expectedEndpoint: APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}}, - {apiEndpoint: "1.2.3.4:-1", expectedErr: true}, - {apiEndpoint: "1.2.::1234", expectedErr: true}, - {apiEndpoint: "1.2.3.4:65536", expectedErr: true}, - {apiEndpoint: "[::1]:1234", expectedEndpoint: APIEndpoint{AdvertiseAddress: "::1", BindPort: 1234}}, - {apiEndpoint: "[::1]:-1", expectedErr: true}, - {apiEndpoint: "[::1]:65536", expectedErr: true}, - {apiEndpoint: "[::1:1234", expectedErr: true}, - } - for _, rt := range tests { - t.Run(rt.apiEndpoint, func(t *testing.T) { - apiEndpoint, err := APIEndpointFromString(rt.apiEndpoint) - if (err != nil) != rt.expectedErr { - t.Errorf("expected error %v, got %v, error: %v", rt.expectedErr, err != nil, err) - } - if !reflect.DeepEqual(apiEndpoint, rt.expectedEndpoint) { - t.Errorf("expected API endpoint: %v; got: %v", rt.expectedEndpoint, apiEndpoint) - } - }) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go b/cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go deleted file mode 100644 index 48d89b0ce28e6..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "sort" - "strings" - "time" - - "github.com/pkg/errors" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - bootstrapsecretutil "k8s.io/cluster-bootstrap/util/secrets" -) - -// ToSecret converts the given BootstrapToken object to its Secret representation that -// may be submitted to the API Server in order to be stored. -func (bt *BootstrapToken) ToSecret() *v1.Secret { - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: bootstraputil.BootstrapTokenSecretName(bt.Token.ID), - Namespace: metav1.NamespaceSystem, - }, - Type: v1.SecretType(bootstrapapi.SecretTypeBootstrapToken), - Data: encodeTokenSecretData(bt, time.Now()), - } -} - -// encodeTokenSecretData takes the token discovery object and an optional duration and returns the .Data for the Secret -// now is passed in order to be able to used in unit testing -func encodeTokenSecretData(token *BootstrapToken, now time.Time) map[string][]byte { - data := map[string][]byte{ - bootstrapapi.BootstrapTokenIDKey: []byte(token.Token.ID), - bootstrapapi.BootstrapTokenSecretKey: []byte(token.Token.Secret), - } - - if len(token.Description) > 0 { - data[bootstrapapi.BootstrapTokenDescriptionKey] = []byte(token.Description) - } - - // If for some strange reason both token.TTL and token.Expires would be set - // (they are mutually exclusive in validation so this shouldn't be the case), - // token.Expires has higher priority, as can be seen in the logic here. - if token.Expires != nil { - // Format the expiration date accordingly - // TODO: This maybe should be a helper function in bootstraputil? - expirationString := token.Expires.Time.Format(time.RFC3339) - data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) - - } else if token.TTL != nil && token.TTL.Duration > 0 { - // Only if .Expires is unset, TTL might have an effect - // Get the current time, add the specified duration, and format it accordingly - expirationString := now.Add(token.TTL.Duration).Format(time.RFC3339) - data[bootstrapapi.BootstrapTokenExpirationKey] = []byte(expirationString) - } - - for _, usage := range token.Usages { - data[bootstrapapi.BootstrapTokenUsagePrefix+usage] = []byte("true") - } - - if len(token.Groups) > 0 { - data[bootstrapapi.BootstrapTokenExtraGroupsKey] = []byte(strings.Join(token.Groups, ",")) - } - return data -} - -// BootstrapTokenFromSecret returns a BootstrapToken object from the given Secret -func BootstrapTokenFromSecret(secret *v1.Secret) (*BootstrapToken, error) { - // Get the Token ID field from the Secret data - tokenID := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenIDKey) - if len(tokenID) == 0 { - return nil, errors.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name) - } - - // Enforce the right naming convention - if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) { - return nil, errors.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q", - bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID)) - } - - tokenSecret := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenSecretKey) - if len(tokenSecret) == 0 { - return nil, errors.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name) - } - - // Create the BootstrapTokenString object based on the ID and Secret - bts, err := NewBootstrapTokenStringFromIDAndSecret(tokenID, tokenSecret) - if err != nil { - return nil, errors.Wrap(err, "bootstrap Token Secret is invalid and couldn't be parsed") - } - - // Get the description (if any) from the Secret - description := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenDescriptionKey) - - // Expiration time is optional, if not specified this implies the token - // never expires. - secretExpiration := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExpirationKey) - var expires *metav1.Time - if len(secretExpiration) > 0 { - expTime, err := time.Parse(time.RFC3339, secretExpiration) - if err != nil { - return nil, errors.Wrapf(err, "can't parse expiration time of bootstrap token %q", secret.Name) - } - expires = &metav1.Time{Time: expTime} - } - - // Build an usages string slice from the Secret data - var usages []string - for k, v := range secret.Data { - // Skip all fields that don't include this prefix - if !strings.HasPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix) { - continue - } - // Skip those that don't have this usage set to true - if string(v) != "true" { - continue - } - usages = append(usages, strings.TrimPrefix(k, bootstrapapi.BootstrapTokenUsagePrefix)) - } - // Only sort the slice if defined - if usages != nil { - sort.Strings(usages) - } - - // Get the extra groups information from the Secret - // It's done this way to make .Groups be nil in case there is no items, rather than an - // empty slice or an empty slice with a "" string only - var groups []string - groupsString := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenExtraGroupsKey) - g := strings.Split(groupsString, ",") - if len(g) > 0 && len(g[0]) > 0 { - groups = g - } - - return &BootstrapToken{ - Token: bts, - Description: description, - Expires: expires, - Usages: usages, - Groups: groups, - }, nil -} diff --git a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers_test.go b/cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers_test.go deleted file mode 100644 index 58eac205142bb..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenhelpers_test.go +++ /dev/null @@ -1,456 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "encoding/json" - "reflect" - "testing" - "time" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// This timestamp is used as the reference value when computing expiration dates based on TTLs in these unit tests -var refTime = time.Date(1970, time.January, 1, 1, 1, 1, 0, time.UTC) - -func TestToSecret(t *testing.T) { - - var tests = []struct { - bt *BootstrapToken - secret *v1.Secret - }{ - { - &BootstrapToken{ // all together - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Description: "foo", - Expires: &metav1.Time{ - Time: refTime, - }, - Usages: []string{"signing", "authentication"}, - Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, - }, - &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "bootstrap-token-abcdef", - Namespace: "kube-system", - }, - Type: v1.SecretType("bootstrap.kubernetes.io/token"), - Data: map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "description": []byte("foo"), - "expiration": []byte(refTime.Format(time.RFC3339)), - "usage-bootstrap-signing": []byte("true"), - "usage-bootstrap-authentication": []byte("true"), - "auth-extra-groups": []byte("system:bootstrappers,system:bootstrappers:foo"), - }, - }, - }, - } - for _, rt := range tests { - t.Run(rt.bt.Token.ID, func(t *testing.T) { - actual := rt.bt.ToSecret() - if !reflect.DeepEqual(actual, rt.secret) { - t.Errorf( - "failed BootstrapToken.ToSecret():\n\texpected: %v\n\t actual: %v", - rt.secret, - actual, - ) - } - }) - } -} - -func TestBootstrapTokenToSecretRoundtrip(t *testing.T) { - var tests = []struct { - bt *BootstrapToken - }{ - { - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Description: "foo", - Expires: &metav1.Time{ - Time: refTime, - }, - Usages: []string{"authentication", "signing"}, - Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, - }, - }, - } - for _, rt := range tests { - t.Run(rt.bt.Token.ID, func(t *testing.T) { - actual, err := BootstrapTokenFromSecret(rt.bt.ToSecret()) - if err != nil { - t.Errorf("failed BootstrapToken to Secret roundtrip with error: %v", err) - } - if !reflect.DeepEqual(actual, rt.bt) { - t.Errorf( - "failed BootstrapToken to Secret roundtrip:\n\texpected: %v\n\t actual: %v", - rt.bt, - actual, - ) - } - }) - } -} - -func TestEncodeTokenSecretData(t *testing.T) { - var tests = []struct { - name string - bt *BootstrapToken - data map[string][]byte - }{ - { - "the minimum amount of information needed to be specified", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - }, - }, - { - "adds description", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Description: "foo", - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "description": []byte("foo"), - }, - }, - { - "adds ttl", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - TTL: &metav1.Duration{ - Duration: mustParseDuration("2h", t), - }, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "expiration": []byte(refTime.Add(mustParseDuration("2h", t)).Format(time.RFC3339)), - }, - }, - { - "adds expiration", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Expires: &metav1.Time{ - Time: refTime, - }, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "expiration": []byte(refTime.Format(time.RFC3339)), - }, - }, - { - "adds ttl and expiration, should favor expiration", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - TTL: &metav1.Duration{ - Duration: mustParseDuration("2h", t), - }, - Expires: &metav1.Time{ - Time: refTime, - }, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "expiration": []byte(refTime.Format(time.RFC3339)), - }, - }, - { - "adds usages", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Usages: []string{"authentication", "signing"}, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "usage-bootstrap-signing": []byte("true"), - "usage-bootstrap-authentication": []byte("true"), - }, - }, - { - "adds groups", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "auth-extra-groups": []byte("system:bootstrappers,system:bootstrappers:foo"), - }, - }, - { - "all together", - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Description: "foo", - TTL: &metav1.Duration{ - Duration: mustParseDuration("2h", t), - }, - Expires: &metav1.Time{ - Time: refTime, - }, - Usages: []string{"authentication", "signing"}, - Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, - }, - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "description": []byte("foo"), - "expiration": []byte(refTime.Format(time.RFC3339)), - "usage-bootstrap-signing": []byte("true"), - "usage-bootstrap-authentication": []byte("true"), - "auth-extra-groups": []byte("system:bootstrappers,system:bootstrappers:foo"), - }, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := encodeTokenSecretData(rt.bt, refTime) - if !reflect.DeepEqual(actual, rt.data) { - t.Errorf( - "failed encodeTokenSecretData:\n\texpected: %v\n\t actual: %v", - rt.data, - actual, - ) - } - }) - } -} - -func mustParseDuration(durationStr string, t *testing.T) time.Duration { - d, err := time.ParseDuration(durationStr) - if err != nil { - t.Fatalf("couldn't parse duration %q: %v", durationStr, err) - } - return d -} - -func TestBootstrapTokenFromSecret(t *testing.T) { - var tests = []struct { - desc string - name string - data map[string][]byte - bt *BootstrapToken - expectedError bool - }{ - { - "minimum information", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - }, - false, - }, - { - "invalid token id", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdeF"), - "token-secret": []byte("abcdef0123456789"), - }, - nil, - true, - }, - { - "invalid secret naming", - "foo", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - }, - nil, - true, - }, - { - "invalid token secret", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("ABCDEF0123456789"), - }, - nil, - true, - }, - { - "adds description", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "description": []byte("foo"), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Description: "foo", - }, - false, - }, - { - "adds expiration", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "expiration": []byte(refTime.Format(time.RFC3339)), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Expires: &metav1.Time{ - Time: refTime, - }, - }, - false, - }, - { - "invalid expiration", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "expiration": []byte("invalid date"), - }, - nil, - true, - }, - { - "adds usages", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "usage-bootstrap-signing": []byte("true"), - "usage-bootstrap-authentication": []byte("true"), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Usages: []string{"authentication", "signing"}, - }, - false, - }, - { - "should ignore usages that aren't set to true", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "usage-bootstrap-signing": []byte("true"), - "usage-bootstrap-authentication": []byte("true"), - "usage-bootstrap-foo": []byte("false"), - "usage-bootstrap-bar": []byte(""), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Usages: []string{"authentication", "signing"}, - }, - false, - }, - { - "adds groups", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "auth-extra-groups": []byte("system:bootstrappers,system:bootstrappers:foo"), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, - }, - false, - }, - { - "all fields set", - "bootstrap-token-abcdef", - map[string][]byte{ - "token-id": []byte("abcdef"), - "token-secret": []byte("abcdef0123456789"), - "description": []byte("foo"), - "expiration": []byte(refTime.Format(time.RFC3339)), - "usage-bootstrap-signing": []byte("true"), - "usage-bootstrap-authentication": []byte("true"), - "auth-extra-groups": []byte("system:bootstrappers,system:bootstrappers:foo"), - }, - &BootstrapToken{ - Token: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - Description: "foo", - Expires: &metav1.Time{ - Time: refTime, - }, - Usages: []string{"authentication", "signing"}, - Groups: []string{"system:bootstrappers", "system:bootstrappers:foo"}, - }, - false, - }, - } - for _, rt := range tests { - t.Run(rt.desc, func(t *testing.T) { - actual, err := BootstrapTokenFromSecret(&v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: rt.name, - Namespace: "kube-system", - }, - Type: v1.SecretType("bootstrap.kubernetes.io/token"), - Data: rt.data, - }) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed BootstrapTokenFromSecret\n\texpected error: %t\n\t actual error: %v", - rt.expectedError, - err, - ) - } else { - if actual == nil && rt.bt == nil { - // if both pointers are nil, it's okay, just continue - return - } - // If one of the pointers is defined but the other isn't, throw error. If both pointers are defined but unequal, throw error - if (actual == nil && rt.bt != nil) || (actual != nil && rt.bt == nil) || !reflect.DeepEqual(*actual, *rt.bt) { - t.Errorf( - "failed BootstrapTokenFromSecret\n\texpected: %s\n\t actual: %s", - jsonMarshal(rt.bt), - jsonMarshal(actual), - ) - } - } - }) - } -} - -func jsonMarshal(bt *BootstrapToken) string { - b, _ := json.Marshal(*bt) - return string(b) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring.go b/cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring.go deleted file mode 100644 index 2c1175b9d8b8a..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package kubeadm holds the internal kubeadm API types -// Note: This file should be kept in sync with the similar one for the external API -// TODO: The BootstrapTokenString object should move out to either k8s.io/client-go or k8s.io/api in the future -// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now. -package kubeadm - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" -) - -// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used -// for both validation of the practically of the API server from a joining node's point -// of view and as an authentication method for the node in the bootstrap phase of -// "kubeadm join". This token is and should be short-lived -type BootstrapTokenString struct { - ID string - Secret string -} - -// MarshalJSON implements the json.Marshaler interface. -func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil -} - -// UnmarshalJSON implements the json.Unmarshaller interface. -func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error { - // If the token is represented as "", just return quickly without an error - if len(b) == 0 { - return nil - } - - // Remove unnecessary " characters coming from the JSON parser - token := strings.Replace(string(b), `"`, ``, -1) - // Convert the string Token to a BootstrapTokenString object - newbts, err := NewBootstrapTokenString(token) - if err != nil { - return err - } - bts.ID = newbts.ID - bts.Secret = newbts.Secret - return nil -} - -// String returns the string representation of the BootstrapTokenString -func (bts BootstrapTokenString) String() string { - if len(bts.ID) > 0 && len(bts.Secret) > 0 { - return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret) - } - return "" -} - -// NewBootstrapTokenString converts the given Bootstrap Token as a string -// to the BootstrapTokenString object used for serialization/deserialization -// and internal usage. It also automatically validates that the given token -// is of the right format -func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) { - substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) - // TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works) - if len(substrs) != 3 { - return nil, errors.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) - } - - return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil -} - -// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString -// that allows the caller to specify the ID and Secret separately -func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) { - return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret)) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring_test.go b/cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring_test.go deleted file mode 100644 index 040a209050804..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/bootstraptokenstring_test.go +++ /dev/null @@ -1,249 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/pkg/errors" -) - -func TestMarshalJSON(t *testing.T) { - var tests = []struct { - bts BootstrapTokenString - expected string - }{ - {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, `"abcdef.abcdef0123456789"`}, - {BootstrapTokenString{ID: "foo", Secret: "bar"}, `"foo.bar"`}, - {BootstrapTokenString{ID: "h", Secret: "b"}, `"h.b"`}, - } - for _, rt := range tests { - t.Run(rt.bts.ID, func(t *testing.T) { - b, err := json.Marshal(rt.bts) - if err != nil { - t.Fatalf("json.Marshal returned an unexpected error: %v", err) - } - if string(b) != rt.expected { - t.Errorf( - "failed BootstrapTokenString.MarshalJSON:\n\texpected: %s\n\t actual: %s", - rt.expected, - string(b), - ) - } - }) - } -} - -func TestUnmarshalJSON(t *testing.T) { - var tests = []struct { - input string - bts *BootstrapTokenString - expectedError bool - }{ - {`"f.s"`, &BootstrapTokenString{}, true}, - {`"abcdef."`, &BootstrapTokenString{}, true}, - {`"abcdef:abcdef0123456789"`, &BootstrapTokenString{}, true}, - {`abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, - {`"abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, - {`"abcdef.ABCDEF0123456789"`, &BootstrapTokenString{}, true}, - {`"abcdef.abcdef0123456789"`, &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, false}, - {`"123456.aabbccddeeffgghh"`, &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}, false}, - } - for _, rt := range tests { - t.Run(rt.input, func(t *testing.T) { - newbts := &BootstrapTokenString{} - err := json.Unmarshal([]byte(rt.input), newbts) - if (err != nil) != rt.expectedError { - t.Errorf("failed BootstrapTokenString.UnmarshalJSON:\n\texpected error: %t\n\t actual error: %v", rt.expectedError, err) - } else if !reflect.DeepEqual(rt.bts, newbts) { - t.Errorf( - "failed BootstrapTokenString.UnmarshalJSON:\n\texpected: %v\n\t actual: %v", - rt.bts, - newbts, - ) - } - }) - } -} - -func TestJSONRoundtrip(t *testing.T) { - var tests = []struct { - input string - bts *BootstrapTokenString - }{ - {`"abcdef.abcdef0123456789"`, nil}, - {"", &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - } - for _, rt := range tests { - t.Run(rt.input, func(t *testing.T) { - if err := roundtrip(rt.input, rt.bts); err != nil { - t.Errorf("failed BootstrapTokenString JSON roundtrip with error: %v", err) - } - }) - } -} - -func roundtrip(input string, bts *BootstrapTokenString) error { - var b []byte - var err error - newbts := &BootstrapTokenString{} - // If string input was specified, roundtrip like this: string -> (unmarshal) -> object -> (marshal) -> string - if len(input) > 0 { - if err := json.Unmarshal([]byte(input), newbts); err != nil { - return errors.Wrap(err, "expected no unmarshal error, got error") - } - if b, err = json.Marshal(newbts); err != nil { - return errors.Wrap(err, "expected no marshal error, got error") - } - if input != string(b) { - return errors.Errorf( - "expected token: %s\n\t actual: %s", - input, - string(b), - ) - } - } else { // Otherwise, roundtrip like this: object -> (marshal) -> string -> (unmarshal) -> object - if b, err = json.Marshal(bts); err != nil { - return errors.Wrap(err, "expected no marshal error, got error") - } - if err := json.Unmarshal(b, newbts); err != nil { - return errors.Wrap(err, "expected no unmarshal error, got error") - } - if !reflect.DeepEqual(bts, newbts) { - return errors.Errorf( - "expected object: %v\n\t actual: %v", - bts, - newbts, - ) - } - } - return nil -} - -func TestTokenFromIDAndSecret(t *testing.T) { - var tests = []struct { - bts BootstrapTokenString - expected string - }{ - {BootstrapTokenString{ID: "foo", Secret: "bar"}, "foo.bar"}, - {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, "abcdef.abcdef0123456789"}, - {BootstrapTokenString{ID: "h", Secret: "b"}, "h.b"}, - } - for _, rt := range tests { - t.Run(rt.bts.ID, func(t *testing.T) { - actual := rt.bts.String() - if actual != rt.expected { - t.Errorf( - "failed BootstrapTokenString.String():\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - }) - } -} - -func TestNewBootstrapTokenString(t *testing.T) { - var tests = []struct { - token string - expectedError bool - bts *BootstrapTokenString - }{ - {token: "", expectedError: true, bts: nil}, - {token: ".", expectedError: true, bts: nil}, - {token: "1234567890123456789012", expectedError: true, bts: nil}, // invalid parcel size - {token: "12345.1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {token: ".1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {token: "123456.", expectedError: true, bts: nil}, // invalid parcel size - {token: "123456:1234567890.123456", expectedError: true, bts: nil}, // invalid separation - {token: "abcdef:1234567890123456", expectedError: true, bts: nil}, // invalid separation - {token: "Abcdef.1234567890123456", expectedError: true, bts: nil}, // invalid token id - {token: "123456.AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret - {token: "123456.AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character - {token: "abc*ef.1234567890123456", expectedError: true, bts: nil}, // invalid character - {token: "abcdef.1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, - {token: "123456.aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, - {token: "abcdef.abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - {token: "123456.1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, - } - for _, rt := range tests { - t.Run(rt.token, func(t *testing.T) { - actual, err := NewBootstrapTokenString(rt.token) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed NewBootstrapTokenString for the token %q\n\texpected error: %t\n\t actual error: %v", - rt.token, - rt.expectedError, - err, - ) - } else if !reflect.DeepEqual(actual, rt.bts) { - t.Errorf( - "failed NewBootstrapTokenString for the token %q\n\texpected: %v\n\t actual: %v", - rt.token, - rt.bts, - actual, - ) - } - }) - } -} - -func TestNewBootstrapTokenStringFromIDAndSecret(t *testing.T) { - var tests = []struct { - id, secret string - expectedError bool - bts *BootstrapTokenString - }{ - {id: "", secret: "", expectedError: true, bts: nil}, - {id: "1234567890123456789012", secret: "", expectedError: true, bts: nil}, // invalid parcel size - {id: "12345", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {id: "", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {id: "123456", secret: "", expectedError: true, bts: nil}, // invalid parcel size - {id: "Abcdef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid token id - {id: "123456", secret: "AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret - {id: "123456", secret: "AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character - {id: "abc*ef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid character - {id: "abcdef", secret: "1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, - {id: "123456", secret: "aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, - {id: "abcdef", secret: "abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - {id: "123456", secret: "1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, - } - for _, rt := range tests { - t.Run(rt.id, func(t *testing.T) { - actual, err := NewBootstrapTokenStringFromIDAndSecret(rt.id, rt.secret) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected error: %t\n\t actual error: %v", - rt.id, - rt.secret, - rt.expectedError, - err, - ) - } else if !reflect.DeepEqual(actual, rt.bts) { - t.Errorf( - "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected: %v\n\t actual: %v", - rt.id, - rt.secret, - rt.bts, - actual, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/doc.go b/cmd/kubeadm/app/apis/kubeadm/doc.go deleted file mode 100644 index 8998f14fe7952..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/doc.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:deepcopy-gen=package -// +groupName=kubeadm.k8s.io - -// Package kubeadm is the package that contains the libraries that drive the kubeadm binary. -// kubeadm is responsible for handling a Kubernetes cluster's lifecycle. -package kubeadm // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD b/cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD deleted file mode 100644 index da48e87fc4479..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["fuzzer.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/fuzzer", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//vendor/github.com/google/gofuzz:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["fuzzer_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go deleted file mode 100644 index c27bcdbca9616..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fuzzer - -import ( - fuzz "github.com/google/gofuzz" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// Funcs returns the fuzzer functions for the kubeadm apis. -func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { - return []interface{}{ - fuzzInitConfiguration, - fuzzClusterConfiguration, - fuzzComponentConfigMap, - fuzzDNS, - fuzzNodeRegistration, - fuzzLocalEtcd, - fuzzNetworking, - fuzzJoinConfiguration, - fuzzJoinControlPlane, - } -} - -func fuzzInitConfiguration(obj *kubeadm.InitConfiguration, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - - // Since ClusterConfiguration never get serialized in the external variant of InitConfiguration, - // it is necessary to apply external api defaults here to get the round trip internal->external->internal working. - // More specifically: - // internal with manually applied defaults -> external object : loosing ClusterConfiguration) -> internal object with automatically applied defaults - obj.ClusterConfiguration = kubeadm.ClusterConfiguration{ - APIServer: kubeadm.APIServer{ - TimeoutForControlPlane: &metav1.Duration{ - Duration: constants.DefaultControlPlaneTimeout, - }, - }, - DNS: kubeadm.DNS{ - Type: kubeadm.CoreDNS, - }, - CertificatesDir: v1beta2.DefaultCertificatesDir, - ClusterName: v1beta2.DefaultClusterName, - Etcd: kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: v1beta2.DefaultEtcdDataDir, - }, - }, - ImageRepository: v1beta2.DefaultImageRepository, - KubernetesVersion: v1beta2.DefaultKubernetesVersion, - Networking: kubeadm.Networking{ - ServiceSubnet: v1beta2.DefaultServicesSubnet, - DNSDomain: v1beta2.DefaultServiceDNSDomain, - }, - } - // Adds the default bootstrap token to get the round working - obj.BootstrapTokens = []kubeadm.BootstrapToken{ - { - // Description - // Expires - Groups: []string{"foo"}, - // Token - TTL: &metav1.Duration{Duration: 1234}, - Usages: []string{"foo"}, - }, - } - - // Pin values for fields that are not present in v1beta1 - obj.CertificateKey = "" -} - -func fuzzNodeRegistration(obj *kubeadm.NodeRegistrationOptions, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.IgnorePreflightErrors = nil -} - -func fuzzClusterConfiguration(obj *kubeadm.ClusterConfiguration, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.CertificatesDir = "foo" - obj.CIImageRepository = "" //This fields doesn't exists in public API >> using default to get the roundtrip test pass - obj.ClusterName = "bar" - obj.ImageRepository = "baz" - obj.KubernetesVersion = "qux" - obj.APIServer.TimeoutForControlPlane = &metav1.Duration{ - Duration: constants.DefaultControlPlaneTimeout, - } -} - -func fuzzDNS(obj *kubeadm.DNS, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil - obj.Type = kubeadm.CoreDNS -} - -func fuzzComponentConfigMap(obj *kubeadm.ComponentConfigMap, c fuzz.Continue) { - // This is intentionally empty because component config does not exists in the public api - // (empty mean all ComponentConfigs fields nil, and this is necessary for getting roundtrip passing) -} - -func fuzzLocalEtcd(obj *kubeadm.LocalEtcd, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.DataDir = "foo" -} - -func fuzzNetworking(obj *kubeadm.Networking, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.DNSDomain = "foo" - obj.ServiceSubnet = "bar" -} - -func fuzzJoinConfiguration(obj *kubeadm.JoinConfiguration, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail) - obj.CACertPath = "foo" - obj.Discovery = kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{Token: "baz"}, - TLSBootstrapToken: "qux", - Timeout: &metav1.Duration{Duration: 1234}, - } -} - -func fuzzJoinControlPlane(obj *kubeadm.JoinControlPlane, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - // Pin values for fields that are not present in v1beta1 - obj.CertificateKey = "" -} diff --git a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer_test.go b/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer_test.go deleted file mode 100644 index 76014a56ff1db..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer_test.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fuzzer - -import ( - "testing" - - "k8s.io/apimachinery/pkg/api/apitesting/roundtrip" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" -) - -func TestRoundTripTypes(t *testing.T) { - roundtrip.RoundTripTestForAPIGroup(t, scheme.AddToScheme, Funcs) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/register.go b/cmd/kubeadm/app/apis/kubeadm/register.go deleted file mode 100644 index 3f09aed5bbc82..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/register.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupName is the group name use in this package -const GroupName = "kubeadm.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} - -var ( - // SchemeBuilder points to a list of functions added to Scheme. - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme applies all the stored functions to the scheme. - AddToScheme = SchemeBuilder.AddToScheme -) - -// Kind takes an unqualified kind and returns a Group qualified GroupKind -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &InitConfiguration{}, - &ClusterConfiguration{}, - &ClusterStatus{}, - &JoinConfiguration{}, - ) - return nil -} diff --git a/cmd/kubeadm/app/apis/kubeadm/scheme/BUILD b/cmd/kubeadm/app/apis/kubeadm/scheme/BUILD deleted file mode 100644 index eea86a49daccb..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/scheme/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["scheme.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta1:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/scheme/scheme.go b/cmd/kubeadm/app/apis/kubeadm/scheme/scheme.go deleted file mode 100644 index 69366d6af03cc..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/scheme/scheme.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheme - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" -) - -// Scheme is the runtime.Scheme to which all kubeadm api types are registered. -var Scheme = runtime.NewScheme() - -// Codecs provides access to encoding and decoding for the scheme. -var Codecs = serializer.NewCodecFactory(Scheme) - -func init() { - metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - AddToScheme(Scheme) -} - -// AddToScheme builds the kubeadm scheme using all known versions of the kubeadm api. -func AddToScheme(scheme *runtime.Scheme) { - utilruntime.Must(kubeadm.AddToScheme(scheme)) - utilruntime.Must(v1beta1.AddToScheme(scheme)) - utilruntime.Must(v1beta2.AddToScheme(scheme)) - utilruntime.Must(scheme.SetVersionPriority(v1beta2.SchemeGroupVersion)) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go deleted file mode 100644 index 1759b76551398..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ /dev/null @@ -1,457 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "crypto/x509" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// InitConfiguration contains a list of fields that are specifically "kubeadm init"-only runtime -// information. The cluster-wide config is stored in ClusterConfiguration. The InitConfiguration -// object IS NOT uploaded to the kubeadm-config ConfigMap in the cluster, only the -// ClusterConfiguration is. -type InitConfiguration struct { - metav1.TypeMeta - - // ClusterConfiguration holds the cluster-wide information, and embeds that struct (which can be (un)marshalled separately as well) - // When InitConfiguration is marshalled to bytes in the external version, this information IS NOT preserved (which can be seen from - // the `json:"-"` tag in the external variant of these API types. - ClusterConfiguration `json:"-"` - - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - BootstrapTokens []BootstrapToken - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions - - // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node - // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint - // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This - // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible - // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process - // fails you may set the desired value here. - LocalAPIEndpoint APIEndpoint - - // CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in - // a secret in the cluster during the uploadcerts init phase. - CertificateKey string -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster -type ClusterConfiguration struct { - metav1.TypeMeta - - // ComponentConfigs holds component configs known to kubeadm, should long-term only exist in the internal kubeadm API - // +k8s:conversion-gen=false - ComponentConfigs ComponentConfigMap - - // Etcd holds configuration for etcd. - Etcd Etcd - - // Networking holds configuration for the networking topology of the cluster. - Networking Networking - // KubernetesVersion is the target version of the control plane. - KubernetesVersion string - - // ControlPlaneEndpoint sets a stable IP address or DNS name for the control plane; it - // can be a valid IP address or a RFC-1123 DNS subdomain, both with optional TCP port. - // In case the ControlPlaneEndpoint is not specified, the AdvertiseAddress + BindPort - // are used; in case the ControlPlaneEndpoint is specified but without a TCP port, - // the BindPort is used. - // Possible usages are: - // e.g. In a cluster with more than one control plane instances, this field should be - // assigned the address of the external load balancer in front of the - // control plane instances. - // e.g. in environments with enforced node recycling, the ControlPlaneEndpoint - // could be used for assigning a stable DNS to the control plane. - ControlPlaneEndpoint string - - // APIServer contains extra settings for the API server control plane component - APIServer APIServer - - // ControllerManager contains extra settings for the controller manager control plane component - ControllerManager ControlPlaneComponent - - // Scheduler contains extra settings for the scheduler control plane component - Scheduler ControlPlaneComponent - - // DNS defines the options for the DNS add-on installed in the cluster. - DNS DNS - - // CertificatesDir specifies where to store or look for all required certificates. - CertificatesDir string - - // ImageRepository sets the container registry to pull images from. - // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) - // `gcr.io/kubernetes-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` - // will be used for all the other images. - ImageRepository string - - // CIImageRepository is the container registry for core images generated by CI. - // Useful for running kubeadm with images from CI builds. - // +k8s:conversion-gen=false - CIImageRepository string - - // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images - // DEPRECATED: As hyperkube is itself deprecated, this fields is too. It will be removed in future kubeadm config versions, kubeadm - // will print multiple warnings when set to true, and at some point it may become ignored. - UseHyperKubeImage bool - - // FeatureGates enabled by the user. - FeatureGates map[string]bool - - // The cluster name - ClusterName string -} - -// ControlPlaneComponent holds settings common to control plane component of the cluster -type ControlPlaneComponent struct { - // ExtraArgs is an extra set of flags to pass to the control plane component. - // TODO: This is temporary and ideally we would like to switch all components to - // use ComponentConfig + ConfigMaps. - ExtraArgs map[string]string - - // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. - ExtraVolumes []HostPathMount -} - -// APIServer holds settings necessary for API server deployments in the cluster -type APIServer struct { - ControlPlaneComponent - - // CertSANs sets extra Subject Alternative Names for the API Server signing cert. - CertSANs []string - - // TimeoutForControlPlane controls the timeout that we use for API server to appear - TimeoutForControlPlane *metav1.Duration -} - -// DNSAddOnType defines string identifying DNS add-on types -type DNSAddOnType string - -const ( - // CoreDNS add-on type - CoreDNS DNSAddOnType = "CoreDNS" - - // KubeDNS add-on type - KubeDNS DNSAddOnType = "kube-dns" -) - -// DNS defines the DNS addon that should be used in the cluster -type DNS struct { - // Type defines the DNS add-on to be used - Type DNSAddOnType - - // ImageMeta allows to customize the image used for the DNS component - ImageMeta `json:",inline"` -} - -// ImageMeta allows to customize the image used for components that are not -// originated from the Kubernetes/Kubernetes release process -type ImageMeta struct { - // ImageRepository sets the container registry to pull images from. - // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. - ImageRepository string - - // ImageTag allows to specify a tag for the image. - // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. - ImageTag string - - //TODO: evaluate if we need also a ImageName based on user feedbacks -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterStatus contains the cluster status. The ClusterStatus will be stored in the kubeadm-config -// ConfigMap in the cluster, and then updated by kubeadm when additional control plane instance joins or leaves the cluster. -type ClusterStatus struct { - metav1.TypeMeta - - // APIEndpoints currently available in the cluster, one for each control plane/api server instance. - // The key of the map is the IP of the host's default interface - APIEndpoints map[string]APIEndpoint -} - -// APIEndpoint struct contains elements of API server instance deployed on a node. -type APIEndpoint struct { - // AdvertiseAddress sets the IP address for the API server to advertise. - AdvertiseAddress string - - // BindPort sets the secure port for the API Server to bind to. - // Defaults to 6443. - BindPort int32 -} - -// NodeRegistrationOptions holds fields that relate to registering a new control-plane or node to the cluster, either via "kubeadm init" or "kubeadm join" -type NodeRegistrationOptions struct { - - // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm join` operation. - // This field is also used in the CommonName field of the kubelet's client certificate to the API server. - // Defaults to the hostname of the node if not provided. - Name string - - // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use - CRISocket string - - // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process - // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an - // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. - Taints []v1.Taint - - // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file - // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap - // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - KubeletExtraArgs map[string]string - - // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered. - IgnorePreflightErrors []string -} - -// Networking contains elements describing cluster's networking configuration. -type Networking struct { - // ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12". - ServiceSubnet string - // PodSubnet is the subnet used by pods. - PodSubnet string - // DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local". - DNSDomain string -} - -// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster -// TODO: The BootstrapToken object should move out to either k8s.io/client-go or k8s.io/api in the future -// (probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now. -type BootstrapToken struct { - // Token is used for establishing bidirectional trust between nodes and control-planes. - // Used for joining nodes in the cluster. - Token *BootstrapTokenString - // Description sets a human-friendly message why this token exists and what it's used - // for, so other administrators can know its purpose. - Description string - // TTL defines the time to live for this token. Defaults to 24h. - // Expires and TTL are mutually exclusive. - TTL *metav1.Duration - // Expires specifies the timestamp when this token expires. Defaults to being set - // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. - Expires *metav1.Time - // Usages describes the ways in which this token can be used. Can by default be used - // for establishing bidirectional trust, but that can be changed here. - Usages []string - // Groups specifies the extra groups that this token will authenticate as when/if - // used for authentication - Groups []string -} - -// Etcd contains elements describing Etcd configuration. -type Etcd struct { - - // Local provides configuration knobs for configuring the local etcd instance - // Local and External are mutually exclusive - Local *LocalEtcd - - // External describes how to connect to an external etcd cluster - // Local and External are mutually exclusive - External *ExternalEtcd -} - -// LocalEtcd describes that kubeadm should run an etcd cluster locally -type LocalEtcd struct { - // ImageMeta allows to customize the container used for etcd - ImageMeta `json:",inline"` - - // DataDir is the directory etcd will place its data. - // Defaults to "/var/lib/etcd". - DataDir string - - // ExtraArgs are extra arguments provided to the etcd binary - // when run inside a static pod. - ExtraArgs map[string]string - - // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. - ServerCertSANs []string - // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. - PeerCertSANs []string -} - -// ExternalEtcd describes an external etcd cluster -type ExternalEtcd struct { - - // Endpoints of etcd members. Useful for using external etcd. - // If not provided, kubeadm will run etcd in a static pod. - Endpoints []string - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - CAFile string - // CertFile is an SSL certification file used to secure etcd communication. - CertFile string - // KeyFile is an SSL key file used to secure etcd communication. - KeyFile string -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// JoinConfiguration contains elements describing a particular node. -type JoinConfiguration struct { - metav1.TypeMeta - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions - - // CACertPath is the path to the SSL certificate authority used to - // secure comunications between node and control-plane. - // Defaults to "/etc/kubernetes/pki/ca.crt". - CACertPath string - - // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process - Discovery Discovery - - // ControlPlane defines the additional control plane instance to be deployed on the joining node. - // If nil, no additional control plane instance will be deployed. - ControlPlane *JoinControlPlane -} - -// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. -type JoinControlPlane struct { - // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. - LocalAPIEndpoint APIEndpoint - - // CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret - // upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration. - CertificateKey string -} - -// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process -type Discovery struct { - // BootstrapToken is used to set the options for bootstrap token based discovery - // BootstrapToken and File are mutually exclusive - BootstrapToken *BootstrapTokenDiscovery - - // File is used to specify a file or URL to a kubeconfig file from which to load cluster information - // BootstrapToken and File are mutually exclusive - File *FileDiscovery - - // TLSBootstrapToken is a token used for TLS bootstrapping. - // If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token, but can be overridden. - // If .File is set, this field **must be set** in case the KubeConfigFile does not contain any other authentication information - TLSBootstrapToken string - - // Timeout modifies the discovery timeout - Timeout *metav1.Duration -} - -// BootstrapTokenDiscovery is used to set the options for bootstrap token based discovery -type BootstrapTokenDiscovery struct { - // Token is a token used to validate cluster information - // fetched from the control-plane. - Token string - - // APIServerEndpoint is an IP or domain name to the API server from which info will be fetched. - APIServerEndpoint string - - // CACertHashes specifies a set of public key pins to verify - // when token-based discovery is used. The root CA found during discovery - // must match one of these values. Specifying an empty set disables root CA - // pinning, which can be unsafe. Each hash is specified as ":", - // where the only currently supported type is "sha256". This is a hex-encoded - // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded - // ASN.1. These hashes can be calculated using, for example, OpenSSL. - CACertHashes []string - - // UnsafeSkipCAVerification allows token-based discovery - // without CA verification via CACertHashes. This can weaken - // the security of kubeadm since other nodes can impersonate the control-plane. - UnsafeSkipCAVerification bool -} - -// FileDiscovery is used to specify a file or URL to a kubeconfig file from which to load cluster information -type FileDiscovery struct { - // KubeConfigPath is used to specify the actual file path or URL to the kubeconfig file from which to load cluster information - KubeConfigPath string -} - -// GetControlPlaneImageRepository returns name of image repository -// for control plane images (API,Controller Manager,Scheduler and Proxy) -// It will override location with CI registry name in case user requests special -// Kubernetes version from CI build area. -// (See: kubeadmconstants.DefaultCIImageRepository) -func (cfg *ClusterConfiguration) GetControlPlaneImageRepository() string { - if cfg.CIImageRepository != "" { - return cfg.CIImageRepository - } - return cfg.ImageRepository -} - -// PublicKeyAlgorithm returns the type of encryption keys used in the cluster. -func (cfg *ClusterConfiguration) PublicKeyAlgorithm() x509.PublicKeyAlgorithm { - if features.Enabled(cfg.FeatureGates, features.PublicKeysECDSA) { - return x509.ECDSA - } - - return x509.RSA -} - -// HostPathMount contains elements describing volumes that are mounted from the -// host. -type HostPathMount struct { - // Name of the volume inside the pod template. - Name string - // HostPath is the path in the host that will be mounted inside - // the pod. - HostPath string - // MountPath is the path inside the pod where hostPath will be mounted. - MountPath string - // ReadOnly controls write access to the volume - ReadOnly bool - // PathType is the type of the HostPath. - PathType v1.HostPathType -} - -// DocumentMap is a convenient way to describe a map between a YAML document and its GVK type -// +k8s:deepcopy-gen=false -type DocumentMap map[schema.GroupVersionKind][]byte - -// ComponentConfig holds a known component config -type ComponentConfig interface { - // DeepCopy should create a new deep copy of the component config in place - DeepCopy() ComponentConfig - - // Marshal is marshalling the config into a YAML document returned as a byte slice - Marshal() ([]byte, error) - - // Unmarshal loads the config from a document map. No config in the document map is no error. - Unmarshal(docmap DocumentMap) error - - // Default patches the component config with kubeadm preferred defaults - Default(cfg *ClusterConfiguration, localAPIEndpoint *APIEndpoint, nodeRegOpts *NodeRegistrationOptions) - - // IsUserSupplied indicates if the component config was supplied or modified by a user or was kubeadm generated - IsUserSupplied() bool - - // SetUserSupplied sets the state of the component config "user supplied" flag to, either true, or false. - SetUserSupplied(userSupplied bool) -} - -// ComponentConfigMap is a map between a group name (as in GVK group) and a ComponentConfig -type ComponentConfigMap map[string]ComponentConfig diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1beta1/BUILD deleted file mode 100644 index 3e2be0164ef6c..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/BUILD +++ /dev/null @@ -1,59 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "bootstraptokenstring.go", - "conversion.go", - "defaults.go", - "defaults_unix.go", - "defaults_windows.go", - "doc.go", - "register.go", - "types.go", - "zz_generated.conversion.go", - "zz_generated.deepcopy.go", - "zz_generated.defaults.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "bootstraptokenstring_test.go", - "conversion_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/bootstraptokenstring.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/bootstraptokenstring.go deleted file mode 100644 index a287a37792792..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/bootstraptokenstring.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" -) - -// DEPRECATED - This group version of BootstrapTokenString is deprecated by apis/kubeadm/v1beta2/BootstrapTokenString. -// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used -// for both validation of the practically of the API server from a joining node's point -// of view and as an authentication method for the node in the bootstrap phase of -// "kubeadm join". This token is and should be short-lived -type BootstrapTokenString struct { - ID string `json:"-"` - Secret string `json:"-"` -} - -// MarshalJSON implements the json.Marshaler interface. -func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil -} - -// UnmarshalJSON implements the json.Unmarshaller interface. -func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error { - // If the token is represented as "", just return quickly without an error - if len(b) == 0 { - return nil - } - - // Remove unnecessary " characters coming from the JSON parser - token := strings.Replace(string(b), `"`, ``, -1) - // Convert the string Token to a BootstrapTokenString object - newbts, err := NewBootstrapTokenString(token) - if err != nil { - return err - } - bts.ID = newbts.ID - bts.Secret = newbts.Secret - return nil -} - -// String returns the string representation of the BootstrapTokenString -func (bts BootstrapTokenString) String() string { - if len(bts.ID) > 0 && len(bts.Secret) > 0 { - return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret) - } - return "" -} - -// NewBootstrapTokenString converts the given Bootstrap Token as a string -// to the BootstrapTokenString object used for serialization/deserialization -// and internal usage. It also automatically validates that the given token -// is of the right format -func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) { - substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) - // TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works) - if len(substrs) != 3 { - return nil, errors.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) - } - - return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil -} - -// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString -// that allows the caller to specify the ID and Secret separately -func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) { - return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret)) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/bootstraptokenstring_test.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/bootstraptokenstring_test.go deleted file mode 100644 index 2601e9b0733fd..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/bootstraptokenstring_test.go +++ /dev/null @@ -1,249 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/pkg/errors" -) - -func TestMarshalJSON(t *testing.T) { - var tests = []struct { - bts BootstrapTokenString - expected string - }{ - {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, `"abcdef.abcdef0123456789"`}, - {BootstrapTokenString{ID: "foo", Secret: "bar"}, `"foo.bar"`}, - {BootstrapTokenString{ID: "h", Secret: "b"}, `"h.b"`}, - } - for _, rt := range tests { - t.Run(rt.bts.ID, func(t *testing.T) { - b, err := json.Marshal(rt.bts) - if err != nil { - t.Fatalf("json.Marshal returned an unexpected error: %v", err) - } - if string(b) != rt.expected { - t.Errorf( - "failed BootstrapTokenString.MarshalJSON:\n\texpected: %s\n\t actual: %s", - rt.expected, - string(b), - ) - } - }) - } -} - -func TestUnmarshalJSON(t *testing.T) { - var tests = []struct { - input string - bts *BootstrapTokenString - expectedError bool - }{ - {`"f.s"`, &BootstrapTokenString{}, true}, - {`"abcdef."`, &BootstrapTokenString{}, true}, - {`"abcdef:abcdef0123456789"`, &BootstrapTokenString{}, true}, - {`abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, - {`"abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, - {`"abcdef.ABCDEF0123456789"`, &BootstrapTokenString{}, true}, - {`"abcdef.abcdef0123456789"`, &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, false}, - {`"123456.aabbccddeeffgghh"`, &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}, false}, - } - for _, rt := range tests { - t.Run(rt.input, func(t *testing.T) { - newbts := &BootstrapTokenString{} - err := json.Unmarshal([]byte(rt.input), newbts) - if (err != nil) != rt.expectedError { - t.Errorf("failed BootstrapTokenString.UnmarshalJSON:\n\texpected error: %t\n\t actual error: %v", rt.expectedError, err) - } else if !reflect.DeepEqual(rt.bts, newbts) { - t.Errorf( - "failed BootstrapTokenString.UnmarshalJSON:\n\texpected: %v\n\t actual: %v", - rt.bts, - newbts, - ) - } - }) - } -} - -func TestJSONRoundtrip(t *testing.T) { - var tests = []struct { - input string - bts *BootstrapTokenString - }{ - {`"abcdef.abcdef0123456789"`, nil}, - {"", &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - } - for _, rt := range tests { - t.Run(rt.input, func(t *testing.T) { - if err := roundtrip(rt.input, rt.bts); err != nil { - t.Errorf("failed BootstrapTokenString JSON roundtrip with error: %v", err) - } - }) - } -} - -func roundtrip(input string, bts *BootstrapTokenString) error { - var b []byte - var err error - newbts := &BootstrapTokenString{} - // If string input was specified, roundtrip like this: string -> (unmarshal) -> object -> (marshal) -> string - if len(input) > 0 { - if err := json.Unmarshal([]byte(input), newbts); err != nil { - return errors.Wrap(err, "expected no unmarshal error, got error") - } - if b, err = json.Marshal(newbts); err != nil { - return errors.Wrap(err, "expected no marshal error, got error") - } - if input != string(b) { - return errors.Errorf( - "expected token: %s\n\t actual: %s", - input, - string(b), - ) - } - } else { // Otherwise, roundtrip like this: object -> (marshal) -> string -> (unmarshal) -> object - if b, err = json.Marshal(bts); err != nil { - return errors.Wrap(err, "expected no marshal error, got error") - } - if err := json.Unmarshal(b, newbts); err != nil { - return errors.Wrap(err, "expected no unmarshal error, got error") - } - if !reflect.DeepEqual(bts, newbts) { - return errors.Errorf( - "expected object: %v\n\t actual: %v", - bts, - newbts, - ) - } - } - return nil -} - -func TestTokenFromIDAndSecret(t *testing.T) { - var tests = []struct { - bts BootstrapTokenString - expected string - }{ - {BootstrapTokenString{ID: "foo", Secret: "bar"}, "foo.bar"}, - {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, "abcdef.abcdef0123456789"}, - {BootstrapTokenString{ID: "h", Secret: "b"}, "h.b"}, - } - for _, rt := range tests { - t.Run(rt.bts.ID, func(t *testing.T) { - actual := rt.bts.String() - if actual != rt.expected { - t.Errorf( - "failed BootstrapTokenString.String():\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - }) - } -} - -func TestNewBootstrapTokenString(t *testing.T) { - var tests = []struct { - token string - expectedError bool - bts *BootstrapTokenString - }{ - {token: "", expectedError: true, bts: nil}, - {token: ".", expectedError: true, bts: nil}, - {token: "1234567890123456789012", expectedError: true, bts: nil}, // invalid parcel size - {token: "12345.1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {token: ".1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {token: "123456.", expectedError: true, bts: nil}, // invalid parcel size - {token: "123456:1234567890.123456", expectedError: true, bts: nil}, // invalid separation - {token: "abcdef:1234567890123456", expectedError: true, bts: nil}, // invalid separation - {token: "Abcdef.1234567890123456", expectedError: true, bts: nil}, // invalid token id - {token: "123456.AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret - {token: "123456.AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character - {token: "abc*ef.1234567890123456", expectedError: true, bts: nil}, // invalid character - {token: "abcdef.1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, - {token: "123456.aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, - {token: "abcdef.abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - {token: "123456.1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, - } - for _, rt := range tests { - t.Run(rt.token, func(t *testing.T) { - actual, err := NewBootstrapTokenString(rt.token) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed NewBootstrapTokenString for the token %q\n\texpected error: %t\n\t actual error: %v", - rt.token, - rt.expectedError, - err, - ) - } else if !reflect.DeepEqual(actual, rt.bts) { - t.Errorf( - "failed NewBootstrapTokenString for the token %q\n\texpected: %v\n\t actual: %v", - rt.token, - rt.bts, - actual, - ) - } - }) - } -} - -func TestNewBootstrapTokenStringFromIDAndSecret(t *testing.T) { - var tests = []struct { - id, secret string - expectedError bool - bts *BootstrapTokenString - }{ - {id: "", secret: "", expectedError: true, bts: nil}, - {id: "1234567890123456789012", secret: "", expectedError: true, bts: nil}, // invalid parcel size - {id: "12345", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {id: "", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {id: "123456", secret: "", expectedError: true, bts: nil}, // invalid parcel size - {id: "Abcdef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid token id - {id: "123456", secret: "AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret - {id: "123456", secret: "AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character - {id: "abc*ef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid character - {id: "abcdef", secret: "1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, - {id: "123456", secret: "aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, - {id: "abcdef", secret: "abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - {id: "123456", secret: "1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, - } - for _, rt := range tests { - t.Run(rt.id, func(t *testing.T) { - actual, err := NewBootstrapTokenStringFromIDAndSecret(rt.id, rt.secret) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected error: %t\n\t actual error: %v", - rt.id, - rt.secret, - rt.expectedError, - err, - ) - } else if !reflect.DeepEqual(actual, rt.bts) { - t.Errorf( - "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected: %v\n\t actual: %v", - rt.id, - rt.secret, - rt.bts, - actual, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion.go deleted file mode 100644 index cc50b68018c34..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/conversion" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { - if err := autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s); err != nil { - return err - } - - if in.CertificateKey != "" { - return errors.New("certificateKey field is not supported by v1beta1 config format") - } - - return nil -} - -func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error { - if err := autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s); err != nil { - return err - } - - if in.CertificateKey != "" { - return errors.New("certificateKey field is not supported by v1beta1 config format") - } - - return nil -} - -func Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { - if err := autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in, out, s); err != nil { - return err - } - - if len(in.IgnorePreflightErrors) > 0 { - return errors.New("ignorePreflightErrors field is not supported by v1beta1 config format") - } - - return nil -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go deleted file mode 100644 index 0609c31e0a785..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "testing" - - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestInternalToVersionedInitConfigurationConversion(t *testing.T) { - testcases := map[string]struct { - in kubeadm.InitConfiguration - expectedError bool - }{ - "conversion succeeds": { - in: kubeadm.InitConfiguration{}, - expectedError: false, - }, - "certificateKey set causes an error": { - in: kubeadm.InitConfiguration{ - CertificateKey: "secret", - }, - expectedError: true, - }, - "ignorePreflightErrors set causes an error": { - in: kubeadm.InitConfiguration{ - NodeRegistration: kubeadm.NodeRegistrationOptions{ - IgnorePreflightErrors: []string{"SomeUndesirableError"}, - }, - }, - expectedError: true, - }, - } - for name, tc := range testcases { - t.Run(name, func(t *testing.T) { - versioned := &InitConfiguration{} - err := Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(&tc.in, versioned, nil) - if err == nil && tc.expectedError { - t.Error("unexpected success") - } else if err != nil && !tc.expectedError { - t.Errorf("unexpected error: %v", err) - } - }) - } -} - -func TestInternalToVersionedJoinConfigurationConversion(t *testing.T) { - testcases := map[string]struct { - in kubeadm.JoinConfiguration - expectedError bool - }{ - "conversion succeeds": { - in: kubeadm.JoinConfiguration{}, - expectedError: false, - }, - "ignorePreflightErrors set causes an error": { - in: kubeadm.JoinConfiguration{ - NodeRegistration: kubeadm.NodeRegistrationOptions{ - IgnorePreflightErrors: []string{"SomeUndesirableError"}, - }, - }, - expectedError: true, - }, - } - for name, tc := range testcases { - t.Run(name, func(t *testing.T) { - versioned := &JoinConfiguration{} - err := Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(&tc.in, versioned, nil) - if err == nil && tc.expectedError { - t.Error("unexpected success") - } else if err != nil && !tc.expectedError { - t.Errorf("unexpected error: %v", err) - } - }) - } -} - -func TestInternalToVersionedNodeRegistrationOptionsConversion(t *testing.T) { - testcases := map[string]struct { - in kubeadm.NodeRegistrationOptions - expectedError bool - }{ - "conversion succeeds": { - in: kubeadm.NodeRegistrationOptions{}, - expectedError: false, - }, - "ignorePreflightErrors set causes an error": { - in: kubeadm.NodeRegistrationOptions{ - IgnorePreflightErrors: []string{"SomeUndesirableError"}, - }, - expectedError: true, - }, - } - for name, tc := range testcases { - t.Run(name, func(t *testing.T) { - versioned := &NodeRegistrationOptions{} - err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(&tc.in, versioned, nil) - if err == nil && tc.expectedError { - t.Error("unexpected success") - } else if err != nil && !tc.expectedError { - t.Errorf("unexpected error: %v", err) - } - }) - } -} - -func TestInternalToVersionedJoinControlPlaneConversion(t *testing.T) { - testcases := map[string]struct { - in kubeadm.JoinControlPlane - expectedError bool - }{ - "conversion succeeds": { - in: kubeadm.JoinControlPlane{}, - expectedError: false, - }, - "certificateKey set causes an error": { - in: kubeadm.JoinControlPlane{ - CertificateKey: "secret", - }, - expectedError: true, - }, - } - for name, tc := range testcases { - t.Run(name, func(t *testing.T) { - versioned := &JoinControlPlane{} - err := Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(&tc.in, versioned, nil) - if err == nil && tc.expectedError { - t.Error("unexpected success") - } else if err != nil && !tc.expectedError { - t.Errorf("unexpected error: %v", err) - } - }) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go deleted file mode 100644 index a241f07cc5198..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults.go +++ /dev/null @@ -1,215 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "net/url" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -const ( - // DefaultServiceDNSDomain defines default cluster-internal domain name for Services and Pods - DefaultServiceDNSDomain = "cluster.local" - // DefaultServicesSubnet defines default service subnet range - DefaultServicesSubnet = "10.96.0.0/12" - // DefaultClusterDNSIP defines default DNS IP - DefaultClusterDNSIP = "10.96.0.10" - // DefaultKubernetesVersion defines default kubernetes version - DefaultKubernetesVersion = "stable-1" - // DefaultAPIBindPort defines default API port - DefaultAPIBindPort = 6443 - // DefaultCertificatesDir defines default certificate directory - DefaultCertificatesDir = "/etc/kubernetes/pki" - // DefaultImageRepository defines default image registry - DefaultImageRepository = "k8s.gcr.io" - // DefaultManifestsDir defines default manifests directory - DefaultManifestsDir = "/etc/kubernetes/manifests" - // DefaultClusterName defines the default cluster name - DefaultClusterName = "kubernetes" - - // DefaultEtcdDataDir defines default location of etcd where static pods will save data to - DefaultEtcdDataDir = "/var/lib/etcd" - // DefaultProxyBindAddressv4 is the default bind address when the advertise address is v4 - DefaultProxyBindAddressv4 = "0.0.0.0" - // DefaultProxyBindAddressv6 is the default bind address when the advertise address is v6 - DefaultProxyBindAddressv6 = "::" - // DefaultDiscoveryTimeout specifies the default discovery timeout for kubeadm (used unless one is specified in the JoinConfiguration) - DefaultDiscoveryTimeout = 5 * time.Minute -) - -var ( - // DefaultAuditPolicyLogMaxAge is defined as a var so its address can be taken - // It is the number of days to store audit logs - DefaultAuditPolicyLogMaxAge = int32(2) -) - -func addDefaultingFuncs(scheme *runtime.Scheme) error { - return RegisterDefaults(scheme) -} - -// SetDefaults_InitConfiguration assigns default values for the InitConfiguration -func SetDefaults_InitConfiguration(obj *InitConfiguration) { - SetDefaults_ClusterConfiguration(&obj.ClusterConfiguration) - SetDefaults_BootstrapTokens(obj) - SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint) -} - -// SetDefaults_ClusterConfiguration assigns default values for the ClusterConfiguration -func SetDefaults_ClusterConfiguration(obj *ClusterConfiguration) { - if obj.KubernetesVersion == "" { - obj.KubernetesVersion = DefaultKubernetesVersion - } - - if obj.Networking.ServiceSubnet == "" { - obj.Networking.ServiceSubnet = DefaultServicesSubnet - } - - if obj.Networking.DNSDomain == "" { - obj.Networking.DNSDomain = DefaultServiceDNSDomain - } - - if obj.CertificatesDir == "" { - obj.CertificatesDir = DefaultCertificatesDir - } - - if obj.ImageRepository == "" { - obj.ImageRepository = DefaultImageRepository - } - - if obj.ClusterName == "" { - obj.ClusterName = DefaultClusterName - } - - SetDefaults_DNS(obj) - SetDefaults_Etcd(obj) - SetDefaults_APIServer(&obj.APIServer) -} - -// SetDefaults_APIServer assigns default values for the API Server -func SetDefaults_APIServer(obj *APIServer) { - if obj.TimeoutForControlPlane == nil { - obj.TimeoutForControlPlane = &metav1.Duration{ - Duration: constants.DefaultControlPlaneTimeout, - } - } -} - -// SetDefaults_DNS assigns default values for the DNS component -func SetDefaults_DNS(obj *ClusterConfiguration) { - if obj.DNS.Type == "" { - obj.DNS.Type = CoreDNS - } -} - -// SetDefaults_Etcd assigns default values for the proxy -func SetDefaults_Etcd(obj *ClusterConfiguration) { - if obj.Etcd.External == nil && obj.Etcd.Local == nil { - obj.Etcd.Local = &LocalEtcd{} - } - if obj.Etcd.Local != nil { - if obj.Etcd.Local.DataDir == "" { - obj.Etcd.Local.DataDir = DefaultEtcdDataDir - } - } -} - -// SetDefaults_JoinConfiguration assigns default values to a regular node -func SetDefaults_JoinConfiguration(obj *JoinConfiguration) { - if obj.CACertPath == "" { - obj.CACertPath = DefaultCACertPath - } - - SetDefaults_JoinControlPlane(obj.ControlPlane) - SetDefaults_Discovery(&obj.Discovery) -} - -func SetDefaults_JoinControlPlane(obj *JoinControlPlane) { - if obj != nil { - SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint) - } -} - -// SetDefaults_Discovery assigns default values for the discovery process -func SetDefaults_Discovery(obj *Discovery) { - if len(obj.TLSBootstrapToken) == 0 && obj.BootstrapToken != nil { - obj.TLSBootstrapToken = obj.BootstrapToken.Token - } - - if obj.Timeout == nil { - obj.Timeout = &metav1.Duration{ - Duration: DefaultDiscoveryTimeout, - } - } - - if obj.File != nil { - SetDefaults_FileDiscovery(obj.File) - } -} - -// SetDefaults_FileDiscovery assigns default values for file based discovery -func SetDefaults_FileDiscovery(obj *FileDiscovery) { - // Make sure file URL becomes path - if len(obj.KubeConfigPath) != 0 { - u, err := url.Parse(obj.KubeConfigPath) - if err == nil && u.Scheme == "file" { - obj.KubeConfigPath = u.Path - } - } -} - -// SetDefaults_BootstrapTokens sets the defaults for the .BootstrapTokens field -// If the slice is empty, it's defaulted with one token. Otherwise it just loops -// through the slice and sets the defaults for the omitempty fields that are TTL, -// Usages and Groups. Token is NOT defaulted with a random one in the API defaulting -// layer, but set to a random value later at runtime if not set before. -func SetDefaults_BootstrapTokens(obj *InitConfiguration) { - - if obj.BootstrapTokens == nil || len(obj.BootstrapTokens) == 0 { - obj.BootstrapTokens = []BootstrapToken{{}} - } - - for i := range obj.BootstrapTokens { - SetDefaults_BootstrapToken(&obj.BootstrapTokens[i]) - } -} - -// SetDefaults_BootstrapToken sets the defaults for an individual Bootstrap Token -func SetDefaults_BootstrapToken(bt *BootstrapToken) { - if bt.TTL == nil { - bt.TTL = &metav1.Duration{ - Duration: constants.DefaultTokenDuration, - } - } - if len(bt.Usages) == 0 { - bt.Usages = constants.DefaultTokenUsages - } - - if len(bt.Groups) == 0 { - bt.Groups = constants.DefaultTokenGroups - } -} - -// SetDefaults_APIEndpoint sets the defaults for the API server instance deployed on a node. -func SetDefaults_APIEndpoint(obj *APIEndpoint) { - if obj.BindPort == 0 { - obj.BindPort = DefaultAPIBindPort - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go deleted file mode 100644 index 18510d4c86ef9..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_unix.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build !windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -const ( - // DefaultCACertPath defines default location of CA certificate on Linux - DefaultCACertPath = "/etc/kubernetes/pki/ca.crt" - // DefaultUrlScheme defines default socket url prefix - DefaultUrlScheme = "unix" -) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go deleted file mode 100644 index 3e2ab2bed5d1c..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/defaults_windows.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -const ( - // DefaultCACertPath defines default location of CA certificate on Windows - DefaultCACertPath = "C:/etc/kubernetes/pki/ca.crt" - // DefaultUrlScheme defines default socket url prefix - DefaultUrlScheme = "npipe" -) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go deleted file mode 100644 index b8ca245f60731..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go +++ /dev/null @@ -1,281 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:defaulter-gen=TypeMeta -// +groupName=kubeadm.k8s.io -// +k8s:deepcopy-gen=package -// +k8s:conversion-gen=k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm - -// Package v1beta1 has been deprecated by v1beta2 -// Package v1beta1 defines the v1beta1 version of the kubeadm configuration file format. -// This version graduates the configuration format to BETA and is a big step towards GA. -// -//A list of changes since v1alpha3: -// - "apiServerEndpoint" in InitConfiguration was renamed to "localAPIEndpoint" for better clarity of what the field -// represents. -// - Common fields in ClusterConfiguration such as "*extraArgs" and "*extraVolumes" for control plane components are now moved -// under component structs - i.e. "apiServer", "controllerManager", "scheduler". -// - "auditPolicy" was removed from ClusterConfiguration. Please use "extraArgs" in "apiServer" to configure this feature instead. -// - "unifiedControlPlaneImage" in ClusterConfiguration was changed to a boolean field called "useHyperKubeImage". -// - ClusterConfiguration now has a "dns" field which can be used to select and configure the cluster DNS addon. -// - "featureGates" still exists under ClusterConfiguration, but there are no supported feature gates in 1.13. -// See the Kubernetes 1.13 changelog for further details. -// - Both "localEtcd" and "dns" configurations now support custom image repositories. -// - The "controlPlane*"-related fields in JoinConfiguration were refactored into a sub-structure. -// - "clusterName" was removed from JoinConfiguration and the name is now fetched from the existing cluster. -// -// Migration from old kubeadm config versions -// -// Please convert your v1alpha3 configuration files to v1beta1 using the "kubeadm config migrate" command of kubeadm v1.13.x -// (conversion from older releases of kubeadm config files requires older release of kubeadm as well e.g. -// kubeadm v1.11 should be used to migrate v1alpha1 to v1alpha2; kubeadm v1.12 should be used to translate v1alpha2 to v1alpha3) -// -// Nevertheless, kubeadm v1.13.x will support reading from v1alpha3 version of the kubeadm config file format, but this support -// will be dropped in the v1.14 release. -// -// Basics -// -// The preferred way to configure kubeadm is to pass an YAML configuration file with the --config option. Some of the -// configuration options defined in the kubeadm config file are also available as command line flags, but only -// the most common/simple use case are supported with this approach. -// -// A kubeadm config file could contain multiple configuration types separated using three dashes (“---”). -// -// kubeadm supports the following configuration types: -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: InitConfiguration -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: ClusterConfiguration -// -// apiVersion: kubelet.config.k8s.io/v1beta1 -// kind: KubeletConfiguration -// -// apiVersion: kubeproxy.config.k8s.io/v1alpha1 -// kind: KubeProxyConfiguration -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: JoinConfiguration -// -// To print the defaults for "init" and "join" actions use the following commands: -// kubeadm config print init-defaults -// kubeadm config print join-defaults -// -// The list of configuration types that must be included in a configuration file depends by the action you are -// performing (init or join) and by the configuration options you are going to use (defaults or advanced customization). -// -// If some configuration types are not provided, or provided only partially, kubeadm will use default values; defaults -// provided by kubeadm includes also enforcing consistency of values across components when required (e.g. -// cluster-cidr flag on controller manager and clusterCIDR on kube-proxy). -// -// Users are always allowed to override default values, with the only exception of a small subset of setting with -// relevance for security (e.g. enforce authorization-mode Node and RBAC on api server) -// -// If the user provides a configuration types that is not expected for the action you are performing, kubeadm will -// ignore those types and print a warning. -// -// Kubeadm init configuration types -// -// When executing kubeadm init with the --config option, the following configuration types could be used: -// InitConfiguration, ClusterConfiguration, KubeProxyConfiguration, KubeletConfiguration, but only one -// between InitConfiguration and ClusterConfiguration is mandatory. -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: InitConfiguration -// bootstrapTokens: -// ... -// nodeRegistration: -// ... -// -// The InitConfiguration type should be used to configure runtime settings, that in case of kubeadm init -// are the configuration of the bootstrap token and all the setting which are specific to the node where kubeadm -// is executed, including: -// -// - NodeRegistration, that holds fields that relate to registering the new node to the cluster; -// use it to customize the node name, the CRI socket to use or any other settings that should apply to this -// node only (e.g. the node ip). -// -// - LocalAPIEndpoint, that represents the endpoint of the instance of the API server to be deployed on this node; -// use it e.g. to customize the API server advertise address. -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: ClusterConfiguration -// networking: -// ... -// etcd: -// ... -// apiServer: -// extraArgs: -// ... -// extraVolumes: -// ... -// ... -// -// The ClusterConfiguration type should be used to configure cluster-wide settings, -// including settings for: -// -// - Networking, that holds configuration for the networking topology of the cluster; use it e.g. to customize -// node subnet or services subnet. -// -// - Etcd configurations; use it e.g. to customize the local etcd or to configure the API server -// for using an external etcd cluster. -// -// - kube-apiserver, kube-scheduler, kube-controller-manager configurations; use it to customize control-plane -// components by adding customized setting or overriding kubeadm default settings. -// -// apiVersion: kubeproxy.config.k8s.io/v1alpha1 -// kind: KubeProxyConfiguration -// ... -// -// The KubeProxyConfiguration type should be used to change the configuration passed to kube-proxy instances deployed -// in the cluster. If this object is not provided or provided only partially, kubeadm applies defaults. -// -// See https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/ or https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration -// for kube proxy official documentation. -// -// apiVersion: kubelet.config.k8s.io/v1beta1 -// kind: KubeletConfiguration -// ... -// -// The KubeletConfiguration type should be used to change the configurations that will be passed to all kubelet instances -// deployed in the cluster. If this object is not provided or provided only partially, kubeadm applies defaults. -// -// See https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ or https://godoc.org/k8s.io/kubelet/config/v1beta1#KubeletConfiguration -// for kubelet official documentation. -// -// Here is a fully populated example of a single YAML file containing multiple -// configuration types to be used during a `kubeadm init` run. -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: InitConfiguration -// bootstrapTokens: -// - token: "9a08jv.c0izixklcxtmnze7" -// description: "kubeadm bootstrap token" -// ttl: "24h" -// - token: "783bde.3f89s0fje9f38fhf" -// description: "another bootstrap token" -// usages: -// - authentication -// - signing -// groups: -// - system:bootstrappers:kubeadm:default-node-token -// nodeRegistration: -// name: "ec2-10-100-0-1" -// criSocket: "/var/run/dockershim.sock" -// taints: -// - key: "kubeadmNode" -// value: "master" -// effect: "NoSchedule" -// kubeletExtraArgs: -// cgroup-driver: "cgroupfs" -// localAPIEndpoint: -// advertiseAddress: "10.100.0.1" -// bindPort: 6443 -// --- -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: ClusterConfiguration -// etcd: -// # one of local or external -// local: -// imageRepository: "k8s.gcr.io" -// imageTag: "3.2.24" -// dataDir: "/var/lib/etcd" -// extraArgs: -// listen-client-urls: "http://10.100.0.1:2379" -// serverCertSANs: -// - "ec2-10-100-0-1.compute-1.amazonaws.com" -// peerCertSANs: -// - "10.100.0.1" -// # external: -// # endpoints: -// # - "10.100.0.1:2379" -// # - "10.100.0.2:2379" -// # caFile: "/etcd/kubernetes/pki/etcd/etcd-ca.crt" -// # certFile: "/etcd/kubernetes/pki/etcd/etcd.crt" -// # keyFile: "/etcd/kubernetes/pki/etcd/etcd.key" -// networking: -// serviceSubnet: "10.96.0.0/12" -// podSubnet: "10.100.0.1/24" -// dnsDomain: "cluster.local" -// kubernetesVersion: "v1.12.0" -// controlPlaneEndpoint: "10.100.0.1:6443" -// apiServer: -// extraArgs: -// authorization-mode: "Node,RBAC" -// extraVolumes: -// - name: "some-volume" -// hostPath: "/etc/some-path" -// mountPath: "/etc/some-pod-path" -// readOnly: false -// pathType: File -// certSANs: -// - "10.100.1.1" -// - "ec2-10-100-0-1.compute-1.amazonaws.com" -// timeoutForControlPlane: 4m0s -// controllerManager: -// extraArgs: -// "node-cidr-mask-size": "20" -// extraVolumes: -// - name: "some-volume" -// hostPath: "/etc/some-path" -// mountPath: "/etc/some-pod-path" -// readOnly: false -// pathType: File -// scheduler: -// extraArgs: -// address: "10.100.0.1" -// extraVolumes: -// - name: "some-volume" -// hostPath: "/etc/some-path" -// mountPath: "/etc/some-pod-path" -// readOnly: false -// pathType: File -// certificatesDir: "/etc/kubernetes/pki" -// imageRepository: "k8s.gcr.io" -// useHyperKubeImage: false -// clusterName: "example-cluster" -// --- -// apiVersion: kubelet.config.k8s.io/v1beta1 -// kind: KubeletConfiguration -// # kubelet specific options here -// --- -// apiVersion: kubeproxy.config.k8s.io/v1alpha1 -// kind: KubeProxyConfiguration -// # kube-proxy specific options here -// -// Kubeadm join configuration types -// -// When executing kubeadm join with the --config option, the JoinConfiguration type should be provided. -// -// apiVersion: kubeadm.k8s.io/v1beta1 -// kind: JoinConfiguration -// ... -// -// The JoinConfiguration type should be used to configure runtime settings, that in case of kubeadm join -// are the discovery method used for accessing the cluster info and all the setting which are specific -// to the node where kubeadm is executed, including: -// -// - NodeRegistration, that holds fields that relate to registering the new node to the cluster; -// use it to customize the node name, the CRI socket to use or any other settings that should apply to this -// node only (e.g. the node ip). -// -// - APIEndpoint, that represents the endpoint of the instance of the API server to be eventually deployed on this node. -// -package v1beta1 // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" - -//TODO: The BootstrapTokenString object should move out to either k8s.io/client-go or k8s.io/api in the future -//(probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/register.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/register.go deleted file mode 100644 index 4446e361aeda1..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/register.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupName is the group name use in this package -const GroupName = "kubeadm.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"} - -var ( - // TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api. - // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. - - // SchemeBuilder points to a list of functions added to Scheme. - SchemeBuilder runtime.SchemeBuilder - localSchemeBuilder = &SchemeBuilder - // AddToScheme applies all the stored functions to the scheme. - AddToScheme = localSchemeBuilder.AddToScheme -) - -func init() { - // We only register manually written functions here. The registration of the - // generated functions takes place in the generated files. The separation - // makes the code compile even when the generated files are missing. - localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) -} - -// Kind takes an unqualified kind and returns a Group qualified GroupKind -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &InitConfiguration{}, - &ClusterConfiguration{}, - &ClusterStatus{}, - &JoinConfiguration{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go deleted file mode 100644 index 61097572c4c10..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go +++ /dev/null @@ -1,394 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// DEPRECATED - This group version of InitConfiguration is deprecated by apis/kubeadm/v1beta2/InitConfiguration. -// InitConfiguration contains a list of elements that is specific "kubeadm init"-only runtime -// information. -type InitConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // ClusterConfiguration holds the cluster-wide information, and embeds that struct (which can be (un)marshalled separately as well) - // When InitConfiguration is marshalled to bytes in the external version, this information IS NOT preserved (which can be seen from - // the `json:"-"` tag. This is due to that when InitConfiguration is (un)marshalled, it turns into two YAML documents, one for the - // InitConfiguration and ClusterConfiguration. Hence, the information must not be duplicated, and is therefore omitted here. - ClusterConfiguration `json:"-"` - - // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. - // After that, the information in the fields IS NOT uploaded to the `kubeadm-config` ConfigMap - // that is used by `kubeadm upgrade` for instance. These fields must be omitempty. - - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature - BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"` - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` - - // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node - // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint - // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This - // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible - // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process - // fails you may set the desired value here. - LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// DEPRECATED - This group version of ClusterConfiguration is deprecated by apis/kubeadm/v1beta2/ClusterConfiguration. -// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster -type ClusterConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // Etcd holds configuration for etcd. - Etcd Etcd `json:"etcd"` - - // Networking holds configuration for the networking topology of the cluster. - Networking Networking `json:"networking"` - - // KubernetesVersion is the target version of the control plane. - KubernetesVersion string `json:"kubernetesVersion"` - - // ControlPlaneEndpoint sets a stable IP address or DNS name for the control plane; it - // can be a valid IP address or a RFC-1123 DNS subdomain, both with optional TCP port. - // In case the ControlPlaneEndpoint is not specified, the AdvertiseAddress + BindPort - // are used; in case the ControlPlaneEndpoint is specified but without a TCP port, - // the BindPort is used. - // Possible usages are: - // e.g. In a cluster with more than one control plane instances, this field should be - // assigned the address of the external load balancer in front of the - // control plane instances. - // e.g. in environments with enforced node recycling, the ControlPlaneEndpoint - // could be used for assigning a stable DNS to the control plane. - ControlPlaneEndpoint string `json:"controlPlaneEndpoint"` - - // APIServer contains extra settings for the API server control plane component - APIServer APIServer `json:"apiServer,omitempty"` - - // ControllerManager contains extra settings for the controller manager control plane component - ControllerManager ControlPlaneComponent `json:"controllerManager,omitempty"` - - // Scheduler contains extra settings for the scheduler control plane component - Scheduler ControlPlaneComponent `json:"scheduler,omitempty"` - - // DNS defines the options for the DNS add-on installed in the cluster. - DNS DNS `json:"dns"` - - // CertificatesDir specifies where to store or look for all required certificates. - CertificatesDir string `json:"certificatesDir"` - - // ImageRepository sets the container registry to pull images from. - // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) - // `gcr.io/kubernetes-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` - // will be used for all the other images. - ImageRepository string `json:"imageRepository"` - - // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images - // DEPRECATED: As hyperkube is itself deprecated, this fields is too. It will be removed in future kubeadm config versions, kubeadm - // will print multiple warnings when set to true, and at some point it may become ignored. - UseHyperKubeImage bool `json:"useHyperKubeImage,omitempty"` - - // FeatureGates enabled by the user. - FeatureGates map[string]bool `json:"featureGates,omitempty"` - - // The cluster name - ClusterName string `json:"clusterName,omitempty"` -} - -// ControlPlaneComponent holds settings common to control plane component of the cluster -type ControlPlaneComponent struct { - // ExtraArgs is an extra set of flags to pass to the control plane component. - // TODO: This is temporary and ideally we would like to switch all components to - // use ComponentConfig + ConfigMaps. - ExtraArgs map[string]string `json:"extraArgs,omitempty"` - - // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. - ExtraVolumes []HostPathMount `json:"extraVolumes,omitempty"` -} - -// APIServer holds settings necessary for API server deployments in the cluster -type APIServer struct { - ControlPlaneComponent `json:",inline"` - - // CertSANs sets extra Subject Alternative Names for the API Server signing cert. - CertSANs []string `json:"certSANs,omitempty"` - - // TimeoutForControlPlane controls the timeout that we use for API server to appear - TimeoutForControlPlane *metav1.Duration `json:"timeoutForControlPlane,omitempty"` -} - -// DNSAddOnType defines string identifying DNS add-on types -type DNSAddOnType string - -const ( - // CoreDNS add-on type - CoreDNS DNSAddOnType = "CoreDNS" - - // KubeDNS add-on type - KubeDNS DNSAddOnType = "kube-dns" -) - -// DNS defines the DNS addon that should be used in the cluster -type DNS struct { - // Type defines the DNS add-on to be used - Type DNSAddOnType `json:"type"` - - // ImageMeta allows to customize the image used for the DNS component - ImageMeta `json:",inline"` -} - -// ImageMeta allows to customize the image used for components that are not -// originated from the Kubernetes/Kubernetes release process -type ImageMeta struct { - // ImageRepository sets the container registry to pull images from. - // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. - ImageRepository string `json:"imageRepository,omitempty"` - - // ImageTag allows to specify a tag for the image. - // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. - ImageTag string `json:"imageTag,omitempty"` - - //TODO: evaluate if we need also a ImageName based on user feedbacks -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterStatus contains the cluster status. The ClusterStatus will be stored in the kubeadm-config -// ConfigMap in the cluster, and then updated by kubeadm when additional control plane instance joins or leaves the cluster. -type ClusterStatus struct { - metav1.TypeMeta `json:",inline"` - - // APIEndpoints currently available in the cluster, one for each control plane/api server instance. - // The key of the map is the IP of the host's default interface - APIEndpoints map[string]APIEndpoint `json:"apiEndpoints"` -} - -// APIEndpoint struct contains elements of API server instance deployed on a node. -type APIEndpoint struct { - // AdvertiseAddress sets the IP address for the API server to advertise. - AdvertiseAddress string `json:"advertiseAddress"` - - // BindPort sets the secure port for the API Server to bind to. - // Defaults to 6443. - BindPort int32 `json:"bindPort"` -} - -// NodeRegistrationOptions holds fields that relate to registering a new control-plane or node to the cluster, either via "kubeadm init" or "kubeadm join" -type NodeRegistrationOptions struct { - - // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm join` operation. - // This field is also used in the CommonName field of the kubelet's client certificate to the API server. - // Defaults to the hostname of the node if not provided. - Name string `json:"name,omitempty"` - - // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use - CRISocket string `json:"criSocket,omitempty"` - - // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process - // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an - // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. - Taints []v1.Taint `json:"taints,omitempty"` - - // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file - // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap - // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` -} - -// Networking contains elements describing cluster's networking configuration -type Networking struct { - // ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12". - ServiceSubnet string `json:"serviceSubnet"` - // PodSubnet is the subnet used by pods. - PodSubnet string `json:"podSubnet"` - // DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local". - DNSDomain string `json:"dnsDomain"` -} - -// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster -type BootstrapToken struct { - // Token is used for establishing bidirectional trust between nodes and control-planes. - // Used for joining nodes in the cluster. - Token *BootstrapTokenString `json:"token"` - // Description sets a human-friendly message why this token exists and what it's used - // for, so other administrators can know its purpose. - Description string `json:"description,omitempty"` - // TTL defines the time to live for this token. Defaults to 24h. - // Expires and TTL are mutually exclusive. - TTL *metav1.Duration `json:"ttl,omitempty"` - // Expires specifies the timestamp when this token expires. Defaults to being set - // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. - Expires *metav1.Time `json:"expires,omitempty"` - // Usages describes the ways in which this token can be used. Can by default be used - // for establishing bidirectional trust, but that can be changed here. - Usages []string `json:"usages,omitempty"` - // Groups specifies the extra groups that this token will authenticate as when/if - // used for authentication - Groups []string `json:"groups,omitempty"` -} - -// Etcd contains elements describing Etcd configuration. -type Etcd struct { - - // Local provides configuration knobs for configuring the local etcd instance - // Local and External are mutually exclusive - Local *LocalEtcd `json:"local,omitempty"` - - // External describes how to connect to an external etcd cluster - // Local and External are mutually exclusive - External *ExternalEtcd `json:"external,omitempty"` -} - -// LocalEtcd describes that kubeadm should run an etcd cluster locally -type LocalEtcd struct { - // ImageMeta allows to customize the container used for etcd - ImageMeta `json:",inline"` - - // DataDir is the directory etcd will place its data. - // Defaults to "/var/lib/etcd". - DataDir string `json:"dataDir"` - - // ExtraArgs are extra arguments provided to the etcd binary - // when run inside a static pod. - ExtraArgs map[string]string `json:"extraArgs,omitempty"` - - // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. - ServerCertSANs []string `json:"serverCertSANs,omitempty"` - // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. - PeerCertSANs []string `json:"peerCertSANs,omitempty"` -} - -// ExternalEtcd describes an external etcd cluster. -// Kubeadm has no knowledge of where certificate files live and they must be supplied. -type ExternalEtcd struct { - // Endpoints of etcd members. Required for ExternalEtcd. - Endpoints []string `json:"endpoints"` - - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - // Required if using a TLS connection. - CAFile string `json:"caFile"` - - // CertFile is an SSL certification file used to secure etcd communication. - // Required if using a TLS connection. - CertFile string `json:"certFile"` - - // KeyFile is an SSL key file used to secure etcd communication. - // Required if using a TLS connection. - KeyFile string `json:"keyFile"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// DEPRECATED - This group version of JoinConfiguration is deprecated by apis/kubeadm/v1beta2/JoinConfiguration. -// JoinConfiguration contains elements describing a particular node. -type JoinConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions `json:"nodeRegistration"` - - // CACertPath is the path to the SSL certificate authority used to - // secure comunications between node and control-plane. - // Defaults to "/etc/kubernetes/pki/ca.crt". - CACertPath string `json:"caCertPath"` - - // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process - Discovery Discovery `json:"discovery"` - - // ControlPlane defines the additional control plane instance to be deployed on the joining node. - // If nil, no additional control plane instance will be deployed. - ControlPlane *JoinControlPlane `json:"controlPlane,omitempty"` -} - -// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. -type JoinControlPlane struct { - // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. - LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` -} - -// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process -type Discovery struct { - // BootstrapToken is used to set the options for bootstrap token based discovery - // BootstrapToken and File are mutually exclusive - BootstrapToken *BootstrapTokenDiscovery `json:"bootstrapToken,omitempty"` - - // File is used to specify a file or URL to a kubeconfig file from which to load cluster information - // BootstrapToken and File are mutually exclusive - File *FileDiscovery `json:"file,omitempty"` - - // TLSBootstrapToken is a token used for TLS bootstrapping. - // If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token, but can be overridden. - // If .File is set, this field **must be set** in case the KubeConfigFile does not contain any other authentication information - TLSBootstrapToken string `json:"tlsBootstrapToken"` - - // Timeout modifies the discovery timeout - Timeout *metav1.Duration `json:"timeout,omitempty"` -} - -// BootstrapTokenDiscovery is used to set the options for bootstrap token based discovery -type BootstrapTokenDiscovery struct { - // Token is a token used to validate cluster information - // fetched from the control-plane. - Token string `json:"token"` - - // APIServerEndpoint is an IP or domain name to the API server from which info will be fetched. - APIServerEndpoint string `json:"apiServerEndpoint,omitempty"` - - // CACertHashes specifies a set of public key pins to verify - // when token-based discovery is used. The root CA found during discovery - // must match one of these values. Specifying an empty set disables root CA - // pinning, which can be unsafe. Each hash is specified as ":", - // where the only currently supported type is "sha256". This is a hex-encoded - // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded - // ASN.1. These hashes can be calculated using, for example, OpenSSL. - CACertHashes []string `json:"caCertHashes,omitempty"` - - // UnsafeSkipCAVerification allows token-based discovery - // without CA verification via CACertHashes. This can weaken - // the security of kubeadm since other nodes can impersonate the control-plane. - UnsafeSkipCAVerification bool `json:"unsafeSkipCAVerification"` -} - -// FileDiscovery is used to specify a file or URL to a kubeconfig file from which to load cluster information -type FileDiscovery struct { - // KubeConfigPath is used to specify the actual file path or URL to the kubeconfig file from which to load cluster information - KubeConfigPath string `json:"kubeConfigPath"` -} - -// HostPathMount contains elements describing volumes that are mounted from the -// host. -type HostPathMount struct { - // Name of the volume inside the pod template. - Name string `json:"name"` - // HostPath is the path in the host that will be mounted inside - // the pod. - HostPath string `json:"hostPath"` - // MountPath is the path inside the pod where hostPath will be mounted. - MountPath string `json:"mountPath"` - // ReadOnly controls write access to the volume - ReadOnly bool `json:"readOnly,omitempty"` - // PathType is the type of the HostPath. - PathType v1.HostPathType `json:"pathType,omitempty"` -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go deleted file mode 100644 index bc2db7ebf3b39..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go +++ /dev/null @@ -1,843 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by conversion-gen. DO NOT EDIT. - -package v1beta1 - -import ( - unsafe "unsafe" - - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - conversion "k8s.io/apimachinery/pkg/conversion" - runtime "k8s.io/apimachinery/pkg/runtime" - kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func init() { - localSchemeBuilder.Register(RegisterConversions) -} - -// RegisterConversions adds conversion functions to the given scheme. -// Public to allow building arbitrary schemes. -func RegisterConversions(s *runtime.Scheme) error { - if err := s.AddGeneratedConversionFunc((*APIEndpoint)(nil), (*kubeadm.APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(a.(*APIEndpoint), b.(*kubeadm.APIEndpoint), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.APIEndpoint)(nil), (*APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(a.(*kubeadm.APIEndpoint), b.(*APIEndpoint), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*APIServer)(nil), (*kubeadm.APIServer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_APIServer_To_kubeadm_APIServer(a.(*APIServer), b.(*kubeadm.APIServer), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.APIServer)(nil), (*APIServer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_APIServer_To_v1beta1_APIServer(a.(*kubeadm.APIServer), b.(*APIServer), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*BootstrapToken)(nil), (*kubeadm.BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken(a.(*BootstrapToken), b.(*kubeadm.BootstrapToken), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.BootstrapToken)(nil), (*BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_BootstrapToken_To_v1beta1_BootstrapToken(a.(*kubeadm.BootstrapToken), b.(*BootstrapToken), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*BootstrapTokenDiscovery)(nil), (*kubeadm.BootstrapTokenDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(a.(*BootstrapTokenDiscovery), b.(*kubeadm.BootstrapTokenDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.BootstrapTokenDiscovery)(nil), (*BootstrapTokenDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_BootstrapTokenDiscovery_To_v1beta1_BootstrapTokenDiscovery(a.(*kubeadm.BootstrapTokenDiscovery), b.(*BootstrapTokenDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*BootstrapTokenString)(nil), (*kubeadm.BootstrapTokenString)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_BootstrapTokenString_To_kubeadm_BootstrapTokenString(a.(*BootstrapTokenString), b.(*kubeadm.BootstrapTokenString), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.BootstrapTokenString)(nil), (*BootstrapTokenString)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_BootstrapTokenString_To_v1beta1_BootstrapTokenString(a.(*kubeadm.BootstrapTokenString), b.(*BootstrapTokenString), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ClusterConfiguration)(nil), (*kubeadm.ClusterConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(a.(*ClusterConfiguration), b.(*kubeadm.ClusterConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ClusterConfiguration)(nil), (*ClusterConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(a.(*kubeadm.ClusterConfiguration), b.(*ClusterConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ClusterStatus)(nil), (*kubeadm.ClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ClusterStatus_To_kubeadm_ClusterStatus(a.(*ClusterStatus), b.(*kubeadm.ClusterStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ClusterStatus)(nil), (*ClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ClusterStatus_To_v1beta1_ClusterStatus(a.(*kubeadm.ClusterStatus), b.(*ClusterStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ControlPlaneComponent)(nil), (*kubeadm.ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(a.(*ControlPlaneComponent), b.(*kubeadm.ControlPlaneComponent), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ControlPlaneComponent)(nil), (*ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(a.(*kubeadm.ControlPlaneComponent), b.(*ControlPlaneComponent), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*DNS)(nil), (*kubeadm.DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_DNS_To_kubeadm_DNS(a.(*DNS), b.(*kubeadm.DNS), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.DNS)(nil), (*DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_DNS_To_v1beta1_DNS(a.(*kubeadm.DNS), b.(*DNS), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Discovery)(nil), (*kubeadm.Discovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_Discovery_To_kubeadm_Discovery(a.(*Discovery), b.(*kubeadm.Discovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.Discovery)(nil), (*Discovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_Discovery_To_v1beta1_Discovery(a.(*kubeadm.Discovery), b.(*Discovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Etcd)(nil), (*kubeadm.Etcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_Etcd_To_kubeadm_Etcd(a.(*Etcd), b.(*kubeadm.Etcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.Etcd)(nil), (*Etcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_Etcd_To_v1beta1_Etcd(a.(*kubeadm.Etcd), b.(*Etcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ExternalEtcd)(nil), (*kubeadm.ExternalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ExternalEtcd_To_kubeadm_ExternalEtcd(a.(*ExternalEtcd), b.(*kubeadm.ExternalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ExternalEtcd)(nil), (*ExternalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ExternalEtcd_To_v1beta1_ExternalEtcd(a.(*kubeadm.ExternalEtcd), b.(*ExternalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*FileDiscovery)(nil), (*kubeadm.FileDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_FileDiscovery_To_kubeadm_FileDiscovery(a.(*FileDiscovery), b.(*kubeadm.FileDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.FileDiscovery)(nil), (*FileDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_FileDiscovery_To_v1beta1_FileDiscovery(a.(*kubeadm.FileDiscovery), b.(*FileDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*HostPathMount)(nil), (*kubeadm.HostPathMount)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_HostPathMount_To_kubeadm_HostPathMount(a.(*HostPathMount), b.(*kubeadm.HostPathMount), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.HostPathMount)(nil), (*HostPathMount)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(a.(*kubeadm.HostPathMount), b.(*HostPathMount), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ImageMeta)(nil), (*kubeadm.ImageMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(a.(*ImageMeta), b.(*kubeadm.ImageMeta), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ImageMeta)(nil), (*ImageMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(a.(*kubeadm.ImageMeta), b.(*ImageMeta), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*InitConfiguration)(nil), (*kubeadm.InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(a.(*InitConfiguration), b.(*kubeadm.InitConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*JoinConfiguration)(nil), (*kubeadm.JoinConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(a.(*JoinConfiguration), b.(*kubeadm.JoinConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.JoinConfiguration)(nil), (*JoinConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(a.(*kubeadm.JoinConfiguration), b.(*JoinConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*JoinControlPlane)(nil), (*kubeadm.JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(a.(*JoinControlPlane), b.(*kubeadm.JoinControlPlane), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*LocalEtcd)(nil), (*kubeadm.LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(a.(*LocalEtcd), b.(*kubeadm.LocalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.LocalEtcd)(nil), (*LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_LocalEtcd_To_v1beta1_LocalEtcd(a.(*kubeadm.LocalEtcd), b.(*LocalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Networking)(nil), (*kubeadm.Networking)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_Networking_To_kubeadm_Networking(a.(*Networking), b.(*kubeadm.Networking), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.Networking)(nil), (*Networking)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_Networking_To_v1beta1_Networking(a.(*kubeadm.Networking), b.(*Networking), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*NodeRegistrationOptions)(nil), (*kubeadm.NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(a.(*NodeRegistrationOptions), b.(*kubeadm.NodeRegistrationOptions), scope) - }); err != nil { - return err - } - if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddConversionFunc((*kubeadm.JoinControlPlane)(nil), (*JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(a.(*kubeadm.JoinControlPlane), b.(*JoinControlPlane), scope) - }); err != nil { - return err - } - if err := s.AddConversionFunc((*kubeadm.NodeRegistrationOptions)(nil), (*NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(a.(*kubeadm.NodeRegistrationOptions), b.(*NodeRegistrationOptions), scope) - }); err != nil { - return err - } - return nil -} - -func autoConvert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(in *APIEndpoint, out *kubeadm.APIEndpoint, s conversion.Scope) error { - out.AdvertiseAddress = in.AdvertiseAddress - out.BindPort = in.BindPort - return nil -} - -// Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint is an autogenerated conversion function. -func Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(in *APIEndpoint, out *kubeadm.APIEndpoint, s conversion.Scope) error { - return autoConvert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(in, out, s) -} - -func autoConvert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(in *kubeadm.APIEndpoint, out *APIEndpoint, s conversion.Scope) error { - out.AdvertiseAddress = in.AdvertiseAddress - out.BindPort = in.BindPort - return nil -} - -// Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint is an autogenerated conversion function. -func Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(in *kubeadm.APIEndpoint, out *APIEndpoint, s conversion.Scope) error { - return autoConvert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(in, out, s) -} - -func autoConvert_v1beta1_APIServer_To_kubeadm_APIServer(in *APIServer, out *kubeadm.APIServer, s conversion.Scope) error { - if err := Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.ControlPlaneComponent, &out.ControlPlaneComponent, s); err != nil { - return err - } - out.CertSANs = *(*[]string)(unsafe.Pointer(&in.CertSANs)) - out.TimeoutForControlPlane = (*v1.Duration)(unsafe.Pointer(in.TimeoutForControlPlane)) - return nil -} - -// Convert_v1beta1_APIServer_To_kubeadm_APIServer is an autogenerated conversion function. -func Convert_v1beta1_APIServer_To_kubeadm_APIServer(in *APIServer, out *kubeadm.APIServer, s conversion.Scope) error { - return autoConvert_v1beta1_APIServer_To_kubeadm_APIServer(in, out, s) -} - -func autoConvert_kubeadm_APIServer_To_v1beta1_APIServer(in *kubeadm.APIServer, out *APIServer, s conversion.Scope) error { - if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(&in.ControlPlaneComponent, &out.ControlPlaneComponent, s); err != nil { - return err - } - out.CertSANs = *(*[]string)(unsafe.Pointer(&in.CertSANs)) - out.TimeoutForControlPlane = (*v1.Duration)(unsafe.Pointer(in.TimeoutForControlPlane)) - return nil -} - -// Convert_kubeadm_APIServer_To_v1beta1_APIServer is an autogenerated conversion function. -func Convert_kubeadm_APIServer_To_v1beta1_APIServer(in *kubeadm.APIServer, out *APIServer, s conversion.Scope) error { - return autoConvert_kubeadm_APIServer_To_v1beta1_APIServer(in, out, s) -} - -func autoConvert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken(in *BootstrapToken, out *kubeadm.BootstrapToken, s conversion.Scope) error { - out.Token = (*kubeadm.BootstrapTokenString)(unsafe.Pointer(in.Token)) - out.Description = in.Description - out.TTL = (*v1.Duration)(unsafe.Pointer(in.TTL)) - out.Expires = (*v1.Time)(unsafe.Pointer(in.Expires)) - out.Usages = *(*[]string)(unsafe.Pointer(&in.Usages)) - out.Groups = *(*[]string)(unsafe.Pointer(&in.Groups)) - return nil -} - -// Convert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken is an autogenerated conversion function. -func Convert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken(in *BootstrapToken, out *kubeadm.BootstrapToken, s conversion.Scope) error { - return autoConvert_v1beta1_BootstrapToken_To_kubeadm_BootstrapToken(in, out, s) -} - -func autoConvert_kubeadm_BootstrapToken_To_v1beta1_BootstrapToken(in *kubeadm.BootstrapToken, out *BootstrapToken, s conversion.Scope) error { - out.Token = (*BootstrapTokenString)(unsafe.Pointer(in.Token)) - out.Description = in.Description - out.TTL = (*v1.Duration)(unsafe.Pointer(in.TTL)) - out.Expires = (*v1.Time)(unsafe.Pointer(in.Expires)) - out.Usages = *(*[]string)(unsafe.Pointer(&in.Usages)) - out.Groups = *(*[]string)(unsafe.Pointer(&in.Groups)) - return nil -} - -// Convert_kubeadm_BootstrapToken_To_v1beta1_BootstrapToken is an autogenerated conversion function. -func Convert_kubeadm_BootstrapToken_To_v1beta1_BootstrapToken(in *kubeadm.BootstrapToken, out *BootstrapToken, s conversion.Scope) error { - return autoConvert_kubeadm_BootstrapToken_To_v1beta1_BootstrapToken(in, out, s) -} - -func autoConvert_v1beta1_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in *BootstrapTokenDiscovery, out *kubeadm.BootstrapTokenDiscovery, s conversion.Scope) error { - out.Token = in.Token - out.APIServerEndpoint = in.APIServerEndpoint - out.CACertHashes = *(*[]string)(unsafe.Pointer(&in.CACertHashes)) - out.UnsafeSkipCAVerification = in.UnsafeSkipCAVerification - return nil -} - -// Convert_v1beta1_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery is an autogenerated conversion function. -func Convert_v1beta1_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in *BootstrapTokenDiscovery, out *kubeadm.BootstrapTokenDiscovery, s conversion.Scope) error { - return autoConvert_v1beta1_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in, out, s) -} - -func autoConvert_kubeadm_BootstrapTokenDiscovery_To_v1beta1_BootstrapTokenDiscovery(in *kubeadm.BootstrapTokenDiscovery, out *BootstrapTokenDiscovery, s conversion.Scope) error { - out.Token = in.Token - out.APIServerEndpoint = in.APIServerEndpoint - out.CACertHashes = *(*[]string)(unsafe.Pointer(&in.CACertHashes)) - out.UnsafeSkipCAVerification = in.UnsafeSkipCAVerification - return nil -} - -// Convert_kubeadm_BootstrapTokenDiscovery_To_v1beta1_BootstrapTokenDiscovery is an autogenerated conversion function. -func Convert_kubeadm_BootstrapTokenDiscovery_To_v1beta1_BootstrapTokenDiscovery(in *kubeadm.BootstrapTokenDiscovery, out *BootstrapTokenDiscovery, s conversion.Scope) error { - return autoConvert_kubeadm_BootstrapTokenDiscovery_To_v1beta1_BootstrapTokenDiscovery(in, out, s) -} - -func autoConvert_v1beta1_BootstrapTokenString_To_kubeadm_BootstrapTokenString(in *BootstrapTokenString, out *kubeadm.BootstrapTokenString, s conversion.Scope) error { - out.ID = in.ID - out.Secret = in.Secret - return nil -} - -// Convert_v1beta1_BootstrapTokenString_To_kubeadm_BootstrapTokenString is an autogenerated conversion function. -func Convert_v1beta1_BootstrapTokenString_To_kubeadm_BootstrapTokenString(in *BootstrapTokenString, out *kubeadm.BootstrapTokenString, s conversion.Scope) error { - return autoConvert_v1beta1_BootstrapTokenString_To_kubeadm_BootstrapTokenString(in, out, s) -} - -func autoConvert_kubeadm_BootstrapTokenString_To_v1beta1_BootstrapTokenString(in *kubeadm.BootstrapTokenString, out *BootstrapTokenString, s conversion.Scope) error { - out.ID = in.ID - out.Secret = in.Secret - return nil -} - -// Convert_kubeadm_BootstrapTokenString_To_v1beta1_BootstrapTokenString is an autogenerated conversion function. -func Convert_kubeadm_BootstrapTokenString_To_v1beta1_BootstrapTokenString(in *kubeadm.BootstrapTokenString, out *BootstrapTokenString, s conversion.Scope) error { - return autoConvert_kubeadm_BootstrapTokenString_To_v1beta1_BootstrapTokenString(in, out, s) -} - -func autoConvert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration, s conversion.Scope) error { - if err := Convert_v1beta1_Etcd_To_kubeadm_Etcd(&in.Etcd, &out.Etcd, s); err != nil { - return err - } - if err := Convert_v1beta1_Networking_To_kubeadm_Networking(&in.Networking, &out.Networking, s); err != nil { - return err - } - out.KubernetesVersion = in.KubernetesVersion - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - if err := Convert_v1beta1_APIServer_To_kubeadm_APIServer(&in.APIServer, &out.APIServer, s); err != nil { - return err - } - if err := Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.ControllerManager, &out.ControllerManager, s); err != nil { - return err - } - if err := Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.Scheduler, &out.Scheduler, s); err != nil { - return err - } - if err := Convert_v1beta1_DNS_To_kubeadm_DNS(&in.DNS, &out.DNS, s); err != nil { - return err - } - out.CertificatesDir = in.CertificatesDir - out.ImageRepository = in.ImageRepository - out.UseHyperKubeImage = in.UseHyperKubeImage - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) - out.ClusterName = in.ClusterName - return nil -} - -// Convert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration is an autogenerated conversion function. -func Convert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration, s conversion.Scope) error { - return autoConvert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in, out, s) -} - -func autoConvert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error { - // INFO: in.ComponentConfigs opted out of conversion generation - if err := Convert_kubeadm_Etcd_To_v1beta1_Etcd(&in.Etcd, &out.Etcd, s); err != nil { - return err - } - if err := Convert_kubeadm_Networking_To_v1beta1_Networking(&in.Networking, &out.Networking, s); err != nil { - return err - } - out.KubernetesVersion = in.KubernetesVersion - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - if err := Convert_kubeadm_APIServer_To_v1beta1_APIServer(&in.APIServer, &out.APIServer, s); err != nil { - return err - } - if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(&in.ControllerManager, &out.ControllerManager, s); err != nil { - return err - } - if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(&in.Scheduler, &out.Scheduler, s); err != nil { - return err - } - if err := Convert_kubeadm_DNS_To_v1beta1_DNS(&in.DNS, &out.DNS, s); err != nil { - return err - } - out.CertificatesDir = in.CertificatesDir - out.ImageRepository = in.ImageRepository - // INFO: in.CIImageRepository opted out of conversion generation - out.UseHyperKubeImage = in.UseHyperKubeImage - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) - out.ClusterName = in.ClusterName - return nil -} - -// Convert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration is an autogenerated conversion function. -func Convert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in, out, s) -} - -func autoConvert_v1beta1_ClusterStatus_To_kubeadm_ClusterStatus(in *ClusterStatus, out *kubeadm.ClusterStatus, s conversion.Scope) error { - out.APIEndpoints = *(*map[string]kubeadm.APIEndpoint)(unsafe.Pointer(&in.APIEndpoints)) - return nil -} - -// Convert_v1beta1_ClusterStatus_To_kubeadm_ClusterStatus is an autogenerated conversion function. -func Convert_v1beta1_ClusterStatus_To_kubeadm_ClusterStatus(in *ClusterStatus, out *kubeadm.ClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta1_ClusterStatus_To_kubeadm_ClusterStatus(in, out, s) -} - -func autoConvert_kubeadm_ClusterStatus_To_v1beta1_ClusterStatus(in *kubeadm.ClusterStatus, out *ClusterStatus, s conversion.Scope) error { - out.APIEndpoints = *(*map[string]APIEndpoint)(unsafe.Pointer(&in.APIEndpoints)) - return nil -} - -// Convert_kubeadm_ClusterStatus_To_v1beta1_ClusterStatus is an autogenerated conversion function. -func Convert_kubeadm_ClusterStatus_To_v1beta1_ClusterStatus(in *kubeadm.ClusterStatus, out *ClusterStatus, s conversion.Scope) error { - return autoConvert_kubeadm_ClusterStatus_To_v1beta1_ClusterStatus(in, out, s) -} - -func autoConvert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) - return nil -} - -// Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent is an autogenerated conversion function. -func Convert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - return autoConvert_v1beta1_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in, out, s) -} - -func autoConvert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) - return nil -} - -// Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent is an autogenerated conversion function. -func Convert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - return autoConvert_kubeadm_ControlPlaneComponent_To_v1beta1_ControlPlaneComponent(in, out, s) -} - -func autoConvert_v1beta1_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { - out.Type = kubeadm.DNSAddOnType(in.Type) - if err := Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta1_DNS_To_kubeadm_DNS is an autogenerated conversion function. -func Convert_v1beta1_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { - return autoConvert_v1beta1_DNS_To_kubeadm_DNS(in, out, s) -} - -func autoConvert_kubeadm_DNS_To_v1beta1_DNS(in *kubeadm.DNS, out *DNS, s conversion.Scope) error { - out.Type = DNSAddOnType(in.Type) - if err := Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - return nil -} - -// Convert_kubeadm_DNS_To_v1beta1_DNS is an autogenerated conversion function. -func Convert_kubeadm_DNS_To_v1beta1_DNS(in *kubeadm.DNS, out *DNS, s conversion.Scope) error { - return autoConvert_kubeadm_DNS_To_v1beta1_DNS(in, out, s) -} - -func autoConvert_v1beta1_Discovery_To_kubeadm_Discovery(in *Discovery, out *kubeadm.Discovery, s conversion.Scope) error { - out.BootstrapToken = (*kubeadm.BootstrapTokenDiscovery)(unsafe.Pointer(in.BootstrapToken)) - out.File = (*kubeadm.FileDiscovery)(unsafe.Pointer(in.File)) - out.TLSBootstrapToken = in.TLSBootstrapToken - out.Timeout = (*v1.Duration)(unsafe.Pointer(in.Timeout)) - return nil -} - -// Convert_v1beta1_Discovery_To_kubeadm_Discovery is an autogenerated conversion function. -func Convert_v1beta1_Discovery_To_kubeadm_Discovery(in *Discovery, out *kubeadm.Discovery, s conversion.Scope) error { - return autoConvert_v1beta1_Discovery_To_kubeadm_Discovery(in, out, s) -} - -func autoConvert_kubeadm_Discovery_To_v1beta1_Discovery(in *kubeadm.Discovery, out *Discovery, s conversion.Scope) error { - out.BootstrapToken = (*BootstrapTokenDiscovery)(unsafe.Pointer(in.BootstrapToken)) - out.File = (*FileDiscovery)(unsafe.Pointer(in.File)) - out.TLSBootstrapToken = in.TLSBootstrapToken - out.Timeout = (*v1.Duration)(unsafe.Pointer(in.Timeout)) - return nil -} - -// Convert_kubeadm_Discovery_To_v1beta1_Discovery is an autogenerated conversion function. -func Convert_kubeadm_Discovery_To_v1beta1_Discovery(in *kubeadm.Discovery, out *Discovery, s conversion.Scope) error { - return autoConvert_kubeadm_Discovery_To_v1beta1_Discovery(in, out, s) -} - -func autoConvert_v1beta1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { - out.Local = (*kubeadm.LocalEtcd)(unsafe.Pointer(in.Local)) - out.External = (*kubeadm.ExternalEtcd)(unsafe.Pointer(in.External)) - return nil -} - -// Convert_v1beta1_Etcd_To_kubeadm_Etcd is an autogenerated conversion function. -func Convert_v1beta1_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { - return autoConvert_v1beta1_Etcd_To_kubeadm_Etcd(in, out, s) -} - -func autoConvert_kubeadm_Etcd_To_v1beta1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { - out.Local = (*LocalEtcd)(unsafe.Pointer(in.Local)) - out.External = (*ExternalEtcd)(unsafe.Pointer(in.External)) - return nil -} - -// Convert_kubeadm_Etcd_To_v1beta1_Etcd is an autogenerated conversion function. -func Convert_kubeadm_Etcd_To_v1beta1_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { - return autoConvert_kubeadm_Etcd_To_v1beta1_Etcd(in, out, s) -} - -func autoConvert_v1beta1_ExternalEtcd_To_kubeadm_ExternalEtcd(in *ExternalEtcd, out *kubeadm.ExternalEtcd, s conversion.Scope) error { - out.Endpoints = *(*[]string)(unsafe.Pointer(&in.Endpoints)) - out.CAFile = in.CAFile - out.CertFile = in.CertFile - out.KeyFile = in.KeyFile - return nil -} - -// Convert_v1beta1_ExternalEtcd_To_kubeadm_ExternalEtcd is an autogenerated conversion function. -func Convert_v1beta1_ExternalEtcd_To_kubeadm_ExternalEtcd(in *ExternalEtcd, out *kubeadm.ExternalEtcd, s conversion.Scope) error { - return autoConvert_v1beta1_ExternalEtcd_To_kubeadm_ExternalEtcd(in, out, s) -} - -func autoConvert_kubeadm_ExternalEtcd_To_v1beta1_ExternalEtcd(in *kubeadm.ExternalEtcd, out *ExternalEtcd, s conversion.Scope) error { - out.Endpoints = *(*[]string)(unsafe.Pointer(&in.Endpoints)) - out.CAFile = in.CAFile - out.CertFile = in.CertFile - out.KeyFile = in.KeyFile - return nil -} - -// Convert_kubeadm_ExternalEtcd_To_v1beta1_ExternalEtcd is an autogenerated conversion function. -func Convert_kubeadm_ExternalEtcd_To_v1beta1_ExternalEtcd(in *kubeadm.ExternalEtcd, out *ExternalEtcd, s conversion.Scope) error { - return autoConvert_kubeadm_ExternalEtcd_To_v1beta1_ExternalEtcd(in, out, s) -} - -func autoConvert_v1beta1_FileDiscovery_To_kubeadm_FileDiscovery(in *FileDiscovery, out *kubeadm.FileDiscovery, s conversion.Scope) error { - out.KubeConfigPath = in.KubeConfigPath - return nil -} - -// Convert_v1beta1_FileDiscovery_To_kubeadm_FileDiscovery is an autogenerated conversion function. -func Convert_v1beta1_FileDiscovery_To_kubeadm_FileDiscovery(in *FileDiscovery, out *kubeadm.FileDiscovery, s conversion.Scope) error { - return autoConvert_v1beta1_FileDiscovery_To_kubeadm_FileDiscovery(in, out, s) -} - -func autoConvert_kubeadm_FileDiscovery_To_v1beta1_FileDiscovery(in *kubeadm.FileDiscovery, out *FileDiscovery, s conversion.Scope) error { - out.KubeConfigPath = in.KubeConfigPath - return nil -} - -// Convert_kubeadm_FileDiscovery_To_v1beta1_FileDiscovery is an autogenerated conversion function. -func Convert_kubeadm_FileDiscovery_To_v1beta1_FileDiscovery(in *kubeadm.FileDiscovery, out *FileDiscovery, s conversion.Scope) error { - return autoConvert_kubeadm_FileDiscovery_To_v1beta1_FileDiscovery(in, out, s) -} - -func autoConvert_v1beta1_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error { - out.Name = in.Name - out.HostPath = in.HostPath - out.MountPath = in.MountPath - out.ReadOnly = in.ReadOnly - out.PathType = corev1.HostPathType(in.PathType) - return nil -} - -// Convert_v1beta1_HostPathMount_To_kubeadm_HostPathMount is an autogenerated conversion function. -func Convert_v1beta1_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error { - return autoConvert_v1beta1_HostPathMount_To_kubeadm_HostPathMount(in, out, s) -} - -func autoConvert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { - out.Name = in.Name - out.HostPath = in.HostPath - out.MountPath = in.MountPath - out.ReadOnly = in.ReadOnly - out.PathType = corev1.HostPathType(in.PathType) - return nil -} - -// Convert_kubeadm_HostPathMount_To_v1beta1_HostPathMount is an autogenerated conversion function. -func Convert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { - return autoConvert_kubeadm_HostPathMount_To_v1beta1_HostPathMount(in, out, s) -} - -func autoConvert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(in *ImageMeta, out *kubeadm.ImageMeta, s conversion.Scope) error { - out.ImageRepository = in.ImageRepository - out.ImageTag = in.ImageTag - return nil -} - -// Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta is an autogenerated conversion function. -func Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(in *ImageMeta, out *kubeadm.ImageMeta, s conversion.Scope) error { - return autoConvert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(in, out, s) -} - -func autoConvert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(in *kubeadm.ImageMeta, out *ImageMeta, s conversion.Scope) error { - out.ImageRepository = in.ImageRepository - out.ImageTag = in.ImageTag - return nil -} - -// Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta is an autogenerated conversion function. -func Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(in *kubeadm.ImageMeta, out *ImageMeta, s conversion.Scope) error { - return autoConvert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(in, out, s) -} - -func autoConvert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { - if err := Convert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(&in.ClusterConfiguration, &out.ClusterConfiguration, s); err != nil { - return err - } - out.BootstrapTokens = *(*[]kubeadm.BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) - if err := Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - if err := Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration is an autogenerated conversion function. -func Convert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { - return autoConvert_v1beta1_InitConfiguration_To_kubeadm_InitConfiguration(in, out, s) -} - -func autoConvert_kubeadm_InitConfiguration_To_v1beta1_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { - if err := Convert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(&in.ClusterConfiguration, &out.ClusterConfiguration, s); err != nil { - return err - } - out.BootstrapTokens = *(*[]BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) - if err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - // WARNING: in.CertificateKey requires manual conversion: does not exist in peer-type - return nil -} - -func autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { - if err := Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - out.CACertPath = in.CACertPath - if err := Convert_v1beta1_Discovery_To_kubeadm_Discovery(&in.Discovery, &out.Discovery, s); err != nil { - return err - } - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(kubeadm.JoinControlPlane) - if err := Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(*in, *out, s); err != nil { - return err - } - } else { - out.ControlPlane = nil - } - return nil -} - -// Convert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration is an autogenerated conversion function. -func Convert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { - return autoConvert_v1beta1_JoinConfiguration_To_kubeadm_JoinConfiguration(in, out, s) -} - -func autoConvert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in *kubeadm.JoinConfiguration, out *JoinConfiguration, s conversion.Scope) error { - if err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - out.CACertPath = in.CACertPath - if err := Convert_kubeadm_Discovery_To_v1beta1_Discovery(&in.Discovery, &out.Discovery, s); err != nil { - return err - } - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(JoinControlPlane) - if err := Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(*in, *out, s); err != nil { - return err - } - } else { - out.ControlPlane = nil - } - return nil -} - -// Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration is an autogenerated conversion function. -func Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in *kubeadm.JoinConfiguration, out *JoinConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(in, out, s) -} - -func autoConvert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinControlPlane, out *kubeadm.JoinControlPlane, s conversion.Scope) error { - if err := Convert_v1beta1_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane is an autogenerated conversion function. -func Convert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinControlPlane, out *kubeadm.JoinControlPlane, s conversion.Scope) error { - return autoConvert_v1beta1_JoinControlPlane_To_kubeadm_JoinControlPlane(in, out, s) -} - -func autoConvert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error { - if err := Convert_kubeadm_APIEndpoint_To_v1beta1_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - // WARNING: in.CertificateKey requires manual conversion: does not exist in peer-type - return nil -} - -func autoConvert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - if err := Convert_v1beta1_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) - out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) - return nil -} - -// Convert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd is an autogenerated conversion function. -func Convert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - return autoConvert_v1beta1_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s) -} - -func autoConvert_kubeadm_LocalEtcd_To_v1beta1_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - if err := Convert_kubeadm_ImageMeta_To_v1beta1_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) - out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) - return nil -} - -// Convert_kubeadm_LocalEtcd_To_v1beta1_LocalEtcd is an autogenerated conversion function. -func Convert_kubeadm_LocalEtcd_To_v1beta1_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - return autoConvert_kubeadm_LocalEtcd_To_v1beta1_LocalEtcd(in, out, s) -} - -func autoConvert_v1beta1_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { - out.ServiceSubnet = in.ServiceSubnet - out.PodSubnet = in.PodSubnet - out.DNSDomain = in.DNSDomain - return nil -} - -// Convert_v1beta1_Networking_To_kubeadm_Networking is an autogenerated conversion function. -func Convert_v1beta1_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { - return autoConvert_v1beta1_Networking_To_kubeadm_Networking(in, out, s) -} - -func autoConvert_kubeadm_Networking_To_v1beta1_Networking(in *kubeadm.Networking, out *Networking, s conversion.Scope) error { - out.ServiceSubnet = in.ServiceSubnet - out.PodSubnet = in.PodSubnet - out.DNSDomain = in.DNSDomain - return nil -} - -// Convert_kubeadm_Networking_To_v1beta1_Networking is an autogenerated conversion function. -func Convert_kubeadm_Networking_To_v1beta1_Networking(in *kubeadm.Networking, out *Networking, s conversion.Scope) error { - return autoConvert_kubeadm_Networking_To_v1beta1_Networking(in, out, s) -} - -func autoConvert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in *NodeRegistrationOptions, out *kubeadm.NodeRegistrationOptions, s conversion.Scope) error { - out.Name = in.Name - out.CRISocket = in.CRISocket - out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) - return nil -} - -// Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions is an autogenerated conversion function. -func Convert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in *NodeRegistrationOptions, out *kubeadm.NodeRegistrationOptions, s conversion.Scope) error { - return autoConvert_v1beta1_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in, out, s) -} - -func autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { - out.Name = in.Name - out.CRISocket = in.CRISocket - out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) - // WARNING: in.IgnorePreflightErrors requires manual conversion: does not exist in peer-type - return nil -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go deleted file mode 100644 index d6cdfda335a49..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.deepcopy.go +++ /dev/null @@ -1,552 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package v1beta1 - -import ( - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIEndpoint) DeepCopyInto(out *APIEndpoint) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIEndpoint. -func (in *APIEndpoint) DeepCopy() *APIEndpoint { - if in == nil { - return nil - } - out := new(APIEndpoint) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIServer) DeepCopyInto(out *APIServer) { - *out = *in - in.ControlPlaneComponent.DeepCopyInto(&out.ControlPlaneComponent) - if in.CertSANs != nil { - in, out := &in.CertSANs, &out.CertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TimeoutForControlPlane != nil { - in, out := &in.TimeoutForControlPlane, &out.TimeoutForControlPlane - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServer. -func (in *APIServer) DeepCopy() *APIServer { - if in == nil { - return nil - } - out := new(APIServer) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { - *out = *in - if in.Token != nil { - in, out := &in.Token, &out.Token - *out = new(BootstrapTokenString) - **out = **in - } - if in.TTL != nil { - in, out := &in.TTL, &out.TTL - *out = new(v1.Duration) - **out = **in - } - if in.Expires != nil { - in, out := &in.Expires, &out.Expires - *out = (*in).DeepCopy() - } - if in.Usages != nil { - in, out := &in.Usages, &out.Usages - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Groups != nil { - in, out := &in.Groups, &out.Groups - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. -func (in *BootstrapToken) DeepCopy() *BootstrapToken { - if in == nil { - return nil - } - out := new(BootstrapToken) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapTokenDiscovery) DeepCopyInto(out *BootstrapTokenDiscovery) { - *out = *in - if in.CACertHashes != nil { - in, out := &in.CACertHashes, &out.CACertHashes - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapTokenDiscovery. -func (in *BootstrapTokenDiscovery) DeepCopy() *BootstrapTokenDiscovery { - if in == nil { - return nil - } - out := new(BootstrapTokenDiscovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapTokenString) DeepCopyInto(out *BootstrapTokenString) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapTokenString. -func (in *BootstrapTokenString) DeepCopy() *BootstrapTokenString { - if in == nil { - return nil - } - out := new(BootstrapTokenString) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.Etcd.DeepCopyInto(&out.Etcd) - out.Networking = in.Networking - in.APIServer.DeepCopyInto(&out.APIServer) - in.ControllerManager.DeepCopyInto(&out.ControllerManager) - in.Scheduler.DeepCopyInto(&out.Scheduler) - out.DNS = in.DNS - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterConfiguration. -func (in *ClusterConfiguration) DeepCopy() *ClusterConfiguration { - if in == nil { - return nil - } - out := new(ClusterConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.APIEndpoints != nil { - in, out := &in.APIEndpoints, &out.APIEndpoints - *out = make(map[string]APIEndpoint, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. -func (in *ClusterStatus) DeepCopy() *ClusterStatus { - if in == nil { - return nil - } - out := new(ClusterStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterStatus) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { - *out = *in - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ExtraVolumes != nil { - in, out := &in.ExtraVolumes, &out.ExtraVolumes - *out = make([]HostPathMount, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponent. -func (in *ControlPlaneComponent) DeepCopy() *ControlPlaneComponent { - if in == nil { - return nil - } - out := new(ControlPlaneComponent) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DNS) DeepCopyInto(out *DNS) { - *out = *in - out.ImageMeta = in.ImageMeta - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNS. -func (in *DNS) DeepCopy() *DNS { - if in == nil { - return nil - } - out := new(DNS) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Discovery) DeepCopyInto(out *Discovery) { - *out = *in - if in.BootstrapToken != nil { - in, out := &in.BootstrapToken, &out.BootstrapToken - *out = new(BootstrapTokenDiscovery) - (*in).DeepCopyInto(*out) - } - if in.File != nil { - in, out := &in.File, &out.File - *out = new(FileDiscovery) - **out = **in - } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Discovery. -func (in *Discovery) DeepCopy() *Discovery { - if in == nil { - return nil - } - out := new(Discovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Etcd) DeepCopyInto(out *Etcd) { - *out = *in - if in.Local != nil { - in, out := &in.Local, &out.Local - *out = new(LocalEtcd) - (*in).DeepCopyInto(*out) - } - if in.External != nil { - in, out := &in.External, &out.External - *out = new(ExternalEtcd) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Etcd. -func (in *Etcd) DeepCopy() *Etcd { - if in == nil { - return nil - } - out := new(Etcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExternalEtcd) DeepCopyInto(out *ExternalEtcd) { - *out = *in - if in.Endpoints != nil { - in, out := &in.Endpoints, &out.Endpoints - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalEtcd. -func (in *ExternalEtcd) DeepCopy() *ExternalEtcd { - if in == nil { - return nil - } - out := new(ExternalEtcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileDiscovery) DeepCopyInto(out *FileDiscovery) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileDiscovery. -func (in *FileDiscovery) DeepCopy() *FileDiscovery { - if in == nil { - return nil - } - out := new(FileDiscovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HostPathMount) DeepCopyInto(out *HostPathMount) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMount. -func (in *HostPathMount) DeepCopy() *HostPathMount { - if in == nil { - return nil - } - out := new(HostPathMount) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageMeta) DeepCopyInto(out *ImageMeta) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageMeta. -func (in *ImageMeta) DeepCopy() *ImageMeta { - if in == nil { - return nil - } - out := new(ImageMeta) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ClusterConfiguration.DeepCopyInto(&out.ClusterConfiguration) - if in.BootstrapTokens != nil { - in, out := &in.BootstrapTokens, &out.BootstrapTokens - *out = make([]BootstrapToken, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - out.LocalAPIEndpoint = in.LocalAPIEndpoint - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitConfiguration. -func (in *InitConfiguration) DeepCopy() *InitConfiguration { - if in == nil { - return nil - } - out := new(InitConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *InitConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JoinConfiguration) DeepCopyInto(out *JoinConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - in.Discovery.DeepCopyInto(&out.Discovery) - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(JoinControlPlane) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinConfiguration. -func (in *JoinConfiguration) DeepCopy() *JoinConfiguration { - if in == nil { - return nil - } - out := new(JoinConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *JoinConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JoinControlPlane) DeepCopyInto(out *JoinControlPlane) { - *out = *in - out.LocalAPIEndpoint = in.LocalAPIEndpoint - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinControlPlane. -func (in *JoinControlPlane) DeepCopy() *JoinControlPlane { - if in == nil { - return nil - } - out := new(JoinControlPlane) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { - *out = *in - out.ImageMeta = in.ImageMeta - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ServerCertSANs != nil { - in, out := &in.ServerCertSANs, &out.ServerCertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.PeerCertSANs != nil { - in, out := &in.PeerCertSANs, &out.PeerCertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalEtcd. -func (in *LocalEtcd) DeepCopy() *LocalEtcd { - if in == nil { - return nil - } - out := new(LocalEtcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Networking) DeepCopyInto(out *Networking) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Networking. -func (in *Networking) DeepCopy() *Networking { - if in == nil { - return nil - } - out := new(Networking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { - *out = *in - if in.Taints != nil { - in, out := &in.Taints, &out.Taints - *out = make([]corev1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.KubeletExtraArgs != nil { - in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeRegistrationOptions. -func (in *NodeRegistrationOptions) DeepCopy() *NodeRegistrationOptions { - if in == nil { - return nil - } - out := new(NodeRegistrationOptions) - in.DeepCopyInto(out) - return out -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go deleted file mode 100644 index 95a8e67963f20..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.defaults.go +++ /dev/null @@ -1,66 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by defaulter-gen. DO NOT EDIT. - -package v1beta1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// RegisterDefaults adds defaulters functions to the given scheme. -// Public to allow building arbitrary schemes. -// All generated defaulters are covering - they call all nested defaulters. -func RegisterDefaults(scheme *runtime.Scheme) error { - scheme.AddTypeDefaultingFunc(&ClusterConfiguration{}, func(obj interface{}) { SetObjectDefaults_ClusterConfiguration(obj.(*ClusterConfiguration)) }) - scheme.AddTypeDefaultingFunc(&ClusterStatus{}, func(obj interface{}) { SetObjectDefaults_ClusterStatus(obj.(*ClusterStatus)) }) - scheme.AddTypeDefaultingFunc(&InitConfiguration{}, func(obj interface{}) { SetObjectDefaults_InitConfiguration(obj.(*InitConfiguration)) }) - scheme.AddTypeDefaultingFunc(&JoinConfiguration{}, func(obj interface{}) { SetObjectDefaults_JoinConfiguration(obj.(*JoinConfiguration)) }) - return nil -} - -func SetObjectDefaults_ClusterConfiguration(in *ClusterConfiguration) { - SetDefaults_ClusterConfiguration(in) - SetDefaults_APIServer(&in.APIServer) -} - -func SetObjectDefaults_ClusterStatus(in *ClusterStatus) { -} - -func SetObjectDefaults_InitConfiguration(in *InitConfiguration) { - SetDefaults_InitConfiguration(in) - SetObjectDefaults_ClusterConfiguration(&in.ClusterConfiguration) - for i := range in.BootstrapTokens { - a := &in.BootstrapTokens[i] - SetDefaults_BootstrapToken(a) - } - SetDefaults_APIEndpoint(&in.LocalAPIEndpoint) -} - -func SetObjectDefaults_JoinConfiguration(in *JoinConfiguration) { - SetDefaults_JoinConfiguration(in) - SetDefaults_Discovery(&in.Discovery) - if in.Discovery.File != nil { - SetDefaults_FileDiscovery(in.Discovery.File) - } - if in.ControlPlane != nil { - SetDefaults_JoinControlPlane(in.ControlPlane) - SetDefaults_APIEndpoint(&in.ControlPlane.LocalAPIEndpoint) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1beta2/BUILD deleted file mode 100644 index fa48d5c8e23b2..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "bootstraptokenstring.go", - "conversion.go", - "defaults.go", - "defaults_unix.go", - "defaults_windows.go", - "doc.go", - "register.go", - "types.go", - "zz_generated.conversion.go", - "zz_generated.deepcopy.go", - "zz_generated.defaults.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["bootstraptokenstring_test.go"], - embed = [":go_default_library"], - deps = ["//vendor/github.com/pkg/errors:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/bootstraptokenstring.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/bootstraptokenstring.go deleted file mode 100644 index 2408f5e33b8ba..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/bootstraptokenstring.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" -) - -// BootstrapTokenString is a token of the format abcdef.abcdef0123456789 that is used -// for both validation of the practically of the API server from a joining node's point -// of view and as an authentication method for the node in the bootstrap phase of -// "kubeadm join". This token is and should be short-lived -type BootstrapTokenString struct { - ID string `json:"-"` - Secret string `json:"-"` -} - -// MarshalJSON implements the json.Marshaler interface. -func (bts BootstrapTokenString) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%s"`, bts.String())), nil -} - -// UnmarshalJSON implements the json.Unmarshaller interface. -func (bts *BootstrapTokenString) UnmarshalJSON(b []byte) error { - // If the token is represented as "", just return quickly without an error - if len(b) == 0 { - return nil - } - - // Remove unnecessary " characters coming from the JSON parser - token := strings.Replace(string(b), `"`, ``, -1) - // Convert the string Token to a BootstrapTokenString object - newbts, err := NewBootstrapTokenString(token) - if err != nil { - return err - } - bts.ID = newbts.ID - bts.Secret = newbts.Secret - return nil -} - -// String returns the string representation of the BootstrapTokenString -func (bts BootstrapTokenString) String() string { - if len(bts.ID) > 0 && len(bts.Secret) > 0 { - return bootstraputil.TokenFromIDAndSecret(bts.ID, bts.Secret) - } - return "" -} - -// NewBootstrapTokenString converts the given Bootstrap Token as a string -// to the BootstrapTokenString object used for serialization/deserialization -// and internal usage. It also automatically validates that the given token -// is of the right format -func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) { - substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token) - // TODO: Add a constant for the 3 value here, and explain better why it's needed (other than because how the regexp parsin works) - if len(substrs) != 3 { - return nil, errors.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern) - } - - return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil -} - -// NewBootstrapTokenStringFromIDAndSecret is a wrapper around NewBootstrapTokenString -// that allows the caller to specify the ID and Secret separately -func NewBootstrapTokenStringFromIDAndSecret(id, secret string) (*BootstrapTokenString, error) { - return NewBootstrapTokenString(bootstraputil.TokenFromIDAndSecret(id, secret)) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/bootstraptokenstring_test.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/bootstraptokenstring_test.go deleted file mode 100644 index 9f37e2df05448..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/bootstraptokenstring_test.go +++ /dev/null @@ -1,249 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/pkg/errors" -) - -func TestMarshalJSON(t *testing.T) { - var tests = []struct { - bts BootstrapTokenString - expected string - }{ - {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, `"abcdef.abcdef0123456789"`}, - {BootstrapTokenString{ID: "foo", Secret: "bar"}, `"foo.bar"`}, - {BootstrapTokenString{ID: "h", Secret: "b"}, `"h.b"`}, - } - for _, rt := range tests { - t.Run(rt.bts.ID, func(t *testing.T) { - b, err := json.Marshal(rt.bts) - if err != nil { - t.Fatalf("json.Marshal returned an unexpected error: %v", err) - } - if string(b) != rt.expected { - t.Errorf( - "failed BootstrapTokenString.MarshalJSON:\n\texpected: %s\n\t actual: %s", - rt.expected, - string(b), - ) - } - }) - } -} - -func TestUnmarshalJSON(t *testing.T) { - var tests = []struct { - input string - bts *BootstrapTokenString - expectedError bool - }{ - {`"f.s"`, &BootstrapTokenString{}, true}, - {`"abcdef."`, &BootstrapTokenString{}, true}, - {`"abcdef:abcdef0123456789"`, &BootstrapTokenString{}, true}, - {`abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, - {`"abcdef.abcdef0123456789`, &BootstrapTokenString{}, true}, - {`"abcdef.ABCDEF0123456789"`, &BootstrapTokenString{}, true}, - {`"abcdef.abcdef0123456789"`, &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, false}, - {`"123456.aabbccddeeffgghh"`, &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}, false}, - } - for _, rt := range tests { - t.Run(rt.input, func(t *testing.T) { - newbts := &BootstrapTokenString{} - err := json.Unmarshal([]byte(rt.input), newbts) - if (err != nil) != rt.expectedError { - t.Errorf("failed BootstrapTokenString.UnmarshalJSON:\n\texpected error: %t\n\t actual error: %v", rt.expectedError, err) - } else if !reflect.DeepEqual(rt.bts, newbts) { - t.Errorf( - "failed BootstrapTokenString.UnmarshalJSON:\n\texpected: %v\n\t actual: %v", - rt.bts, - newbts, - ) - } - }) - } -} - -func TestJSONRoundtrip(t *testing.T) { - var tests = []struct { - input string - bts *BootstrapTokenString - }{ - {`"abcdef.abcdef0123456789"`, nil}, - {"", &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - } - for _, rt := range tests { - t.Run(rt.input, func(t *testing.T) { - if err := roundtrip(rt.input, rt.bts); err != nil { - t.Errorf("failed BootstrapTokenString JSON roundtrip with error: %v", err) - } - }) - } -} - -func roundtrip(input string, bts *BootstrapTokenString) error { - var b []byte - var err error - newbts := &BootstrapTokenString{} - // If string input was specified, roundtrip like this: string -> (unmarshal) -> object -> (marshal) -> string - if len(input) > 0 { - if err := json.Unmarshal([]byte(input), newbts); err != nil { - return errors.Wrap(err, "expected no unmarshal error, got error") - } - if b, err = json.Marshal(newbts); err != nil { - return errors.Wrap(err, "expected no marshal error, got error") - } - if input != string(b) { - return errors.Errorf( - "expected token: %s\n\t actual: %s", - input, - string(b), - ) - } - } else { // Otherwise, roundtrip like this: object -> (marshal) -> string -> (unmarshal) -> object - if b, err = json.Marshal(bts); err != nil { - return errors.Wrap(err, "expected no marshal error, got error") - } - if err := json.Unmarshal(b, newbts); err != nil { - return errors.Wrap(err, "expected no unmarshal error, got error") - } - if !reflect.DeepEqual(bts, newbts) { - return errors.Errorf( - "expected object: %v\n\t actual: %v", - bts, - newbts, - ) - } - } - return nil -} - -func TestTokenFromIDAndSecret(t *testing.T) { - var tests = []struct { - bts BootstrapTokenString - expected string - }{ - {BootstrapTokenString{ID: "foo", Secret: "bar"}, "foo.bar"}, - {BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, "abcdef.abcdef0123456789"}, - {BootstrapTokenString{ID: "h", Secret: "b"}, "h.b"}, - } - for _, rt := range tests { - t.Run(rt.bts.ID, func(t *testing.T) { - actual := rt.bts.String() - if actual != rt.expected { - t.Errorf( - "failed BootstrapTokenString.String():\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - }) - } -} - -func TestNewBootstrapTokenString(t *testing.T) { - var tests = []struct { - token string - expectedError bool - bts *BootstrapTokenString - }{ - {token: "", expectedError: true, bts: nil}, - {token: ".", expectedError: true, bts: nil}, - {token: "1234567890123456789012", expectedError: true, bts: nil}, // invalid parcel size - {token: "12345.1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {token: ".1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {token: "123456.", expectedError: true, bts: nil}, // invalid parcel size - {token: "123456:1234567890.123456", expectedError: true, bts: nil}, // invalid separation - {token: "abcdef:1234567890123456", expectedError: true, bts: nil}, // invalid separation - {token: "Abcdef.1234567890123456", expectedError: true, bts: nil}, // invalid token id - {token: "123456.AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret - {token: "123456.AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character - {token: "abc*ef.1234567890123456", expectedError: true, bts: nil}, // invalid character - {token: "abcdef.1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, - {token: "123456.aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, - {token: "abcdef.abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - {token: "123456.1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, - } - for _, rt := range tests { - t.Run(rt.token, func(t *testing.T) { - actual, err := NewBootstrapTokenString(rt.token) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed NewBootstrapTokenString for the token %q\n\texpected error: %t\n\t actual error: %v", - rt.token, - rt.expectedError, - err, - ) - } else if !reflect.DeepEqual(actual, rt.bts) { - t.Errorf( - "failed NewBootstrapTokenString for the token %q\n\texpected: %v\n\t actual: %v", - rt.token, - rt.bts, - actual, - ) - } - }) - } -} - -func TestNewBootstrapTokenStringFromIDAndSecret(t *testing.T) { - var tests = []struct { - id, secret string - expectedError bool - bts *BootstrapTokenString - }{ - {id: "", secret: "", expectedError: true, bts: nil}, - {id: "1234567890123456789012", secret: "", expectedError: true, bts: nil}, // invalid parcel size - {id: "12345", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {id: "", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid parcel size - {id: "123456", secret: "", expectedError: true, bts: nil}, // invalid parcel size - {id: "Abcdef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid token id - {id: "123456", secret: "AABBCCDDEEFFGGHH", expectedError: true, bts: nil}, // invalid token secret - {id: "123456", secret: "AABBCCD-EEFFGGHH", expectedError: true, bts: nil}, // invalid character - {id: "abc*ef", secret: "1234567890123456", expectedError: true, bts: nil}, // invalid character - {id: "abcdef", secret: "1234567890123456", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "1234567890123456"}}, - {id: "123456", secret: "aabbccddeeffgghh", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "aabbccddeeffgghh"}}, - {id: "abcdef", secret: "abcdef0123456789", expectedError: false, bts: &BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}}, - {id: "123456", secret: "1234560123456789", expectedError: false, bts: &BootstrapTokenString{ID: "123456", Secret: "1234560123456789"}}, - } - for _, rt := range tests { - t.Run(rt.id, func(t *testing.T) { - actual, err := NewBootstrapTokenStringFromIDAndSecret(rt.id, rt.secret) - if (err != nil) != rt.expectedError { - t.Errorf( - "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected error: %t\n\t actual error: %v", - rt.id, - rt.secret, - rt.expectedError, - err, - ) - } else if !reflect.DeepEqual(actual, rt.bts) { - t.Errorf( - "failed NewBootstrapTokenStringFromIDAndSecret for the token with id %q and secret %q\n\texpected: %v\n\t actual: %v", - rt.id, - rt.secret, - rt.bts, - actual, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/conversion.go deleted file mode 100644 index d92ce96f2894c..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/conversion.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -import ( - conversion "k8s.io/apimachinery/pkg/conversion" - kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func Convert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(in, out, s) -} - -func Convert_v1beta2_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { - err := autoConvert_v1beta2_InitConfiguration_To_kubeadm_InitConfiguration(in, out, s) - if err != nil { - return err - } - - // Keep the fuzzer test happy by setting out.ClusterConfiguration to defaults - clusterCfg := &ClusterConfiguration{} - SetDefaults_ClusterConfiguration(clusterCfg) - return Convert_v1beta2_ClusterConfiguration_To_kubeadm_ClusterConfiguration(clusterCfg, &out.ClusterConfiguration, s) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go deleted file mode 100644 index 86e4f507ad2a2..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go +++ /dev/null @@ -1,214 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -import ( - "net/url" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -const ( - // DefaultServiceDNSDomain defines default cluster-internal domain name for Services and Pods - DefaultServiceDNSDomain = "cluster.local" - // DefaultServicesSubnet defines default service subnet range - DefaultServicesSubnet = "10.96.0.0/12" - // DefaultClusterDNSIP defines default DNS IP - DefaultClusterDNSIP = "10.96.0.10" - // DefaultKubernetesVersion defines default kubernetes version - DefaultKubernetesVersion = "stable-1" - // DefaultAPIBindPort defines default API port - DefaultAPIBindPort = 6443 - // DefaultCertificatesDir defines default certificate directory - DefaultCertificatesDir = "/etc/kubernetes/pki" - // DefaultImageRepository defines default image registry - DefaultImageRepository = "k8s.gcr.io" - // DefaultManifestsDir defines default manifests directory - DefaultManifestsDir = "/etc/kubernetes/manifests" - // DefaultClusterName defines the default cluster name - DefaultClusterName = "kubernetes" - - // DefaultEtcdDataDir defines default location of etcd where static pods will save data to - DefaultEtcdDataDir = "/var/lib/etcd" - // DefaultProxyBindAddressv4 is the default bind address when the advertise address is v4 - DefaultProxyBindAddressv4 = "0.0.0.0" - // DefaultProxyBindAddressv6 is the default bind address when the advertise address is v6 - DefaultProxyBindAddressv6 = "::" - // DefaultDiscoveryTimeout specifies the default discovery timeout for kubeadm (used unless one is specified in the JoinConfiguration) - DefaultDiscoveryTimeout = 5 * time.Minute -) - -var ( - // DefaultAuditPolicyLogMaxAge is defined as a var so its address can be taken - // It is the number of days to store audit logs - DefaultAuditPolicyLogMaxAge = int32(2) -) - -func addDefaultingFuncs(scheme *runtime.Scheme) error { - return RegisterDefaults(scheme) -} - -// SetDefaults_InitConfiguration assigns default values for the InitConfiguration -func SetDefaults_InitConfiguration(obj *InitConfiguration) { - SetDefaults_BootstrapTokens(obj) - SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint) -} - -// SetDefaults_ClusterConfiguration assigns default values for the ClusterConfiguration -func SetDefaults_ClusterConfiguration(obj *ClusterConfiguration) { - if obj.KubernetesVersion == "" { - obj.KubernetesVersion = DefaultKubernetesVersion - } - - if obj.Networking.ServiceSubnet == "" { - obj.Networking.ServiceSubnet = DefaultServicesSubnet - } - - if obj.Networking.DNSDomain == "" { - obj.Networking.DNSDomain = DefaultServiceDNSDomain - } - - if obj.CertificatesDir == "" { - obj.CertificatesDir = DefaultCertificatesDir - } - - if obj.ImageRepository == "" { - obj.ImageRepository = DefaultImageRepository - } - - if obj.ClusterName == "" { - obj.ClusterName = DefaultClusterName - } - - SetDefaults_DNS(obj) - SetDefaults_Etcd(obj) - SetDefaults_APIServer(&obj.APIServer) -} - -// SetDefaults_APIServer assigns default values for the API Server -func SetDefaults_APIServer(obj *APIServer) { - if obj.TimeoutForControlPlane == nil { - obj.TimeoutForControlPlane = &metav1.Duration{ - Duration: constants.DefaultControlPlaneTimeout, - } - } -} - -// SetDefaults_DNS assigns default values for the DNS component -func SetDefaults_DNS(obj *ClusterConfiguration) { - if obj.DNS.Type == "" { - obj.DNS.Type = CoreDNS - } -} - -// SetDefaults_Etcd assigns default values for the proxy -func SetDefaults_Etcd(obj *ClusterConfiguration) { - if obj.Etcd.External == nil && obj.Etcd.Local == nil { - obj.Etcd.Local = &LocalEtcd{} - } - if obj.Etcd.Local != nil { - if obj.Etcd.Local.DataDir == "" { - obj.Etcd.Local.DataDir = DefaultEtcdDataDir - } - } -} - -// SetDefaults_JoinConfiguration assigns default values to a regular node -func SetDefaults_JoinConfiguration(obj *JoinConfiguration) { - if obj.CACertPath == "" { - obj.CACertPath = DefaultCACertPath - } - - SetDefaults_JoinControlPlane(obj.ControlPlane) - SetDefaults_Discovery(&obj.Discovery) -} - -func SetDefaults_JoinControlPlane(obj *JoinControlPlane) { - if obj != nil { - SetDefaults_APIEndpoint(&obj.LocalAPIEndpoint) - } -} - -// SetDefaults_Discovery assigns default values for the discovery process -func SetDefaults_Discovery(obj *Discovery) { - if len(obj.TLSBootstrapToken) == 0 && obj.BootstrapToken != nil { - obj.TLSBootstrapToken = obj.BootstrapToken.Token - } - - if obj.Timeout == nil { - obj.Timeout = &metav1.Duration{ - Duration: DefaultDiscoveryTimeout, - } - } - - if obj.File != nil { - SetDefaults_FileDiscovery(obj.File) - } -} - -// SetDefaults_FileDiscovery assigns default values for file based discovery -func SetDefaults_FileDiscovery(obj *FileDiscovery) { - // Make sure file URL becomes path - if len(obj.KubeConfigPath) != 0 { - u, err := url.Parse(obj.KubeConfigPath) - if err == nil && u.Scheme == "file" { - obj.KubeConfigPath = u.Path - } - } -} - -// SetDefaults_BootstrapTokens sets the defaults for the .BootstrapTokens field -// If the slice is empty, it's defaulted with one token. Otherwise it just loops -// through the slice and sets the defaults for the omitempty fields that are TTL, -// Usages and Groups. Token is NOT defaulted with a random one in the API defaulting -// layer, but set to a random value later at runtime if not set before. -func SetDefaults_BootstrapTokens(obj *InitConfiguration) { - - if obj.BootstrapTokens == nil || len(obj.BootstrapTokens) == 0 { - obj.BootstrapTokens = []BootstrapToken{{}} - } - - for i := range obj.BootstrapTokens { - SetDefaults_BootstrapToken(&obj.BootstrapTokens[i]) - } -} - -// SetDefaults_BootstrapToken sets the defaults for an individual Bootstrap Token -func SetDefaults_BootstrapToken(bt *BootstrapToken) { - if bt.TTL == nil { - bt.TTL = &metav1.Duration{ - Duration: constants.DefaultTokenDuration, - } - } - if len(bt.Usages) == 0 { - bt.Usages = constants.DefaultTokenUsages - } - - if len(bt.Groups) == 0 { - bt.Groups = constants.DefaultTokenGroups - } -} - -// SetDefaults_APIEndpoint sets the defaults for the API server instance deployed on a node. -func SetDefaults_APIEndpoint(obj *APIEndpoint) { - if obj.BindPort == 0 { - obj.BindPort = DefaultAPIBindPort - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_unix.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_unix.go deleted file mode 100644 index 5211bc8ea7fd1..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_unix.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build !windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -const ( - // DefaultCACertPath defines default location of CA certificate on Linux - DefaultCACertPath = "/etc/kubernetes/pki/ca.crt" - // DefaultUrlScheme defines default socket url prefix - DefaultUrlScheme = "unix" -) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go deleted file mode 100644 index 9f0986db46e58..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults_windows.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -const ( - // DefaultCACertPath defines default location of CA certificate on Windows - DefaultCACertPath = "C:/etc/kubernetes/pki/ca.crt" - // DefaultUrlScheme defines default socket url prefix - DefaultUrlScheme = "npipe" -) diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go deleted file mode 100644 index 2d4db547d3a27..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go +++ /dev/null @@ -1,276 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:defaulter-gen=TypeMeta -// +groupName=kubeadm.k8s.io -// +k8s:deepcopy-gen=package -// +k8s:conversion-gen=k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm - -// Package v1beta2 defines the v1beta2 version of the kubeadm configuration file format. -// This version improves on the v1beta1 format by fixing some minor issues and adding a few new fields. -// -// A list of changes since v1beta1: -// - "certificateKey" field is added to InitConfiguration and JoinConfiguration. -// - "ignorePreflightErrors" field is added to the NodeRegistrationOptions. -// - The JSON "omitempty" tag is used in a more places where appropriate. -// - The JSON "omitempty" tag of the "taints" field (inside NodeRegistrationOptions) is removed. -// See the Kubernetes 1.15 changelog for further details. -// -// Migration from old kubeadm config versions -// -// Please convert your v1beta1 configuration files to v1beta2 using the "kubeadm config migrate" command of kubeadm v1.15.x -// (conversion from older releases of kubeadm config files requires older release of kubeadm as well e.g. -// kubeadm v1.11 should be used to migrate v1alpha1 to v1alpha2; kubeadm v1.12 should be used to translate v1alpha2 to v1alpha3; -// kubeadm v1.13 or v1.14 should be used to translate v1alpha3 to v1beta1) -// -// Nevertheless, kubeadm v1.15.x will support reading from v1beta1 version of the kubeadm config file format. -// -// Basics -// -// The preferred way to configure kubeadm is to pass an YAML configuration file with the --config option. Some of the -// configuration options defined in the kubeadm config file are also available as command line flags, but only -// the most common/simple use case are supported with this approach. -// -// A kubeadm config file could contain multiple configuration types separated using three dashes (“---”). -// -// kubeadm supports the following configuration types: -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: InitConfiguration -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: ClusterConfiguration -// -// apiVersion: kubelet.config.k8s.io/v1beta1 -// kind: KubeletConfiguration -// -// apiVersion: kubeproxy.config.k8s.io/v1alpha1 -// kind: KubeProxyConfiguration -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: JoinConfiguration -// -// To print the defaults for "init" and "join" actions use the following commands: -// kubeadm config print init-defaults -// kubeadm config print join-defaults -// -// The list of configuration types that must be included in a configuration file depends by the action you are -// performing (init or join) and by the configuration options you are going to use (defaults or advanced customization). -// -// If some configuration types are not provided, or provided only partially, kubeadm will use default values; defaults -// provided by kubeadm includes also enforcing consistency of values across components when required (e.g. -// cluster-cidr flag on controller manager and clusterCIDR on kube-proxy). -// -// Users are always allowed to override default values, with the only exception of a small subset of setting with -// relevance for security (e.g. enforce authorization-mode Node and RBAC on api server) -// -// If the user provides a configuration types that is not expected for the action you are performing, kubeadm will -// ignore those types and print a warning. -// -// Kubeadm init configuration types -// -// When executing kubeadm init with the --config option, the following configuration types could be used: -// InitConfiguration, ClusterConfiguration, KubeProxyConfiguration, KubeletConfiguration, but only one -// between InitConfiguration and ClusterConfiguration is mandatory. -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: InitConfiguration -// bootstrapTokens: -// ... -// nodeRegistration: -// ... -// -// The InitConfiguration type should be used to configure runtime settings, that in case of kubeadm init -// are the configuration of the bootstrap token and all the setting which are specific to the node where kubeadm -// is executed, including: -// -// - NodeRegistration, that holds fields that relate to registering the new node to the cluster; -// use it to customize the node name, the CRI socket to use or any other settings that should apply to this -// node only (e.g. the node ip). -// -// - LocalAPIEndpoint, that represents the endpoint of the instance of the API server to be deployed on this node; -// use it e.g. to customize the API server advertise address. -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: ClusterConfiguration -// networking: -// ... -// etcd: -// ... -// apiServer: -// extraArgs: -// ... -// extraVolumes: -// ... -// ... -// -// The ClusterConfiguration type should be used to configure cluster-wide settings, -// including settings for: -// -// - Networking, that holds configuration for the networking topology of the cluster; use it e.g. to customize -// node subnet or services subnet. -// -// - Etcd configurations; use it e.g. to customize the local etcd or to configure the API server -// for using an external etcd cluster. -// -// - kube-apiserver, kube-scheduler, kube-controller-manager configurations; use it to customize control-plane -// components by adding customized setting or overriding kubeadm default settings. -// -// apiVersion: kubeproxy.config.k8s.io/v1alpha1 -// kind: KubeProxyConfiguration -// ... -// -// The KubeProxyConfiguration type should be used to change the configuration passed to kube-proxy instances deployed -// in the cluster. If this object is not provided or provided only partially, kubeadm applies defaults. -// -// See https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/ or https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration -// for kube proxy official documentation. -// -// apiVersion: kubelet.config.k8s.io/v1beta1 -// kind: KubeletConfiguration -// ... -// -// The KubeletConfiguration type should be used to change the configurations that will be passed to all kubelet instances -// deployed in the cluster. If this object is not provided or provided only partially, kubeadm applies defaults. -// -// See https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ or https://godoc.org/k8s.io/kubelet/config/v1beta1#KubeletConfiguration -// for kubelet official documentation. -// -// Here is a fully populated example of a single YAML file containing multiple -// configuration types to be used during a `kubeadm init` run. -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: InitConfiguration -// bootstrapTokens: -// - token: "9a08jv.c0izixklcxtmnze7" -// description: "kubeadm bootstrap token" -// ttl: "24h" -// - token: "783bde.3f89s0fje9f38fhf" -// description: "another bootstrap token" -// usages: -// - authentication -// - signing -// groups: -// - system:bootstrappers:kubeadm:default-node-token -// nodeRegistration: -// name: "ec2-10-100-0-1" -// criSocket: "/var/run/dockershim.sock" -// taints: -// - key: "kubeadmNode" -// value: "master" -// effect: "NoSchedule" -// kubeletExtraArgs: -// cgroup-driver: "cgroupfs" -// ignorePreflightErrors: -// - IsPrivilegedUser -// localAPIEndpoint: -// advertiseAddress: "10.100.0.1" -// bindPort: 6443 -// certificateKey: "e6a2eb8581237ab72a4f494f30285ec12a9694d750b9785706a83bfcbbbd2204" -// --- -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: ClusterConfiguration -// etcd: -// # one of local or external -// local: -// imageRepository: "k8s.gcr.io" -// imageTag: "3.2.24" -// dataDir: "/var/lib/etcd" -// extraArgs: -// listen-client-urls: "http://10.100.0.1:2379" -// serverCertSANs: -// - "ec2-10-100-0-1.compute-1.amazonaws.com" -// peerCertSANs: -// - "10.100.0.1" -// # external: -// # endpoints: -// # - "10.100.0.1:2379" -// # - "10.100.0.2:2379" -// # caFile: "/etcd/kubernetes/pki/etcd/etcd-ca.crt" -// # certFile: "/etcd/kubernetes/pki/etcd/etcd.crt" -// # keyFile: "/etcd/kubernetes/pki/etcd/etcd.key" -// networking: -// serviceSubnet: "10.96.0.0/12" -// podSubnet: "10.100.0.1/24" -// dnsDomain: "cluster.local" -// kubernetesVersion: "v1.12.0" -// controlPlaneEndpoint: "10.100.0.1:6443" -// apiServer: -// extraArgs: -// authorization-mode: "Node,RBAC" -// extraVolumes: -// - name: "some-volume" -// hostPath: "/etc/some-path" -// mountPath: "/etc/some-pod-path" -// readOnly: false -// pathType: File -// certSANs: -// - "10.100.1.1" -// - "ec2-10-100-0-1.compute-1.amazonaws.com" -// timeoutForControlPlane: 4m0s -// controllerManager: -// extraArgs: -// "node-cidr-mask-size": "20" -// extraVolumes: -// - name: "some-volume" -// hostPath: "/etc/some-path" -// mountPath: "/etc/some-pod-path" -// readOnly: false -// pathType: File -// scheduler: -// extraArgs: -// address: "10.100.0.1" -// extraVolumes: -// - name: "some-volume" -// hostPath: "/etc/some-path" -// mountPath: "/etc/some-pod-path" -// readOnly: false -// pathType: File -// certificatesDir: "/etc/kubernetes/pki" -// imageRepository: "k8s.gcr.io" -// useHyperKubeImage: false -// clusterName: "example-cluster" -// --- -// apiVersion: kubelet.config.k8s.io/v1beta1 -// kind: KubeletConfiguration -// # kubelet specific options here -// --- -// apiVersion: kubeproxy.config.k8s.io/v1alpha1 -// kind: KubeProxyConfiguration -// # kube-proxy specific options here -// -// Kubeadm join configuration types -// -// When executing kubeadm join with the --config option, the JoinConfiguration type should be provided. -// -// apiVersion: kubeadm.k8s.io/v1beta2 -// kind: JoinConfiguration -// ... -// -// The JoinConfiguration type should be used to configure runtime settings, that in case of kubeadm join -// are the discovery method used for accessing the cluster info and all the setting which are specific -// to the node where kubeadm is executed, including: -// -// - NodeRegistration, that holds fields that relate to registering the new node to the cluster; -// use it to customize the node name, the CRI socket to use or any other settings that should apply to this -// node only (e.g. the node ip). -// -// - APIEndpoint, that represents the endpoint of the instance of the API server to be eventually deployed on this node. -// -package v1beta2 // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - -//TODO: The BootstrapTokenString object should move out to either k8s.io/client-go or k8s.io/api in the future -//(probably as part of Bootstrap Tokens going GA). It should not be staged under the kubeadm API as it is now. diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/register.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/register.go deleted file mode 100644 index d14e9d24f23c4..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/register.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupName is the group name use in this package -const GroupName = "kubeadm.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta2"} - -var ( - // TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api. - // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. - - // SchemeBuilder points to a list of functions added to Scheme. - SchemeBuilder runtime.SchemeBuilder - localSchemeBuilder = &SchemeBuilder - // AddToScheme applies all the stored functions to the scheme. - AddToScheme = localSchemeBuilder.AddToScheme -) - -func init() { - // We only register manually written functions here. The registration of the - // generated functions takes place in the generated files. The separation - // makes the code compile even when the generated files are missing. - localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) -} - -// Kind takes an unqualified kind and returns a Group qualified GroupKind -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &InitConfiguration{}, - &ClusterConfiguration{}, - &ClusterStatus{}, - &JoinConfiguration{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go deleted file mode 100644 index a9261d06731fe..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go +++ /dev/null @@ -1,396 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta2 - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// InitConfiguration contains a list of elements that is specific "kubeadm init"-only runtime -// information. -type InitConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. - // After that, the information in the fields IS NOT uploaded to the `kubeadm-config` ConfigMap - // that is used by `kubeadm upgrade` for instance. These fields must be omitempty. - - // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. - // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature - BootstrapTokens []BootstrapToken `json:"bootstrapTokens,omitempty"` - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` - - // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node - // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint - // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This - // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible - // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process - // fails you may set the desired value here. - LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` - - // CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in - // a secret in the cluster during the uploadcerts init phase. - CertificateKey string `json:"certificateKey,omitempty"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster -type ClusterConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // Etcd holds configuration for etcd. - Etcd Etcd `json:"etcd,omitempty"` - - // Networking holds configuration for the networking topology of the cluster. - Networking Networking `json:"networking,omitempty"` - - // KubernetesVersion is the target version of the control plane. - KubernetesVersion string `json:"kubernetesVersion,omitempty"` - - // ControlPlaneEndpoint sets a stable IP address or DNS name for the control plane; it - // can be a valid IP address or a RFC-1123 DNS subdomain, both with optional TCP port. - // In case the ControlPlaneEndpoint is not specified, the AdvertiseAddress + BindPort - // are used; in case the ControlPlaneEndpoint is specified but without a TCP port, - // the BindPort is used. - // Possible usages are: - // e.g. In a cluster with more than one control plane instances, this field should be - // assigned the address of the external load balancer in front of the - // control plane instances. - // e.g. in environments with enforced node recycling, the ControlPlaneEndpoint - // could be used for assigning a stable DNS to the control plane. - ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"` - - // APIServer contains extra settings for the API server control plane component - APIServer APIServer `json:"apiServer,omitempty"` - - // ControllerManager contains extra settings for the controller manager control plane component - ControllerManager ControlPlaneComponent `json:"controllerManager,omitempty"` - - // Scheduler contains extra settings for the scheduler control plane component - Scheduler ControlPlaneComponent `json:"scheduler,omitempty"` - - // DNS defines the options for the DNS add-on installed in the cluster. - DNS DNS `json:"dns,omitempty"` - - // CertificatesDir specifies where to store or look for all required certificates. - CertificatesDir string `json:"certificatesDir,omitempty"` - - // ImageRepository sets the container registry to pull images from. - // If empty, `k8s.gcr.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/` or `ci-cross/`) - // `gcr.io/kubernetes-ci-images` will be used as a default for control plane components and for kube-proxy, while `k8s.gcr.io` - // will be used for all the other images. - ImageRepository string `json:"imageRepository,omitempty"` - - // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images - // DEPRECATED: As hyperkube is itself deprecated, this fields is too. It will be removed in future kubeadm config versions, kubeadm - // will print multiple warnings when set to true, and at some point it may become ignored. - UseHyperKubeImage bool `json:"useHyperKubeImage,omitempty"` - - // FeatureGates enabled by the user. - FeatureGates map[string]bool `json:"featureGates,omitempty"` - - // The cluster name - ClusterName string `json:"clusterName,omitempty"` -} - -// ControlPlaneComponent holds settings common to control plane component of the cluster -type ControlPlaneComponent struct { - // ExtraArgs is an extra set of flags to pass to the control plane component. - // TODO: This is temporary and ideally we would like to switch all components to - // use ComponentConfig + ConfigMaps. - ExtraArgs map[string]string `json:"extraArgs,omitempty"` - - // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. - ExtraVolumes []HostPathMount `json:"extraVolumes,omitempty"` -} - -// APIServer holds settings necessary for API server deployments in the cluster -type APIServer struct { - ControlPlaneComponent `json:",inline"` - - // CertSANs sets extra Subject Alternative Names for the API Server signing cert. - CertSANs []string `json:"certSANs,omitempty"` - - // TimeoutForControlPlane controls the timeout that we use for API server to appear - TimeoutForControlPlane *metav1.Duration `json:"timeoutForControlPlane,omitempty"` -} - -// DNSAddOnType defines string identifying DNS add-on types -type DNSAddOnType string - -const ( - // CoreDNS add-on type - CoreDNS DNSAddOnType = "CoreDNS" - - // KubeDNS add-on type - KubeDNS DNSAddOnType = "kube-dns" -) - -// DNS defines the DNS addon that should be used in the cluster -type DNS struct { - // Type defines the DNS add-on to be used - Type DNSAddOnType `json:"type"` - - // ImageMeta allows to customize the image used for the DNS component - ImageMeta `json:",inline"` -} - -// ImageMeta allows to customize the image used for components that are not -// originated from the Kubernetes/Kubernetes release process -type ImageMeta struct { - // ImageRepository sets the container registry to pull images from. - // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. - ImageRepository string `json:"imageRepository,omitempty"` - - // ImageTag allows to specify a tag for the image. - // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. - ImageTag string `json:"imageTag,omitempty"` - - //TODO: evaluate if we need also a ImageName based on user feedbacks -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterStatus contains the cluster status. The ClusterStatus will be stored in the kubeadm-config -// ConfigMap in the cluster, and then updated by kubeadm when additional control plane instance joins or leaves the cluster. -type ClusterStatus struct { - metav1.TypeMeta `json:",inline"` - - // APIEndpoints currently available in the cluster, one for each control plane/api server instance. - // The key of the map is the IP of the host's default interface - APIEndpoints map[string]APIEndpoint `json:"apiEndpoints"` -} - -// APIEndpoint struct contains elements of API server instance deployed on a node. -type APIEndpoint struct { - // AdvertiseAddress sets the IP address for the API server to advertise. - AdvertiseAddress string `json:"advertiseAddress,omitempty"` - - // BindPort sets the secure port for the API Server to bind to. - // Defaults to 6443. - BindPort int32 `json:"bindPort,omitempty"` -} - -// NodeRegistrationOptions holds fields that relate to registering a new control-plane or node to the cluster, either via "kubeadm init" or "kubeadm join" -type NodeRegistrationOptions struct { - - // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm join` operation. - // This field is also used in the CommonName field of the kubelet's client certificate to the API server. - // Defaults to the hostname of the node if not provided. - Name string `json:"name,omitempty"` - - // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use - CRISocket string `json:"criSocket,omitempty"` - - // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process - // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an - // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. - Taints []v1.Taint `json:"taints"` - - // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file - // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap - // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. - KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` - - // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered. - IgnorePreflightErrors []string `json:"ignorePreflightErrors,omitempty"` -} - -// Networking contains elements describing cluster's networking configuration -type Networking struct { - // ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12". - ServiceSubnet string `json:"serviceSubnet,omitempty"` - // PodSubnet is the subnet used by pods. - PodSubnet string `json:"podSubnet,omitempty"` - // DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local". - DNSDomain string `json:"dnsDomain,omitempty"` -} - -// BootstrapToken describes one bootstrap token, stored as a Secret in the cluster -type BootstrapToken struct { - // Token is used for establishing bidirectional trust between nodes and control-planes. - // Used for joining nodes in the cluster. - Token *BootstrapTokenString `json:"token"` - // Description sets a human-friendly message why this token exists and what it's used - // for, so other administrators can know its purpose. - Description string `json:"description,omitempty"` - // TTL defines the time to live for this token. Defaults to 24h. - // Expires and TTL are mutually exclusive. - TTL *metav1.Duration `json:"ttl,omitempty"` - // Expires specifies the timestamp when this token expires. Defaults to being set - // dynamically at runtime based on the TTL. Expires and TTL are mutually exclusive. - Expires *metav1.Time `json:"expires,omitempty"` - // Usages describes the ways in which this token can be used. Can by default be used - // for establishing bidirectional trust, but that can be changed here. - Usages []string `json:"usages,omitempty"` - // Groups specifies the extra groups that this token will authenticate as when/if - // used for authentication - Groups []string `json:"groups,omitempty"` -} - -// Etcd contains elements describing Etcd configuration. -type Etcd struct { - - // Local provides configuration knobs for configuring the local etcd instance - // Local and External are mutually exclusive - Local *LocalEtcd `json:"local,omitempty"` - - // External describes how to connect to an external etcd cluster - // Local and External are mutually exclusive - External *ExternalEtcd `json:"external,omitempty"` -} - -// LocalEtcd describes that kubeadm should run an etcd cluster locally -type LocalEtcd struct { - // ImageMeta allows to customize the container used for etcd - ImageMeta `json:",inline"` - - // DataDir is the directory etcd will place its data. - // Defaults to "/var/lib/etcd". - DataDir string `json:"dataDir"` - - // ExtraArgs are extra arguments provided to the etcd binary - // when run inside a static pod. - ExtraArgs map[string]string `json:"extraArgs,omitempty"` - - // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. - ServerCertSANs []string `json:"serverCertSANs,omitempty"` - // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. - PeerCertSANs []string `json:"peerCertSANs,omitempty"` -} - -// ExternalEtcd describes an external etcd cluster. -// Kubeadm has no knowledge of where certificate files live and they must be supplied. -type ExternalEtcd struct { - // Endpoints of etcd members. Required for ExternalEtcd. - Endpoints []string `json:"endpoints"` - - // CAFile is an SSL Certificate Authority file used to secure etcd communication. - // Required if using a TLS connection. - CAFile string `json:"caFile"` - - // CertFile is an SSL certification file used to secure etcd communication. - // Required if using a TLS connection. - CertFile string `json:"certFile"` - - // KeyFile is an SSL key file used to secure etcd communication. - // Required if using a TLS connection. - KeyFile string `json:"keyFile"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// JoinConfiguration contains elements describing a particular node. -type JoinConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster - NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` - - // CACertPath is the path to the SSL certificate authority used to - // secure comunications between node and control-plane. - // Defaults to "/etc/kubernetes/pki/ca.crt". - CACertPath string `json:"caCertPath,omitempty"` - - // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process - Discovery Discovery `json:"discovery"` - - // ControlPlane defines the additional control plane instance to be deployed on the joining node. - // If nil, no additional control plane instance will be deployed. - ControlPlane *JoinControlPlane `json:"controlPlane,omitempty"` -} - -// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. -type JoinControlPlane struct { - // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. - LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` - - // CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret - // upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration. - CertificateKey string `json:"certificateKey,omitempty"` -} - -// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process -type Discovery struct { - // BootstrapToken is used to set the options for bootstrap token based discovery - // BootstrapToken and File are mutually exclusive - BootstrapToken *BootstrapTokenDiscovery `json:"bootstrapToken,omitempty"` - - // File is used to specify a file or URL to a kubeconfig file from which to load cluster information - // BootstrapToken and File are mutually exclusive - File *FileDiscovery `json:"file,omitempty"` - - // TLSBootstrapToken is a token used for TLS bootstrapping. - // If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token, but can be overridden. - // If .File is set, this field **must be set** in case the KubeConfigFile does not contain any other authentication information - TLSBootstrapToken string `json:"tlsBootstrapToken,omitempty"` - - // Timeout modifies the discovery timeout - Timeout *metav1.Duration `json:"timeout,omitempty"` -} - -// BootstrapTokenDiscovery is used to set the options for bootstrap token based discovery -type BootstrapTokenDiscovery struct { - // Token is a token used to validate cluster information - // fetched from the control-plane. - Token string `json:"token"` - - // APIServerEndpoint is an IP or domain name to the API server from which info will be fetched. - APIServerEndpoint string `json:"apiServerEndpoint,omitempty"` - - // CACertHashes specifies a set of public key pins to verify - // when token-based discovery is used. The root CA found during discovery - // must match one of these values. Specifying an empty set disables root CA - // pinning, which can be unsafe. Each hash is specified as ":", - // where the only currently supported type is "sha256". This is a hex-encoded - // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded - // ASN.1. These hashes can be calculated using, for example, OpenSSL. - CACertHashes []string `json:"caCertHashes,omitempty"` - - // UnsafeSkipCAVerification allows token-based discovery - // without CA verification via CACertHashes. This can weaken - // the security of kubeadm since other nodes can impersonate the control-plane. - UnsafeSkipCAVerification bool `json:"unsafeSkipCAVerification,omitempty"` -} - -// FileDiscovery is used to specify a file or URL to a kubeconfig file from which to load cluster information -type FileDiscovery struct { - // KubeConfigPath is used to specify the actual file path or URL to the kubeconfig file from which to load cluster information - KubeConfigPath string `json:"kubeConfigPath"` -} - -// HostPathMount contains elements describing volumes that are mounted from the -// host. -type HostPathMount struct { - // Name of the volume inside the pod template. - Name string `json:"name"` - // HostPath is the path in the host that will be mounted inside - // the pod. - HostPath string `json:"hostPath"` - // MountPath is the path inside the pod where hostPath will be mounted. - MountPath string `json:"mountPath"` - // ReadOnly controls write access to the volume - ReadOnly bool `json:"readOnly,omitempty"` - // PathType is the type of the HostPath. - PathType v1.HostPathType `json:"pathType,omitempty"` -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go deleted file mode 100644 index 6bcfa1e2a150d..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.conversion.go +++ /dev/null @@ -1,830 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by conversion-gen. DO NOT EDIT. - -package v1beta2 - -import ( - unsafe "unsafe" - - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - conversion "k8s.io/apimachinery/pkg/conversion" - runtime "k8s.io/apimachinery/pkg/runtime" - kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func init() { - localSchemeBuilder.Register(RegisterConversions) -} - -// RegisterConversions adds conversion functions to the given scheme. -// Public to allow building arbitrary schemes. -func RegisterConversions(s *runtime.Scheme) error { - if err := s.AddGeneratedConversionFunc((*APIEndpoint)(nil), (*kubeadm.APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(a.(*APIEndpoint), b.(*kubeadm.APIEndpoint), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.APIEndpoint)(nil), (*APIEndpoint)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(a.(*kubeadm.APIEndpoint), b.(*APIEndpoint), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*APIServer)(nil), (*kubeadm.APIServer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_APIServer_To_kubeadm_APIServer(a.(*APIServer), b.(*kubeadm.APIServer), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.APIServer)(nil), (*APIServer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_APIServer_To_v1beta2_APIServer(a.(*kubeadm.APIServer), b.(*APIServer), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*BootstrapToken)(nil), (*kubeadm.BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_BootstrapToken_To_kubeadm_BootstrapToken(a.(*BootstrapToken), b.(*kubeadm.BootstrapToken), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.BootstrapToken)(nil), (*BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_BootstrapToken_To_v1beta2_BootstrapToken(a.(*kubeadm.BootstrapToken), b.(*BootstrapToken), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*BootstrapTokenDiscovery)(nil), (*kubeadm.BootstrapTokenDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(a.(*BootstrapTokenDiscovery), b.(*kubeadm.BootstrapTokenDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.BootstrapTokenDiscovery)(nil), (*BootstrapTokenDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_BootstrapTokenDiscovery_To_v1beta2_BootstrapTokenDiscovery(a.(*kubeadm.BootstrapTokenDiscovery), b.(*BootstrapTokenDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*BootstrapTokenString)(nil), (*kubeadm.BootstrapTokenString)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_BootstrapTokenString_To_kubeadm_BootstrapTokenString(a.(*BootstrapTokenString), b.(*kubeadm.BootstrapTokenString), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.BootstrapTokenString)(nil), (*BootstrapTokenString)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_BootstrapTokenString_To_v1beta2_BootstrapTokenString(a.(*kubeadm.BootstrapTokenString), b.(*BootstrapTokenString), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ClusterConfiguration)(nil), (*kubeadm.ClusterConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ClusterConfiguration_To_kubeadm_ClusterConfiguration(a.(*ClusterConfiguration), b.(*kubeadm.ClusterConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ClusterConfiguration)(nil), (*ClusterConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ClusterConfiguration_To_v1beta2_ClusterConfiguration(a.(*kubeadm.ClusterConfiguration), b.(*ClusterConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ClusterStatus)(nil), (*kubeadm.ClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ClusterStatus_To_kubeadm_ClusterStatus(a.(*ClusterStatus), b.(*kubeadm.ClusterStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ClusterStatus)(nil), (*ClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ClusterStatus_To_v1beta2_ClusterStatus(a.(*kubeadm.ClusterStatus), b.(*ClusterStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ControlPlaneComponent)(nil), (*kubeadm.ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(a.(*ControlPlaneComponent), b.(*kubeadm.ControlPlaneComponent), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ControlPlaneComponent)(nil), (*ControlPlaneComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(a.(*kubeadm.ControlPlaneComponent), b.(*ControlPlaneComponent), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*DNS)(nil), (*kubeadm.DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_DNS_To_kubeadm_DNS(a.(*DNS), b.(*kubeadm.DNS), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.DNS)(nil), (*DNS)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_DNS_To_v1beta2_DNS(a.(*kubeadm.DNS), b.(*DNS), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Discovery)(nil), (*kubeadm.Discovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_Discovery_To_kubeadm_Discovery(a.(*Discovery), b.(*kubeadm.Discovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.Discovery)(nil), (*Discovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_Discovery_To_v1beta2_Discovery(a.(*kubeadm.Discovery), b.(*Discovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Etcd)(nil), (*kubeadm.Etcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_Etcd_To_kubeadm_Etcd(a.(*Etcd), b.(*kubeadm.Etcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.Etcd)(nil), (*Etcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_Etcd_To_v1beta2_Etcd(a.(*kubeadm.Etcd), b.(*Etcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ExternalEtcd)(nil), (*kubeadm.ExternalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ExternalEtcd_To_kubeadm_ExternalEtcd(a.(*ExternalEtcd), b.(*kubeadm.ExternalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ExternalEtcd)(nil), (*ExternalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ExternalEtcd_To_v1beta2_ExternalEtcd(a.(*kubeadm.ExternalEtcd), b.(*ExternalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*FileDiscovery)(nil), (*kubeadm.FileDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_FileDiscovery_To_kubeadm_FileDiscovery(a.(*FileDiscovery), b.(*kubeadm.FileDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.FileDiscovery)(nil), (*FileDiscovery)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_FileDiscovery_To_v1beta2_FileDiscovery(a.(*kubeadm.FileDiscovery), b.(*FileDiscovery), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*HostPathMount)(nil), (*kubeadm.HostPathMount)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_HostPathMount_To_kubeadm_HostPathMount(a.(*HostPathMount), b.(*kubeadm.HostPathMount), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.HostPathMount)(nil), (*HostPathMount)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_HostPathMount_To_v1beta2_HostPathMount(a.(*kubeadm.HostPathMount), b.(*HostPathMount), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ImageMeta)(nil), (*kubeadm.ImageMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ImageMeta_To_kubeadm_ImageMeta(a.(*ImageMeta), b.(*kubeadm.ImageMeta), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.ImageMeta)(nil), (*ImageMeta)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_ImageMeta_To_v1beta2_ImageMeta(a.(*kubeadm.ImageMeta), b.(*ImageMeta), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*JoinConfiguration)(nil), (*kubeadm.JoinConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_JoinConfiguration_To_kubeadm_JoinConfiguration(a.(*JoinConfiguration), b.(*kubeadm.JoinConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.JoinConfiguration)(nil), (*JoinConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_JoinConfiguration_To_v1beta2_JoinConfiguration(a.(*kubeadm.JoinConfiguration), b.(*JoinConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*JoinControlPlane)(nil), (*kubeadm.JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane(a.(*JoinControlPlane), b.(*kubeadm.JoinControlPlane), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.JoinControlPlane)(nil), (*JoinControlPlane)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane(a.(*kubeadm.JoinControlPlane), b.(*JoinControlPlane), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*LocalEtcd)(nil), (*kubeadm.LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_LocalEtcd_To_kubeadm_LocalEtcd(a.(*LocalEtcd), b.(*kubeadm.LocalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.LocalEtcd)(nil), (*LocalEtcd)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_LocalEtcd_To_v1beta2_LocalEtcd(a.(*kubeadm.LocalEtcd), b.(*LocalEtcd), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Networking)(nil), (*kubeadm.Networking)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_Networking_To_kubeadm_Networking(a.(*Networking), b.(*kubeadm.Networking), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.Networking)(nil), (*Networking)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_Networking_To_v1beta2_Networking(a.(*kubeadm.Networking), b.(*Networking), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*NodeRegistrationOptions)(nil), (*kubeadm.NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(a.(*NodeRegistrationOptions), b.(*kubeadm.NodeRegistrationOptions), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*kubeadm.NodeRegistrationOptions)(nil), (*NodeRegistrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions(a.(*kubeadm.NodeRegistrationOptions), b.(*NodeRegistrationOptions), scope) - }); err != nil { - return err - } - if err := s.AddConversionFunc((*kubeadm.InitConfiguration)(nil), (*InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(a.(*kubeadm.InitConfiguration), b.(*InitConfiguration), scope) - }); err != nil { - return err - } - if err := s.AddConversionFunc((*InitConfiguration)(nil), (*kubeadm.InitConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_InitConfiguration_To_kubeadm_InitConfiguration(a.(*InitConfiguration), b.(*kubeadm.InitConfiguration), scope) - }); err != nil { - return err - } - return nil -} - -func autoConvert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(in *APIEndpoint, out *kubeadm.APIEndpoint, s conversion.Scope) error { - out.AdvertiseAddress = in.AdvertiseAddress - out.BindPort = in.BindPort - return nil -} - -// Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint is an autogenerated conversion function. -func Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(in *APIEndpoint, out *kubeadm.APIEndpoint, s conversion.Scope) error { - return autoConvert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(in, out, s) -} - -func autoConvert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(in *kubeadm.APIEndpoint, out *APIEndpoint, s conversion.Scope) error { - out.AdvertiseAddress = in.AdvertiseAddress - out.BindPort = in.BindPort - return nil -} - -// Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint is an autogenerated conversion function. -func Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(in *kubeadm.APIEndpoint, out *APIEndpoint, s conversion.Scope) error { - return autoConvert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(in, out, s) -} - -func autoConvert_v1beta2_APIServer_To_kubeadm_APIServer(in *APIServer, out *kubeadm.APIServer, s conversion.Scope) error { - if err := Convert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.ControlPlaneComponent, &out.ControlPlaneComponent, s); err != nil { - return err - } - out.CertSANs = *(*[]string)(unsafe.Pointer(&in.CertSANs)) - out.TimeoutForControlPlane = (*v1.Duration)(unsafe.Pointer(in.TimeoutForControlPlane)) - return nil -} - -// Convert_v1beta2_APIServer_To_kubeadm_APIServer is an autogenerated conversion function. -func Convert_v1beta2_APIServer_To_kubeadm_APIServer(in *APIServer, out *kubeadm.APIServer, s conversion.Scope) error { - return autoConvert_v1beta2_APIServer_To_kubeadm_APIServer(in, out, s) -} - -func autoConvert_kubeadm_APIServer_To_v1beta2_APIServer(in *kubeadm.APIServer, out *APIServer, s conversion.Scope) error { - if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(&in.ControlPlaneComponent, &out.ControlPlaneComponent, s); err != nil { - return err - } - out.CertSANs = *(*[]string)(unsafe.Pointer(&in.CertSANs)) - out.TimeoutForControlPlane = (*v1.Duration)(unsafe.Pointer(in.TimeoutForControlPlane)) - return nil -} - -// Convert_kubeadm_APIServer_To_v1beta2_APIServer is an autogenerated conversion function. -func Convert_kubeadm_APIServer_To_v1beta2_APIServer(in *kubeadm.APIServer, out *APIServer, s conversion.Scope) error { - return autoConvert_kubeadm_APIServer_To_v1beta2_APIServer(in, out, s) -} - -func autoConvert_v1beta2_BootstrapToken_To_kubeadm_BootstrapToken(in *BootstrapToken, out *kubeadm.BootstrapToken, s conversion.Scope) error { - out.Token = (*kubeadm.BootstrapTokenString)(unsafe.Pointer(in.Token)) - out.Description = in.Description - out.TTL = (*v1.Duration)(unsafe.Pointer(in.TTL)) - out.Expires = (*v1.Time)(unsafe.Pointer(in.Expires)) - out.Usages = *(*[]string)(unsafe.Pointer(&in.Usages)) - out.Groups = *(*[]string)(unsafe.Pointer(&in.Groups)) - return nil -} - -// Convert_v1beta2_BootstrapToken_To_kubeadm_BootstrapToken is an autogenerated conversion function. -func Convert_v1beta2_BootstrapToken_To_kubeadm_BootstrapToken(in *BootstrapToken, out *kubeadm.BootstrapToken, s conversion.Scope) error { - return autoConvert_v1beta2_BootstrapToken_To_kubeadm_BootstrapToken(in, out, s) -} - -func autoConvert_kubeadm_BootstrapToken_To_v1beta2_BootstrapToken(in *kubeadm.BootstrapToken, out *BootstrapToken, s conversion.Scope) error { - out.Token = (*BootstrapTokenString)(unsafe.Pointer(in.Token)) - out.Description = in.Description - out.TTL = (*v1.Duration)(unsafe.Pointer(in.TTL)) - out.Expires = (*v1.Time)(unsafe.Pointer(in.Expires)) - out.Usages = *(*[]string)(unsafe.Pointer(&in.Usages)) - out.Groups = *(*[]string)(unsafe.Pointer(&in.Groups)) - return nil -} - -// Convert_kubeadm_BootstrapToken_To_v1beta2_BootstrapToken is an autogenerated conversion function. -func Convert_kubeadm_BootstrapToken_To_v1beta2_BootstrapToken(in *kubeadm.BootstrapToken, out *BootstrapToken, s conversion.Scope) error { - return autoConvert_kubeadm_BootstrapToken_To_v1beta2_BootstrapToken(in, out, s) -} - -func autoConvert_v1beta2_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in *BootstrapTokenDiscovery, out *kubeadm.BootstrapTokenDiscovery, s conversion.Scope) error { - out.Token = in.Token - out.APIServerEndpoint = in.APIServerEndpoint - out.CACertHashes = *(*[]string)(unsafe.Pointer(&in.CACertHashes)) - out.UnsafeSkipCAVerification = in.UnsafeSkipCAVerification - return nil -} - -// Convert_v1beta2_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery is an autogenerated conversion function. -func Convert_v1beta2_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in *BootstrapTokenDiscovery, out *kubeadm.BootstrapTokenDiscovery, s conversion.Scope) error { - return autoConvert_v1beta2_BootstrapTokenDiscovery_To_kubeadm_BootstrapTokenDiscovery(in, out, s) -} - -func autoConvert_kubeadm_BootstrapTokenDiscovery_To_v1beta2_BootstrapTokenDiscovery(in *kubeadm.BootstrapTokenDiscovery, out *BootstrapTokenDiscovery, s conversion.Scope) error { - out.Token = in.Token - out.APIServerEndpoint = in.APIServerEndpoint - out.CACertHashes = *(*[]string)(unsafe.Pointer(&in.CACertHashes)) - out.UnsafeSkipCAVerification = in.UnsafeSkipCAVerification - return nil -} - -// Convert_kubeadm_BootstrapTokenDiscovery_To_v1beta2_BootstrapTokenDiscovery is an autogenerated conversion function. -func Convert_kubeadm_BootstrapTokenDiscovery_To_v1beta2_BootstrapTokenDiscovery(in *kubeadm.BootstrapTokenDiscovery, out *BootstrapTokenDiscovery, s conversion.Scope) error { - return autoConvert_kubeadm_BootstrapTokenDiscovery_To_v1beta2_BootstrapTokenDiscovery(in, out, s) -} - -func autoConvert_v1beta2_BootstrapTokenString_To_kubeadm_BootstrapTokenString(in *BootstrapTokenString, out *kubeadm.BootstrapTokenString, s conversion.Scope) error { - out.ID = in.ID - out.Secret = in.Secret - return nil -} - -// Convert_v1beta2_BootstrapTokenString_To_kubeadm_BootstrapTokenString is an autogenerated conversion function. -func Convert_v1beta2_BootstrapTokenString_To_kubeadm_BootstrapTokenString(in *BootstrapTokenString, out *kubeadm.BootstrapTokenString, s conversion.Scope) error { - return autoConvert_v1beta2_BootstrapTokenString_To_kubeadm_BootstrapTokenString(in, out, s) -} - -func autoConvert_kubeadm_BootstrapTokenString_To_v1beta2_BootstrapTokenString(in *kubeadm.BootstrapTokenString, out *BootstrapTokenString, s conversion.Scope) error { - out.ID = in.ID - out.Secret = in.Secret - return nil -} - -// Convert_kubeadm_BootstrapTokenString_To_v1beta2_BootstrapTokenString is an autogenerated conversion function. -func Convert_kubeadm_BootstrapTokenString_To_v1beta2_BootstrapTokenString(in *kubeadm.BootstrapTokenString, out *BootstrapTokenString, s conversion.Scope) error { - return autoConvert_kubeadm_BootstrapTokenString_To_v1beta2_BootstrapTokenString(in, out, s) -} - -func autoConvert_v1beta2_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration, s conversion.Scope) error { - if err := Convert_v1beta2_Etcd_To_kubeadm_Etcd(&in.Etcd, &out.Etcd, s); err != nil { - return err - } - if err := Convert_v1beta2_Networking_To_kubeadm_Networking(&in.Networking, &out.Networking, s); err != nil { - return err - } - out.KubernetesVersion = in.KubernetesVersion - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - if err := Convert_v1beta2_APIServer_To_kubeadm_APIServer(&in.APIServer, &out.APIServer, s); err != nil { - return err - } - if err := Convert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.ControllerManager, &out.ControllerManager, s); err != nil { - return err - } - if err := Convert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(&in.Scheduler, &out.Scheduler, s); err != nil { - return err - } - if err := Convert_v1beta2_DNS_To_kubeadm_DNS(&in.DNS, &out.DNS, s); err != nil { - return err - } - out.CertificatesDir = in.CertificatesDir - out.ImageRepository = in.ImageRepository - out.UseHyperKubeImage = in.UseHyperKubeImage - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) - out.ClusterName = in.ClusterName - return nil -} - -// Convert_v1beta2_ClusterConfiguration_To_kubeadm_ClusterConfiguration is an autogenerated conversion function. -func Convert_v1beta2_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration, s conversion.Scope) error { - return autoConvert_v1beta2_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in, out, s) -} - -func autoConvert_kubeadm_ClusterConfiguration_To_v1beta2_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error { - // INFO: in.ComponentConfigs opted out of conversion generation - if err := Convert_kubeadm_Etcd_To_v1beta2_Etcd(&in.Etcd, &out.Etcd, s); err != nil { - return err - } - if err := Convert_kubeadm_Networking_To_v1beta2_Networking(&in.Networking, &out.Networking, s); err != nil { - return err - } - out.KubernetesVersion = in.KubernetesVersion - out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - if err := Convert_kubeadm_APIServer_To_v1beta2_APIServer(&in.APIServer, &out.APIServer, s); err != nil { - return err - } - if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(&in.ControllerManager, &out.ControllerManager, s); err != nil { - return err - } - if err := Convert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(&in.Scheduler, &out.Scheduler, s); err != nil { - return err - } - if err := Convert_kubeadm_DNS_To_v1beta2_DNS(&in.DNS, &out.DNS, s); err != nil { - return err - } - out.CertificatesDir = in.CertificatesDir - out.ImageRepository = in.ImageRepository - // INFO: in.CIImageRepository opted out of conversion generation - out.UseHyperKubeImage = in.UseHyperKubeImage - out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) - out.ClusterName = in.ClusterName - return nil -} - -// Convert_kubeadm_ClusterConfiguration_To_v1beta2_ClusterConfiguration is an autogenerated conversion function. -func Convert_kubeadm_ClusterConfiguration_To_v1beta2_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_ClusterConfiguration_To_v1beta2_ClusterConfiguration(in, out, s) -} - -func autoConvert_v1beta2_ClusterStatus_To_kubeadm_ClusterStatus(in *ClusterStatus, out *kubeadm.ClusterStatus, s conversion.Scope) error { - out.APIEndpoints = *(*map[string]kubeadm.APIEndpoint)(unsafe.Pointer(&in.APIEndpoints)) - return nil -} - -// Convert_v1beta2_ClusterStatus_To_kubeadm_ClusterStatus is an autogenerated conversion function. -func Convert_v1beta2_ClusterStatus_To_kubeadm_ClusterStatus(in *ClusterStatus, out *kubeadm.ClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta2_ClusterStatus_To_kubeadm_ClusterStatus(in, out, s) -} - -func autoConvert_kubeadm_ClusterStatus_To_v1beta2_ClusterStatus(in *kubeadm.ClusterStatus, out *ClusterStatus, s conversion.Scope) error { - out.APIEndpoints = *(*map[string]APIEndpoint)(unsafe.Pointer(&in.APIEndpoints)) - return nil -} - -// Convert_kubeadm_ClusterStatus_To_v1beta2_ClusterStatus is an autogenerated conversion function. -func Convert_kubeadm_ClusterStatus_To_v1beta2_ClusterStatus(in *kubeadm.ClusterStatus, out *ClusterStatus, s conversion.Scope) error { - return autoConvert_kubeadm_ClusterStatus_To_v1beta2_ClusterStatus(in, out, s) -} - -func autoConvert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) - return nil -} - -// Convert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent is an autogenerated conversion function. -func Convert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in *ControlPlaneComponent, out *kubeadm.ControlPlaneComponent, s conversion.Scope) error { - return autoConvert_v1beta2_ControlPlaneComponent_To_kubeadm_ControlPlaneComponent(in, out, s) -} - -func autoConvert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ExtraVolumes)) - return nil -} - -// Convert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent is an autogenerated conversion function. -func Convert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(in *kubeadm.ControlPlaneComponent, out *ControlPlaneComponent, s conversion.Scope) error { - return autoConvert_kubeadm_ControlPlaneComponent_To_v1beta2_ControlPlaneComponent(in, out, s) -} - -func autoConvert_v1beta2_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { - out.Type = kubeadm.DNSAddOnType(in.Type) - if err := Convert_v1beta2_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - return nil -} - -// Convert_v1beta2_DNS_To_kubeadm_DNS is an autogenerated conversion function. -func Convert_v1beta2_DNS_To_kubeadm_DNS(in *DNS, out *kubeadm.DNS, s conversion.Scope) error { - return autoConvert_v1beta2_DNS_To_kubeadm_DNS(in, out, s) -} - -func autoConvert_kubeadm_DNS_To_v1beta2_DNS(in *kubeadm.DNS, out *DNS, s conversion.Scope) error { - out.Type = DNSAddOnType(in.Type) - if err := Convert_kubeadm_ImageMeta_To_v1beta2_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - return nil -} - -// Convert_kubeadm_DNS_To_v1beta2_DNS is an autogenerated conversion function. -func Convert_kubeadm_DNS_To_v1beta2_DNS(in *kubeadm.DNS, out *DNS, s conversion.Scope) error { - return autoConvert_kubeadm_DNS_To_v1beta2_DNS(in, out, s) -} - -func autoConvert_v1beta2_Discovery_To_kubeadm_Discovery(in *Discovery, out *kubeadm.Discovery, s conversion.Scope) error { - out.BootstrapToken = (*kubeadm.BootstrapTokenDiscovery)(unsafe.Pointer(in.BootstrapToken)) - out.File = (*kubeadm.FileDiscovery)(unsafe.Pointer(in.File)) - out.TLSBootstrapToken = in.TLSBootstrapToken - out.Timeout = (*v1.Duration)(unsafe.Pointer(in.Timeout)) - return nil -} - -// Convert_v1beta2_Discovery_To_kubeadm_Discovery is an autogenerated conversion function. -func Convert_v1beta2_Discovery_To_kubeadm_Discovery(in *Discovery, out *kubeadm.Discovery, s conversion.Scope) error { - return autoConvert_v1beta2_Discovery_To_kubeadm_Discovery(in, out, s) -} - -func autoConvert_kubeadm_Discovery_To_v1beta2_Discovery(in *kubeadm.Discovery, out *Discovery, s conversion.Scope) error { - out.BootstrapToken = (*BootstrapTokenDiscovery)(unsafe.Pointer(in.BootstrapToken)) - out.File = (*FileDiscovery)(unsafe.Pointer(in.File)) - out.TLSBootstrapToken = in.TLSBootstrapToken - out.Timeout = (*v1.Duration)(unsafe.Pointer(in.Timeout)) - return nil -} - -// Convert_kubeadm_Discovery_To_v1beta2_Discovery is an autogenerated conversion function. -func Convert_kubeadm_Discovery_To_v1beta2_Discovery(in *kubeadm.Discovery, out *Discovery, s conversion.Scope) error { - return autoConvert_kubeadm_Discovery_To_v1beta2_Discovery(in, out, s) -} - -func autoConvert_v1beta2_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { - out.Local = (*kubeadm.LocalEtcd)(unsafe.Pointer(in.Local)) - out.External = (*kubeadm.ExternalEtcd)(unsafe.Pointer(in.External)) - return nil -} - -// Convert_v1beta2_Etcd_To_kubeadm_Etcd is an autogenerated conversion function. -func Convert_v1beta2_Etcd_To_kubeadm_Etcd(in *Etcd, out *kubeadm.Etcd, s conversion.Scope) error { - return autoConvert_v1beta2_Etcd_To_kubeadm_Etcd(in, out, s) -} - -func autoConvert_kubeadm_Etcd_To_v1beta2_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { - out.Local = (*LocalEtcd)(unsafe.Pointer(in.Local)) - out.External = (*ExternalEtcd)(unsafe.Pointer(in.External)) - return nil -} - -// Convert_kubeadm_Etcd_To_v1beta2_Etcd is an autogenerated conversion function. -func Convert_kubeadm_Etcd_To_v1beta2_Etcd(in *kubeadm.Etcd, out *Etcd, s conversion.Scope) error { - return autoConvert_kubeadm_Etcd_To_v1beta2_Etcd(in, out, s) -} - -func autoConvert_v1beta2_ExternalEtcd_To_kubeadm_ExternalEtcd(in *ExternalEtcd, out *kubeadm.ExternalEtcd, s conversion.Scope) error { - out.Endpoints = *(*[]string)(unsafe.Pointer(&in.Endpoints)) - out.CAFile = in.CAFile - out.CertFile = in.CertFile - out.KeyFile = in.KeyFile - return nil -} - -// Convert_v1beta2_ExternalEtcd_To_kubeadm_ExternalEtcd is an autogenerated conversion function. -func Convert_v1beta2_ExternalEtcd_To_kubeadm_ExternalEtcd(in *ExternalEtcd, out *kubeadm.ExternalEtcd, s conversion.Scope) error { - return autoConvert_v1beta2_ExternalEtcd_To_kubeadm_ExternalEtcd(in, out, s) -} - -func autoConvert_kubeadm_ExternalEtcd_To_v1beta2_ExternalEtcd(in *kubeadm.ExternalEtcd, out *ExternalEtcd, s conversion.Scope) error { - out.Endpoints = *(*[]string)(unsafe.Pointer(&in.Endpoints)) - out.CAFile = in.CAFile - out.CertFile = in.CertFile - out.KeyFile = in.KeyFile - return nil -} - -// Convert_kubeadm_ExternalEtcd_To_v1beta2_ExternalEtcd is an autogenerated conversion function. -func Convert_kubeadm_ExternalEtcd_To_v1beta2_ExternalEtcd(in *kubeadm.ExternalEtcd, out *ExternalEtcd, s conversion.Scope) error { - return autoConvert_kubeadm_ExternalEtcd_To_v1beta2_ExternalEtcd(in, out, s) -} - -func autoConvert_v1beta2_FileDiscovery_To_kubeadm_FileDiscovery(in *FileDiscovery, out *kubeadm.FileDiscovery, s conversion.Scope) error { - out.KubeConfigPath = in.KubeConfigPath - return nil -} - -// Convert_v1beta2_FileDiscovery_To_kubeadm_FileDiscovery is an autogenerated conversion function. -func Convert_v1beta2_FileDiscovery_To_kubeadm_FileDiscovery(in *FileDiscovery, out *kubeadm.FileDiscovery, s conversion.Scope) error { - return autoConvert_v1beta2_FileDiscovery_To_kubeadm_FileDiscovery(in, out, s) -} - -func autoConvert_kubeadm_FileDiscovery_To_v1beta2_FileDiscovery(in *kubeadm.FileDiscovery, out *FileDiscovery, s conversion.Scope) error { - out.KubeConfigPath = in.KubeConfigPath - return nil -} - -// Convert_kubeadm_FileDiscovery_To_v1beta2_FileDiscovery is an autogenerated conversion function. -func Convert_kubeadm_FileDiscovery_To_v1beta2_FileDiscovery(in *kubeadm.FileDiscovery, out *FileDiscovery, s conversion.Scope) error { - return autoConvert_kubeadm_FileDiscovery_To_v1beta2_FileDiscovery(in, out, s) -} - -func autoConvert_v1beta2_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error { - out.Name = in.Name - out.HostPath = in.HostPath - out.MountPath = in.MountPath - out.ReadOnly = in.ReadOnly - out.PathType = corev1.HostPathType(in.PathType) - return nil -} - -// Convert_v1beta2_HostPathMount_To_kubeadm_HostPathMount is an autogenerated conversion function. -func Convert_v1beta2_HostPathMount_To_kubeadm_HostPathMount(in *HostPathMount, out *kubeadm.HostPathMount, s conversion.Scope) error { - return autoConvert_v1beta2_HostPathMount_To_kubeadm_HostPathMount(in, out, s) -} - -func autoConvert_kubeadm_HostPathMount_To_v1beta2_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { - out.Name = in.Name - out.HostPath = in.HostPath - out.MountPath = in.MountPath - out.ReadOnly = in.ReadOnly - out.PathType = corev1.HostPathType(in.PathType) - return nil -} - -// Convert_kubeadm_HostPathMount_To_v1beta2_HostPathMount is an autogenerated conversion function. -func Convert_kubeadm_HostPathMount_To_v1beta2_HostPathMount(in *kubeadm.HostPathMount, out *HostPathMount, s conversion.Scope) error { - return autoConvert_kubeadm_HostPathMount_To_v1beta2_HostPathMount(in, out, s) -} - -func autoConvert_v1beta2_ImageMeta_To_kubeadm_ImageMeta(in *ImageMeta, out *kubeadm.ImageMeta, s conversion.Scope) error { - out.ImageRepository = in.ImageRepository - out.ImageTag = in.ImageTag - return nil -} - -// Convert_v1beta2_ImageMeta_To_kubeadm_ImageMeta is an autogenerated conversion function. -func Convert_v1beta2_ImageMeta_To_kubeadm_ImageMeta(in *ImageMeta, out *kubeadm.ImageMeta, s conversion.Scope) error { - return autoConvert_v1beta2_ImageMeta_To_kubeadm_ImageMeta(in, out, s) -} - -func autoConvert_kubeadm_ImageMeta_To_v1beta2_ImageMeta(in *kubeadm.ImageMeta, out *ImageMeta, s conversion.Scope) error { - out.ImageRepository = in.ImageRepository - out.ImageTag = in.ImageTag - return nil -} - -// Convert_kubeadm_ImageMeta_To_v1beta2_ImageMeta is an autogenerated conversion function. -func Convert_kubeadm_ImageMeta_To_v1beta2_ImageMeta(in *kubeadm.ImageMeta, out *ImageMeta, s conversion.Scope) error { - return autoConvert_kubeadm_ImageMeta_To_v1beta2_ImageMeta(in, out, s) -} - -func autoConvert_v1beta2_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error { - out.BootstrapTokens = *(*[]kubeadm.BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) - if err := Convert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - if err := Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - out.CertificateKey = in.CertificateKey - return nil -} - -func autoConvert_kubeadm_InitConfiguration_To_v1beta2_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error { - // WARNING: in.ClusterConfiguration requires manual conversion: does not exist in peer-type - out.BootstrapTokens = *(*[]BootstrapToken)(unsafe.Pointer(&in.BootstrapTokens)) - if err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - if err := Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - out.CertificateKey = in.CertificateKey - return nil -} - -func autoConvert_v1beta2_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { - if err := Convert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - out.CACertPath = in.CACertPath - if err := Convert_v1beta2_Discovery_To_kubeadm_Discovery(&in.Discovery, &out.Discovery, s); err != nil { - return err - } - out.ControlPlane = (*kubeadm.JoinControlPlane)(unsafe.Pointer(in.ControlPlane)) - return nil -} - -// Convert_v1beta2_JoinConfiguration_To_kubeadm_JoinConfiguration is an autogenerated conversion function. -func Convert_v1beta2_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { - return autoConvert_v1beta2_JoinConfiguration_To_kubeadm_JoinConfiguration(in, out, s) -} - -func autoConvert_kubeadm_JoinConfiguration_To_v1beta2_JoinConfiguration(in *kubeadm.JoinConfiguration, out *JoinConfiguration, s conversion.Scope) error { - if err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions(&in.NodeRegistration, &out.NodeRegistration, s); err != nil { - return err - } - out.CACertPath = in.CACertPath - if err := Convert_kubeadm_Discovery_To_v1beta2_Discovery(&in.Discovery, &out.Discovery, s); err != nil { - return err - } - out.ControlPlane = (*JoinControlPlane)(unsafe.Pointer(in.ControlPlane)) - return nil -} - -// Convert_kubeadm_JoinConfiguration_To_v1beta2_JoinConfiguration is an autogenerated conversion function. -func Convert_kubeadm_JoinConfiguration_To_v1beta2_JoinConfiguration(in *kubeadm.JoinConfiguration, out *JoinConfiguration, s conversion.Scope) error { - return autoConvert_kubeadm_JoinConfiguration_To_v1beta2_JoinConfiguration(in, out, s) -} - -func autoConvert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinControlPlane, out *kubeadm.JoinControlPlane, s conversion.Scope) error { - if err := Convert_v1beta2_APIEndpoint_To_kubeadm_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - out.CertificateKey = in.CertificateKey - return nil -} - -// Convert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane is an autogenerated conversion function. -func Convert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane(in *JoinControlPlane, out *kubeadm.JoinControlPlane, s conversion.Scope) error { - return autoConvert_v1beta2_JoinControlPlane_To_kubeadm_JoinControlPlane(in, out, s) -} - -func autoConvert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error { - if err := Convert_kubeadm_APIEndpoint_To_v1beta2_APIEndpoint(&in.LocalAPIEndpoint, &out.LocalAPIEndpoint, s); err != nil { - return err - } - out.CertificateKey = in.CertificateKey - return nil -} - -// Convert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane is an autogenerated conversion function. -func Convert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane(in *kubeadm.JoinControlPlane, out *JoinControlPlane, s conversion.Scope) error { - return autoConvert_kubeadm_JoinControlPlane_To_v1beta2_JoinControlPlane(in, out, s) -} - -func autoConvert_v1beta2_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - if err := Convert_v1beta2_ImageMeta_To_kubeadm_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) - out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) - return nil -} - -// Convert_v1beta2_LocalEtcd_To_kubeadm_LocalEtcd is an autogenerated conversion function. -func Convert_v1beta2_LocalEtcd_To_kubeadm_LocalEtcd(in *LocalEtcd, out *kubeadm.LocalEtcd, s conversion.Scope) error { - return autoConvert_v1beta2_LocalEtcd_To_kubeadm_LocalEtcd(in, out, s) -} - -func autoConvert_kubeadm_LocalEtcd_To_v1beta2_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - if err := Convert_kubeadm_ImageMeta_To_v1beta2_ImageMeta(&in.ImageMeta, &out.ImageMeta, s); err != nil { - return err - } - out.DataDir = in.DataDir - out.ExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ExtraArgs)) - out.ServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ServerCertSANs)) - out.PeerCertSANs = *(*[]string)(unsafe.Pointer(&in.PeerCertSANs)) - return nil -} - -// Convert_kubeadm_LocalEtcd_To_v1beta2_LocalEtcd is an autogenerated conversion function. -func Convert_kubeadm_LocalEtcd_To_v1beta2_LocalEtcd(in *kubeadm.LocalEtcd, out *LocalEtcd, s conversion.Scope) error { - return autoConvert_kubeadm_LocalEtcd_To_v1beta2_LocalEtcd(in, out, s) -} - -func autoConvert_v1beta2_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { - out.ServiceSubnet = in.ServiceSubnet - out.PodSubnet = in.PodSubnet - out.DNSDomain = in.DNSDomain - return nil -} - -// Convert_v1beta2_Networking_To_kubeadm_Networking is an autogenerated conversion function. -func Convert_v1beta2_Networking_To_kubeadm_Networking(in *Networking, out *kubeadm.Networking, s conversion.Scope) error { - return autoConvert_v1beta2_Networking_To_kubeadm_Networking(in, out, s) -} - -func autoConvert_kubeadm_Networking_To_v1beta2_Networking(in *kubeadm.Networking, out *Networking, s conversion.Scope) error { - out.ServiceSubnet = in.ServiceSubnet - out.PodSubnet = in.PodSubnet - out.DNSDomain = in.DNSDomain - return nil -} - -// Convert_kubeadm_Networking_To_v1beta2_Networking is an autogenerated conversion function. -func Convert_kubeadm_Networking_To_v1beta2_Networking(in *kubeadm.Networking, out *Networking, s conversion.Scope) error { - return autoConvert_kubeadm_Networking_To_v1beta2_Networking(in, out, s) -} - -func autoConvert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in *NodeRegistrationOptions, out *kubeadm.NodeRegistrationOptions, s conversion.Scope) error { - out.Name = in.Name - out.CRISocket = in.CRISocket - out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) - out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) - return nil -} - -// Convert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions is an autogenerated conversion function. -func Convert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in *NodeRegistrationOptions, out *kubeadm.NodeRegistrationOptions, s conversion.Scope) error { - return autoConvert_v1beta2_NodeRegistrationOptions_To_kubeadm_NodeRegistrationOptions(in, out, s) -} - -func autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { - out.Name = in.Name - out.CRISocket = in.CRISocket - out.Taints = *(*[]corev1.Taint)(unsafe.Pointer(&in.Taints)) - out.KubeletExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.KubeletExtraArgs)) - out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors)) - return nil -} - -// Convert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions is an autogenerated conversion function. -func Convert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error { - return autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta2_NodeRegistrationOptions(in, out, s) -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go deleted file mode 100644 index 399923a6164d7..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.deepcopy.go +++ /dev/null @@ -1,556 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package v1beta2 - -import ( - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIEndpoint) DeepCopyInto(out *APIEndpoint) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIEndpoint. -func (in *APIEndpoint) DeepCopy() *APIEndpoint { - if in == nil { - return nil - } - out := new(APIEndpoint) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIServer) DeepCopyInto(out *APIServer) { - *out = *in - in.ControlPlaneComponent.DeepCopyInto(&out.ControlPlaneComponent) - if in.CertSANs != nil { - in, out := &in.CertSANs, &out.CertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TimeoutForControlPlane != nil { - in, out := &in.TimeoutForControlPlane, &out.TimeoutForControlPlane - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServer. -func (in *APIServer) DeepCopy() *APIServer { - if in == nil { - return nil - } - out := new(APIServer) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { - *out = *in - if in.Token != nil { - in, out := &in.Token, &out.Token - *out = new(BootstrapTokenString) - **out = **in - } - if in.TTL != nil { - in, out := &in.TTL, &out.TTL - *out = new(v1.Duration) - **out = **in - } - if in.Expires != nil { - in, out := &in.Expires, &out.Expires - *out = (*in).DeepCopy() - } - if in.Usages != nil { - in, out := &in.Usages, &out.Usages - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Groups != nil { - in, out := &in.Groups, &out.Groups - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. -func (in *BootstrapToken) DeepCopy() *BootstrapToken { - if in == nil { - return nil - } - out := new(BootstrapToken) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapTokenDiscovery) DeepCopyInto(out *BootstrapTokenDiscovery) { - *out = *in - if in.CACertHashes != nil { - in, out := &in.CACertHashes, &out.CACertHashes - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapTokenDiscovery. -func (in *BootstrapTokenDiscovery) DeepCopy() *BootstrapTokenDiscovery { - if in == nil { - return nil - } - out := new(BootstrapTokenDiscovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapTokenString) DeepCopyInto(out *BootstrapTokenString) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapTokenString. -func (in *BootstrapTokenString) DeepCopy() *BootstrapTokenString { - if in == nil { - return nil - } - out := new(BootstrapTokenString) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.Etcd.DeepCopyInto(&out.Etcd) - out.Networking = in.Networking - in.APIServer.DeepCopyInto(&out.APIServer) - in.ControllerManager.DeepCopyInto(&out.ControllerManager) - in.Scheduler.DeepCopyInto(&out.Scheduler) - out.DNS = in.DNS - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterConfiguration. -func (in *ClusterConfiguration) DeepCopy() *ClusterConfiguration { - if in == nil { - return nil - } - out := new(ClusterConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.APIEndpoints != nil { - in, out := &in.APIEndpoints, &out.APIEndpoints - *out = make(map[string]APIEndpoint, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. -func (in *ClusterStatus) DeepCopy() *ClusterStatus { - if in == nil { - return nil - } - out := new(ClusterStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterStatus) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { - *out = *in - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ExtraVolumes != nil { - in, out := &in.ExtraVolumes, &out.ExtraVolumes - *out = make([]HostPathMount, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponent. -func (in *ControlPlaneComponent) DeepCopy() *ControlPlaneComponent { - if in == nil { - return nil - } - out := new(ControlPlaneComponent) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DNS) DeepCopyInto(out *DNS) { - *out = *in - out.ImageMeta = in.ImageMeta - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNS. -func (in *DNS) DeepCopy() *DNS { - if in == nil { - return nil - } - out := new(DNS) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Discovery) DeepCopyInto(out *Discovery) { - *out = *in - if in.BootstrapToken != nil { - in, out := &in.BootstrapToken, &out.BootstrapToken - *out = new(BootstrapTokenDiscovery) - (*in).DeepCopyInto(*out) - } - if in.File != nil { - in, out := &in.File, &out.File - *out = new(FileDiscovery) - **out = **in - } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Discovery. -func (in *Discovery) DeepCopy() *Discovery { - if in == nil { - return nil - } - out := new(Discovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Etcd) DeepCopyInto(out *Etcd) { - *out = *in - if in.Local != nil { - in, out := &in.Local, &out.Local - *out = new(LocalEtcd) - (*in).DeepCopyInto(*out) - } - if in.External != nil { - in, out := &in.External, &out.External - *out = new(ExternalEtcd) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Etcd. -func (in *Etcd) DeepCopy() *Etcd { - if in == nil { - return nil - } - out := new(Etcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExternalEtcd) DeepCopyInto(out *ExternalEtcd) { - *out = *in - if in.Endpoints != nil { - in, out := &in.Endpoints, &out.Endpoints - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalEtcd. -func (in *ExternalEtcd) DeepCopy() *ExternalEtcd { - if in == nil { - return nil - } - out := new(ExternalEtcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileDiscovery) DeepCopyInto(out *FileDiscovery) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileDiscovery. -func (in *FileDiscovery) DeepCopy() *FileDiscovery { - if in == nil { - return nil - } - out := new(FileDiscovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HostPathMount) DeepCopyInto(out *HostPathMount) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMount. -func (in *HostPathMount) DeepCopy() *HostPathMount { - if in == nil { - return nil - } - out := new(HostPathMount) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageMeta) DeepCopyInto(out *ImageMeta) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageMeta. -func (in *ImageMeta) DeepCopy() *ImageMeta { - if in == nil { - return nil - } - out := new(ImageMeta) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.BootstrapTokens != nil { - in, out := &in.BootstrapTokens, &out.BootstrapTokens - *out = make([]BootstrapToken, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - out.LocalAPIEndpoint = in.LocalAPIEndpoint - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitConfiguration. -func (in *InitConfiguration) DeepCopy() *InitConfiguration { - if in == nil { - return nil - } - out := new(InitConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *InitConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JoinConfiguration) DeepCopyInto(out *JoinConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - in.Discovery.DeepCopyInto(&out.Discovery) - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(JoinControlPlane) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinConfiguration. -func (in *JoinConfiguration) DeepCopy() *JoinConfiguration { - if in == nil { - return nil - } - out := new(JoinConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *JoinConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JoinControlPlane) DeepCopyInto(out *JoinControlPlane) { - *out = *in - out.LocalAPIEndpoint = in.LocalAPIEndpoint - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinControlPlane. -func (in *JoinControlPlane) DeepCopy() *JoinControlPlane { - if in == nil { - return nil - } - out := new(JoinControlPlane) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { - *out = *in - out.ImageMeta = in.ImageMeta - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ServerCertSANs != nil { - in, out := &in.ServerCertSANs, &out.ServerCertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.PeerCertSANs != nil { - in, out := &in.PeerCertSANs, &out.PeerCertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalEtcd. -func (in *LocalEtcd) DeepCopy() *LocalEtcd { - if in == nil { - return nil - } - out := new(LocalEtcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Networking) DeepCopyInto(out *Networking) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Networking. -func (in *Networking) DeepCopy() *Networking { - if in == nil { - return nil - } - out := new(Networking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { - *out = *in - if in.Taints != nil { - in, out := &in.Taints, &out.Taints - *out = make([]corev1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.KubeletExtraArgs != nil { - in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.IgnorePreflightErrors != nil { - in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeRegistrationOptions. -func (in *NodeRegistrationOptions) DeepCopy() *NodeRegistrationOptions { - if in == nil { - return nil - } - out := new(NodeRegistrationOptions) - in.DeepCopyInto(out) - return out -} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.defaults.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.defaults.go deleted file mode 100644 index 5267e05f6e269..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/zz_generated.defaults.go +++ /dev/null @@ -1,65 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by defaulter-gen. DO NOT EDIT. - -package v1beta2 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// RegisterDefaults adds defaulters functions to the given scheme. -// Public to allow building arbitrary schemes. -// All generated defaulters are covering - they call all nested defaulters. -func RegisterDefaults(scheme *runtime.Scheme) error { - scheme.AddTypeDefaultingFunc(&ClusterConfiguration{}, func(obj interface{}) { SetObjectDefaults_ClusterConfiguration(obj.(*ClusterConfiguration)) }) - scheme.AddTypeDefaultingFunc(&ClusterStatus{}, func(obj interface{}) { SetObjectDefaults_ClusterStatus(obj.(*ClusterStatus)) }) - scheme.AddTypeDefaultingFunc(&InitConfiguration{}, func(obj interface{}) { SetObjectDefaults_InitConfiguration(obj.(*InitConfiguration)) }) - scheme.AddTypeDefaultingFunc(&JoinConfiguration{}, func(obj interface{}) { SetObjectDefaults_JoinConfiguration(obj.(*JoinConfiguration)) }) - return nil -} - -func SetObjectDefaults_ClusterConfiguration(in *ClusterConfiguration) { - SetDefaults_ClusterConfiguration(in) - SetDefaults_APIServer(&in.APIServer) -} - -func SetObjectDefaults_ClusterStatus(in *ClusterStatus) { -} - -func SetObjectDefaults_InitConfiguration(in *InitConfiguration) { - SetDefaults_InitConfiguration(in) - for i := range in.BootstrapTokens { - a := &in.BootstrapTokens[i] - SetDefaults_BootstrapToken(a) - } - SetDefaults_APIEndpoint(&in.LocalAPIEndpoint) -} - -func SetObjectDefaults_JoinConfiguration(in *JoinConfiguration) { - SetDefaults_JoinConfiguration(in) - SetDefaults_Discovery(&in.Discovery) - if in.Discovery.File != nil { - SetDefaults_FileDiscovery(in.Discovery.File) - } - if in.ControlPlane != nil { - SetDefaults_JoinControlPlane(in.ControlPlane) - SetDefaults_APIEndpoint(&in.ControlPlane.LocalAPIEndpoint) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD b/cmd/kubeadm/app/apis/kubeadm/validation/BUILD deleted file mode 100644 index 671c45447669a..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/validation/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["validation.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["validation_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation.go deleted file mode 100644 index ac3bb84bbfce0..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation.go +++ /dev/null @@ -1,627 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validation - -import ( - "fmt" - "net" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/apimachinery/pkg/util/validation/field" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmcmdoptions "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - utilnet "k8s.io/utils/net" -) - -// ValidateInitConfiguration validates an InitConfiguration object and collects all encountered errors -func ValidateInitConfiguration(c *kubeadm.InitConfiguration) field.ErrorList { - allErrs := field.ErrorList{} - allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...) - allErrs = append(allErrs, ValidateBootstrapTokens(c.BootstrapTokens, field.NewPath("bootstrapTokens"))...) - allErrs = append(allErrs, ValidateClusterConfiguration(&c.ClusterConfiguration)...) - // TODO(Arvinderpal): update advertiseAddress validation for dual-stack once it's implemented. - allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, field.NewPath("localAPIEndpoint"))...) - // TODO: Maybe validate that .CertificateKey is a valid hex encoded AES key - return allErrs -} - -// ValidateClusterConfiguration validates an ClusterConfiguration object and collects all encountered errors -func ValidateClusterConfiguration(c *kubeadm.ClusterConfiguration) field.ErrorList { - allErrs := field.ErrorList{} - allErrs = append(allErrs, ValidateNetworking(c, field.NewPath("networking"))...) - allErrs = append(allErrs, ValidateAPIServer(&c.APIServer, field.NewPath("apiServer"))...) - allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificatesDir"))...) - allErrs = append(allErrs, ValidateFeatureGates(c.FeatureGates, field.NewPath("featureGates"))...) - allErrs = append(allErrs, ValidateHostPort(c.ControlPlaneEndpoint, field.NewPath("controlPlaneEndpoint"))...) - allErrs = append(allErrs, ValidateEtcd(&c.Etcd, field.NewPath("etcd"))...) - allErrs = append(allErrs, componentconfigs.Validate(c)...) - return allErrs -} - -// ValidateAPIServer validates a APIServer object and collects all encountered errors -func ValidateAPIServer(a *kubeadm.APIServer, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - allErrs = append(allErrs, ValidateCertSANs(a.CertSANs, fldPath.Child("certSANs"))...) - return allErrs -} - -// ValidateJoinConfiguration validates node configuration and collects all encountered errors -func ValidateJoinConfiguration(c *kubeadm.JoinConfiguration) field.ErrorList { - allErrs := field.ErrorList{} - allErrs = append(allErrs, ValidateDiscovery(&c.Discovery, field.NewPath("discovery"))...) - allErrs = append(allErrs, ValidateNodeRegistrationOptions(&c.NodeRegistration, field.NewPath("nodeRegistration"))...) - allErrs = append(allErrs, ValidateJoinControlPlane(c.ControlPlane, field.NewPath("controlPlane"))...) - - if !filepath.IsAbs(c.CACertPath) || !strings.HasSuffix(c.CACertPath, ".crt") { - allErrs = append(allErrs, field.Invalid(field.NewPath("caCertPath"), c.CACertPath, "the ca certificate path must be an absolute path")) - } - return allErrs -} - -// ValidateJoinControlPlane validates joining control plane configuration and collects all encountered errors -func ValidateJoinControlPlane(c *kubeadm.JoinControlPlane, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if c != nil { - allErrs = append(allErrs, ValidateAPIEndpoint(&c.LocalAPIEndpoint, fldPath.Child("localAPIEndpoint"))...) - // TODO: Maybe validate that .CertificateKey is a valid hex encoded AES key - } - return allErrs -} - -// ValidateNodeRegistrationOptions validates the NodeRegistrationOptions object -func ValidateNodeRegistrationOptions(nro *kubeadm.NodeRegistrationOptions, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if len(nro.Name) == 0 { - allErrs = append(allErrs, field.Required(fldPath, "--node-name or .nodeRegistration.name in the config file is a required value. It seems like this value couldn't be automatically detected in your environment, please specify the desired value using the CLI or config file.")) - } else { - nameFldPath := fldPath.Child("name") - for _, err := range validation.IsDNS1123Subdomain(nro.Name) { - allErrs = append(allErrs, field.Invalid(nameFldPath, nro.Name, err)) - } - } - allErrs = append(allErrs, ValidateSocketPath(nro.CRISocket, fldPath.Child("criSocket"))...) - // TODO: Maybe validate .Taints as well in the future using something like validateNodeTaints() in pkg/apis/core/validation - return allErrs -} - -// ValidateDiscovery validates discovery related configuration and collects all encountered errors -func ValidateDiscovery(d *kubeadm.Discovery, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if d.BootstrapToken == nil && d.File == nil { - allErrs = append(allErrs, field.Invalid(fldPath, "", "bootstrapToken or file must be set")) - } - - if d.BootstrapToken != nil && d.File != nil { - allErrs = append(allErrs, field.Invalid(fldPath, "", "bootstrapToken and file cannot both be set")) - } - - if d.BootstrapToken != nil { - allErrs = append(allErrs, ValidateDiscoveryBootstrapToken(d.BootstrapToken, fldPath.Child("bootstrapToken"))...) - allErrs = append(allErrs, ValidateToken(d.TLSBootstrapToken, fldPath.Child("tlsBootstrapToken"))...) - } - - if d.File != nil { - allErrs = append(allErrs, ValidateDiscoveryFile(d.File, fldPath.Child("file"))...) - if len(d.TLSBootstrapToken) != 0 { - allErrs = append(allErrs, ValidateToken(d.TLSBootstrapToken, fldPath.Child("tlsBootstrapToken"))...) - } - } - - return allErrs -} - -// ValidateDiscoveryBootstrapToken validates bootstrap token discovery configuration -func ValidateDiscoveryBootstrapToken(b *kubeadm.BootstrapTokenDiscovery, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if len(b.APIServerEndpoint) == 0 { - allErrs = append(allErrs, field.Required(fldPath, "APIServerEndpoint is not set")) - } - - if len(b.CACertHashes) == 0 && !b.UnsafeSkipCAVerification { - allErrs = append(allErrs, field.Invalid(fldPath, "", "using token-based discovery without caCertHashes can be unsafe. Set unsafeSkipCAVerification as true in your kubeadm config file or pass --discovery-token-unsafe-skip-ca-verification flag to continue")) - } - - allErrs = append(allErrs, ValidateToken(b.Token, fldPath.Child(kubeadmcmdoptions.TokenStr))...) - allErrs = append(allErrs, ValidateDiscoveryTokenAPIServer(b.APIServerEndpoint, fldPath.Child("apiServerEndpoint"))...) - - return allErrs -} - -// ValidateDiscoveryFile validates file discovery configuration -func ValidateDiscoveryFile(f *kubeadm.FileDiscovery, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - allErrs = append(allErrs, ValidateDiscoveryKubeConfigPath(f.KubeConfigPath, fldPath.Child("kubeConfigPath"))...) - - return allErrs -} - -// ValidateDiscoveryTokenAPIServer validates discovery token for API server -func ValidateDiscoveryTokenAPIServer(apiServer string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - _, _, err := net.SplitHostPort(apiServer) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, apiServer, err.Error())) - } - return allErrs -} - -// ValidateDiscoveryKubeConfigPath validates location of a discovery file -func ValidateDiscoveryKubeConfigPath(discoveryFile string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - u, err := url.Parse(discoveryFile) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, discoveryFile, "not a valid HTTPS URL or a file on disk")) - return allErrs - } - - if u.Scheme == "" { - // URIs with no scheme should be treated as files - if _, err := os.Stat(discoveryFile); os.IsNotExist(err) { - allErrs = append(allErrs, field.Invalid(fldPath, discoveryFile, "not a valid HTTPS URL or a file on disk")) - } - return allErrs - } - - if u.Scheme != "https" { - allErrs = append(allErrs, field.Invalid(fldPath, discoveryFile, "if a URL is used, the scheme must be https")) - } - return allErrs -} - -// ValidateBootstrapTokens validates a slice of BootstrapToken objects -func ValidateBootstrapTokens(bts []kubeadm.BootstrapToken, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - for i, bt := range bts { - btPath := fldPath.Child(fmt.Sprintf("%d", i)) - allErrs = append(allErrs, ValidateToken(bt.Token.String(), btPath.Child(kubeadmcmdoptions.TokenStr))...) - allErrs = append(allErrs, ValidateTokenUsages(bt.Usages, btPath.Child(kubeadmcmdoptions.TokenUsages))...) - allErrs = append(allErrs, ValidateTokenGroups(bt.Usages, bt.Groups, btPath.Child(kubeadmcmdoptions.TokenGroups))...) - - if bt.Expires != nil && bt.TTL != nil { - allErrs = append(allErrs, field.Invalid(btPath, "", "the BootstrapToken .TTL and .Expires fields are mutually exclusive")) - } - } - return allErrs -} - -// ValidateToken validates a Bootstrap Token -func ValidateToken(token string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - if !bootstraputil.IsValidBootstrapToken(token) { - allErrs = append(allErrs, field.Invalid(fldPath, token, "the bootstrap token is invalid")) - } - - return allErrs -} - -// ValidateTokenGroups validates token groups -func ValidateTokenGroups(usages []string, groups []string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - // adding groups only makes sense for authentication - usagesSet := sets.NewString(usages...) - usageAuthentication := strings.TrimPrefix(bootstrapapi.BootstrapTokenUsageAuthentication, bootstrapapi.BootstrapTokenUsagePrefix) - if len(groups) > 0 && !usagesSet.Has(usageAuthentication) { - allErrs = append(allErrs, field.Invalid(fldPath, groups, fmt.Sprintf("token groups cannot be specified unless --usages includes %q", usageAuthentication))) - } - - // validate any extra group names - for _, group := range groups { - if err := bootstraputil.ValidateBootstrapGroupName(group); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, groups, err.Error())) - } - } - - return allErrs -} - -// ValidateTokenUsages validates token usages -func ValidateTokenUsages(usages []string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - // validate usages - if err := bootstraputil.ValidateUsages(usages); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, usages, err.Error())) - } - - return allErrs -} - -// ValidateEtcd validates the .Etcd sub-struct. -func ValidateEtcd(e *kubeadm.Etcd, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - localPath := fldPath.Child("local") - externalPath := fldPath.Child("external") - - if e.Local == nil && e.External == nil { - allErrs = append(allErrs, field.Invalid(fldPath, "", "either .Etcd.Local or .Etcd.External is required")) - return allErrs - } - if e.Local != nil && e.External != nil { - allErrs = append(allErrs, field.Invalid(fldPath, "", ".Etcd.Local and .Etcd.External are mutually exclusive")) - return allErrs - } - if e.Local != nil { - allErrs = append(allErrs, ValidateAbsolutePath(e.Local.DataDir, localPath.Child("dataDir"))...) - allErrs = append(allErrs, ValidateCertSANs(e.Local.ServerCertSANs, localPath.Child("serverCertSANs"))...) - allErrs = append(allErrs, ValidateCertSANs(e.Local.PeerCertSANs, localPath.Child("peerCertSANs"))...) - } - if e.External != nil { - requireHTTPS := true - // Only allow the http scheme if no certs/keys are passed - if e.External.CAFile == "" && e.External.CertFile == "" && e.External.KeyFile == "" { - requireHTTPS = false - } - // Require either none or both of the cert/key pair - if (e.External.CertFile == "" && e.External.KeyFile != "") || (e.External.CertFile != "" && e.External.KeyFile == "") { - allErrs = append(allErrs, field.Invalid(externalPath, "", "either both or none of .Etcd.External.CertFile and .Etcd.External.KeyFile must be set")) - } - // If the cert and key are specified, require the CA as well - if e.External.CertFile != "" && e.External.KeyFile != "" && e.External.CAFile == "" { - allErrs = append(allErrs, field.Invalid(externalPath, "", "setting .Etcd.External.CertFile and .Etcd.External.KeyFile requires .Etcd.External.CAFile")) - } - - allErrs = append(allErrs, ValidateURLs(e.External.Endpoints, requireHTTPS, externalPath.Child("endpoints"))...) - if e.External.CAFile != "" { - allErrs = append(allErrs, ValidateAbsolutePath(e.External.CAFile, externalPath.Child("caFile"))...) - } - if e.External.CertFile != "" { - allErrs = append(allErrs, ValidateAbsolutePath(e.External.CertFile, externalPath.Child("certFile"))...) - } - if e.External.KeyFile != "" { - allErrs = append(allErrs, ValidateAbsolutePath(e.External.KeyFile, externalPath.Child("keyFile"))...) - } - } - return allErrs -} - -// ValidateCertSANs validates alternative names -func ValidateCertSANs(altnames []string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - for _, altname := range altnames { - if errs := validation.IsDNS1123Subdomain(altname); len(errs) != 0 { - if errs2 := validation.IsWildcardDNS1123Subdomain(altname); len(errs2) != 0 { - if net.ParseIP(altname) == nil { - allErrs = append(allErrs, field.Invalid(fldPath, altname, fmt.Sprintf("altname is not a valid IP address, DNS label or a DNS label with subdomain wildcards: %s; %s", strings.Join(errs, "; "), strings.Join(errs2, "; ")))) - } - } - } - } - return allErrs -} - -// ValidateURLs validates the URLs given in the string slice, makes sure they are parsable. Optionally, it can enforces HTTPS usage. -func ValidateURLs(urls []string, requireHTTPS bool, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - for _, urlstr := range urls { - u, err := url.Parse(urlstr) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, urlstr, fmt.Sprintf("URL parse error: %v", err))) - continue - } - if requireHTTPS && u.Scheme != "https" { - allErrs = append(allErrs, field.Invalid(fldPath, urlstr, "the URL must be using the HTTPS scheme")) - } - if u.Scheme == "" { - allErrs = append(allErrs, field.Invalid(fldPath, urlstr, "the URL without scheme is not allowed")) - } - } - return allErrs -} - -// ValidateIPFromString validates ip address -func ValidateIPFromString(ipaddr string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if net.ParseIP(ipaddr) == nil { - allErrs = append(allErrs, field.Invalid(fldPath, ipaddr, "ip address is not valid")) - } - return allErrs -} - -// ValidatePort validates port numbers -func ValidatePort(port int32, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if _, err := kubeadmutil.ParsePort(strconv.Itoa(int(port))); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, port, "port number is not valid")) - } - return allErrs -} - -// ValidateHostPort validates host[:port] endpoints -func ValidateHostPort(endpoint string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if _, _, err := kubeadmutil.ParseHostPort(endpoint); endpoint != "" && err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, endpoint, "endpoint is not valid")) - } - return allErrs -} - -// ValidateIPNetFromString validates network portion of ip address -func ValidateIPNetFromString(subnetStr string, minAddrs int64, isDualStack bool, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - subnets, err := utilnet.ParseCIDRs(strings.Split(subnetStr, ",")) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "couldn't parse subnet")) - return allErrs - } - switch { - // if DualStack only 2 CIDRs allowed - case isDualStack && len(subnets) > 2: - allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking")) - // if DualStack and there are 2 CIDRs validate if there is at least one of each IP family - case isDualStack && len(subnets) == 2: - areDualStackCIDRs, err := utilnet.IsDualStackCIDRs(subnets) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, err.Error())) - } else if !areDualStackCIDRs { - allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking")) - } - // if not DualStack only one CIDR allowed - case !isDualStack && len(subnets) > 1: - allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "only one CIDR allowed for single-stack networking")) - } - // validate the subnet/s - for _, s := range subnets { - numAddresses := utilnet.RangeSize(s) - if numAddresses < minAddrs { - allErrs = append(allErrs, field.Invalid(fldPath, s.String(), "subnet is too small")) - } - } - return allErrs -} - -// ValidateServiceSubnetSize validates that the maximum subnet size is not exceeded -// Should be a small cidr due to how it is stored in etcd. -// bigger cidr (specially those offered by IPv6) will add no value -// and significantly increase snapshotting time. -// NOTE: This is identical to validation performed in the apiserver. -func ValidateServiceSubnetSize(subnetStr string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - // subnets were already validated - subnets, _ := utilnet.ParseCIDRs(strings.Split(subnetStr, ",")) - for _, serviceSubnet := range subnets { - ones, bits := serviceSubnet.Mask.Size() - if bits-ones > constants.MaximumBitsForServiceSubnet { - errMsg := fmt.Sprintf("specified service subnet is too large; for %d-bit addresses, the mask must be >= %d", bits, bits-constants.MaximumBitsForServiceSubnet) - allErrs = append(allErrs, field.Invalid(fldPath, serviceSubnet.String(), errMsg)) - } - } - return allErrs -} - -// ValidatePodSubnetNodeMask validates that the relation between podSubnet and node-masks is correct -func ValidatePodSubnetNodeMask(subnetStr string, c *kubeadm.ClusterConfiguration, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - // subnets were already validated - subnets, _ := utilnet.ParseCIDRs(strings.Split(subnetStr, ",")) - for _, podSubnet := range subnets { - // obtain podSubnet mask - mask := podSubnet.Mask - maskSize, _ := mask.Size() - // obtain node-cidr-mask - nodeMask, err := getClusterNodeMask(c, utilnet.IsIPv6(podSubnet.IP)) - if err != nil { - allErrs = append(allErrs, field.Invalid(fldPath, podSubnet.String(), err.Error())) - continue - } - // the pod subnet mask needs to allow one or multiple node-masks - // i.e. if it has a /24 the node mask must be between 24 and 32 for ipv4 - if maskSize > nodeMask { - allErrs = append(allErrs, field.Invalid(fldPath, podSubnet.String(), "pod subnet size is smaller than the node subnet size")) - } else if (nodeMask - maskSize) > constants.PodSubnetNodeMaskMaxDiff { - allErrs = append(allErrs, field.Invalid(fldPath, podSubnet.String(), fmt.Sprintf("pod subnet mask and node-mask difference can not be greater than %d", constants.PodSubnetNodeMaskMaxDiff))) - } - } - return allErrs -} - -// getClusterNodeMask returns the corresponding node-cidr-mask -// based on the Cluster configuration and the IP family -// Default is 24 for IPv4 and 64 for IPv6 -func getClusterNodeMask(c *kubeadm.ClusterConfiguration, isIPv6 bool) (int, error) { - // defaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr for use by the controller manager - const defaultNodeMaskCIDRIPv4 = 24 - // DefaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr for use by the controller manager - const defaultNodeMaskCIDRIPv6 = 64 - var maskSize int - var maskArg string - var err error - isDualStack := features.Enabled(c.FeatureGates, features.IPv6DualStack) - - if isDualStack && isIPv6 { - maskArg = "node-cidr-mask-size-ipv6" - } else if isDualStack && !isIPv6 { - maskArg = "node-cidr-mask-size-ipv4" - } else { - maskArg = "node-cidr-mask-size" - } - - if v, ok := c.ControllerManager.ExtraArgs[maskArg]; ok && v != "" { - // assume it is an integer, if not it will fail later - maskSize, err = strconv.Atoi(v) - if err != nil { - errors.Wrapf(err, "could not parse the value of the kube-controller-manager flag %s as an integer: %v", maskArg, err) - return 0, err - } - } else if isIPv6 { - maskSize = defaultNodeMaskCIDRIPv6 - } else { - maskSize = defaultNodeMaskCIDRIPv4 - } - return maskSize, nil -} - -// ValidateNetworking validates networking configuration -func ValidateNetworking(c *kubeadm.ClusterConfiguration, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - dnsDomainFldPath := field.NewPath("dnsDomain") - for _, err := range validation.IsDNS1123Subdomain(c.Networking.DNSDomain) { - allErrs = append(allErrs, field.Invalid(dnsDomainFldPath, c.Networking.DNSDomain, err)) - } - // check if dual-stack feature-gate is enabled - isDualStack := features.Enabled(c.FeatureGates, features.IPv6DualStack) - - if len(c.Networking.ServiceSubnet) != 0 { - allErrs = append(allErrs, ValidateIPNetFromString(c.Networking.ServiceSubnet, constants.MinimumAddressesInServiceSubnet, isDualStack, field.NewPath("serviceSubnet"))...) - // Service subnet was already validated, we need to validate now the subnet size - allErrs = append(allErrs, ValidateServiceSubnetSize(c.Networking.ServiceSubnet, field.NewPath("serviceSubnet"))...) - } - if len(c.Networking.PodSubnet) != 0 { - allErrs = append(allErrs, ValidateIPNetFromString(c.Networking.PodSubnet, constants.MinimumAddressesInPodSubnet, isDualStack, field.NewPath("podSubnet"))...) - // Pod subnet was already validated, we need to validate now against the node-mask - allErrs = append(allErrs, ValidatePodSubnetNodeMask(c.Networking.PodSubnet, c, field.NewPath("podSubnet"))...) - } - return allErrs -} - -// ValidateAbsolutePath validates whether provided path is absolute or not -func ValidateAbsolutePath(path string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - if !filepath.IsAbs(path) { - allErrs = append(allErrs, field.Invalid(fldPath, path, "path is not absolute")) - } - return allErrs -} - -// ValidateMixedArguments validates passed arguments -func ValidateMixedArguments(flag *pflag.FlagSet) error { - // If --config isn't set, we have nothing to validate - if !flag.Changed("config") { - return nil - } - - mixedInvalidFlags := []string{} - flag.Visit(func(f *pflag.Flag) { - if isAllowedFlag(f.Name) { - // "--skip-*" flags or other allowed flags can be set with --config - return - } - mixedInvalidFlags = append(mixedInvalidFlags, f.Name) - }) - - if len(mixedInvalidFlags) != 0 { - return errors.Errorf("can not mix '--config' with arguments %v", mixedInvalidFlags) - } - return nil -} - -func isAllowedFlag(flagName string) bool { - knownFlags := sets.NewString(kubeadmcmdoptions.CfgPath, - kubeadmcmdoptions.IgnorePreflightErrors, - kubeadmcmdoptions.DryRun, - kubeadmcmdoptions.KubeconfigPath, - kubeadmcmdoptions.NodeName, - kubeadmcmdoptions.NodeCRISocket, - kubeadmcmdoptions.KubeconfigDir, - kubeadmcmdoptions.UploadCerts, - kubeadmcmdoptions.Patches, - "print-join-command", "rootfs", "v") - if knownFlags.Has(flagName) { - return true - } - return strings.HasPrefix(flagName, "skip-") -} - -// ValidateFeatureGates validates provided feature gates -func ValidateFeatureGates(featureGates map[string]bool, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - // check valid feature names are provided - for k := range featureGates { - if !features.Supports(features.InitFeatureGates, k) { - allErrs = append(allErrs, field.Invalid(fldPath, featureGates, - fmt.Sprintf("%s is not a valid feature name.", k))) - } - } - return allErrs -} - -// ValidateAPIEndpoint validates API server's endpoint -func ValidateAPIEndpoint(c *kubeadm.APIEndpoint, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - allErrs = append(allErrs, ValidateIPFromString(c.AdvertiseAddress, fldPath.Child("advertiseAddress"))...) - allErrs = append(allErrs, ValidatePort(c.BindPort, fldPath.Child("bindPort"))...) - return allErrs -} - -// ValidateIgnorePreflightErrors validates duplicates in: -// - ignore-preflight-errors flag and -// - ignorePreflightErrors field in {Init,Join}Configuration files. -func ValidateIgnorePreflightErrors(ignorePreflightErrorsFromCLI, ignorePreflightErrorsFromConfigFile []string) (sets.String, error) { - ignoreErrors := sets.NewString() - allErrs := field.ErrorList{} - - for _, item := range ignorePreflightErrorsFromConfigFile { - ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive - } - - if ignoreErrors.Has("all") { - // "all" is forbidden in config files. Administrators should use an - // explicit list of errors they want to ignore, as it can be risky to - // mask all errors in such a way. Hence, we return an error: - allErrs = append(allErrs, field.Invalid(field.NewPath("ignorePreflightErrors"), "all", "'all' cannot be used in configuration file")) - } - - for _, item := range ignorePreflightErrorsFromCLI { - ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive - } - - if ignoreErrors.Has("all") && ignoreErrors.Len() > 1 { - allErrs = append(allErrs, field.Invalid(field.NewPath("ignore-preflight-errors"), strings.Join(ignoreErrors.List(), ","), "don't specify individual checks if 'all' is used")) - } - - return ignoreErrors, allErrs.ToAggregate() -} - -// ValidateSocketPath validates format of socket path or url -func ValidateSocketPath(socket string, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - u, err := url.Parse(socket) - if err != nil { - return append(allErrs, field.Invalid(fldPath, socket, fmt.Sprintf("URL parsing error: %v", err))) - } - - if u.Scheme == "" { - if !filepath.IsAbs(u.Path) { - return append(allErrs, field.Invalid(fldPath, socket, fmt.Sprintf("path is not absolute: %s", socket))) - } - } else if u.Scheme != kubeadmapiv1beta2.DefaultUrlScheme { - return append(allErrs, field.Invalid(fldPath, socket, fmt.Sprintf("URL scheme %s is not supported", u.Scheme))) - } - - return allErrs -} diff --git a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go b/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go deleted file mode 100644 index 6b34cc902d232..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go +++ /dev/null @@ -1,1286 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package validation - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/features" -) - -func TestValidateToken(t *testing.T) { - var tests = []struct { - token string - expected bool - }{ - {"772ef5.6b6baab1d4a0a171", true}, - {".6b6baab1d4a0a171", false}, - {"772ef5.", false}, - {"abcdef.1234567890123456@foobar", false}, - } - for _, rt := range tests { - err := ValidateToken(rt.token, nil).ToAggregate() - if (err == nil) != rt.expected { - t.Errorf( - "failed ValidateToken:\n\ttoken: %q\n\t expected: %t, got: %t", - rt.token, - rt.expected, - (err == nil), - ) - } - } -} - -func TestValidateValidateTokenUsages(t *testing.T) { - var tests = []struct { - u []string - f *field.Path - expected bool - }{ - {[]string{}, nil, true}, // supported (no usages) - {[]string{"signing", "authentication"}, nil, true}, // supported - {[]string{"something else"}, nil, false}, // usage not supported - } - for _, rt := range tests { - actual := ValidateTokenUsages(rt.u, rt.f) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "failed ValidateTokenUsages:\n\texpected: %t\n\t actual: %t", - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateTokenGroups(t *testing.T) { - var tests = []struct { - u []string - g []string - f *field.Path - expected bool - }{ - {[]string{"some usage"}, []string{"some group"}, nil, false}, // groups doesn't makes sense if usage authentication - {[]string{"authentication"}, []string{"some group"}, nil, false}, // group not supported - {[]string{"authentication"}, []string{"system:bootstrappers:anygroup"}, nil, true}, // supported - } - for _, rt := range tests { - actual := ValidateTokenGroups(rt.u, rt.g, rt.f) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "failed ValidateTokenGroups:\n\texpected: %t\n\t actual: %t", - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateNodeRegistrationOptions(t *testing.T) { - var tests = []struct { - nodeName string - expectedErrors bool - }{ - {"", true}, // node name can't be empty - {"INVALID-NODENAME", true}, // Upper cases is invalid - {"invalid-nodename-", true}, // Can't have trailing dashes - {"invalid-node?name", true}, // Unsupported characters - {"valid-nodename", false}, // supported - // test cases for criSocket are covered in TestValidateSocketPath - } - for _, rt := range tests { - nro := kubeadm.NodeRegistrationOptions{Name: rt.nodeName, CRISocket: "/some/path"} - actual := ValidateNodeRegistrationOptions(&nro, field.NewPath("nodeRegistration")) - actualErrors := len(actual) > 0 - if actualErrors != rt.expectedErrors { - t.Errorf( - "failed ValidateNodeRegistrationOptions: value: %v\n\texpected: %t\n\t actual: %t", - nro, - rt.expectedErrors, - actualErrors, - ) - } - } -} - -func TestValidateCertSANs(t *testing.T) { - var tests = []struct { - sans []string - expected bool - }{ - {[]string{}, true}, // ok if not provided - {[]string{"1,2,,3"}, false}, // not a DNS label or IP - {[]string{"my-hostname", "???&?.garbage"}, false}, // not valid - {[]string{"my-hostname", "my.subdomain", "1.2.3.4"}, true}, // supported - {[]string{"my-hostname2", "my.other.subdomain", "10.0.0.10"}, true}, // supported - {[]string{"my-hostname", "my.subdomain", "2001:db8::4"}, true}, // supported - {[]string{"my-hostname2", "my.other.subdomain", "2001:db8::10"}, true}, // supported - {[]string{"*.my-hostname2", "*.my.other.subdomain"}, true}, // supported Wildcard DNS label - {[]string{"**.my-hostname2", "my.other.subdomain"}, false}, // not a Wildcard DNS label - {[]string{"*.*.my-hostname2", "my.other.subdomain"}, false}, // not a Wildcard DNS label - {[]string{"a.*.my-hostname2", "my.other.subdomain"}, false}, // not a Wildcard DNS label - {[]string{"*", "my.other.subdomain", "2001:db8::10"}, false}, // not a Wildcard DNS label - } - for _, rt := range tests { - actual := ValidateCertSANs(rt.sans, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "failed ValidateCertSANs:\n\texpected: %t\n\t actual: %t", - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateIPFromString(t *testing.T) { - var tests = []struct { - name string - ip string - expected bool - }{ - {"invalid missing address", "", false}, - {"invalid missing decimal points in IPv4 address", "1234", false}, - {"invalid incomplete IPv4 address", "1.2", false}, - {"invalid IPv4 CIDR provided instead of IPv4 address", "1.2.3.4/16", false}, - {"valid IPv4 address", "1.2.3.4", true}, - {"valid IPv6 address", "2001:db8::1", true}, - {"invalid IPv6 CIDR provided instead of IPv6 address", "2001:db8::1/64", false}, - {"invalid hex character in IPv6 address", "2001:xb8::", false}, - {"invalid use of colons in IPv6 address", "2001::db8::", false}, - } - for _, rt := range tests { - actual := ValidateIPFromString(rt.ip, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed:\n\texpected: %t\n\t actual: %t", - rt.name, - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateIPNetFromString(t *testing.T) { - var tests = []struct { - name string - subnet string - minaddrs int64 - checkDualStack bool - expected bool - }{ - {"invalid missing CIDR", "", 0, false, false}, - {"invalid CIDR", "a", 0, false, false}, - {"invalid CIDR missing decimal points in IPv4 address and / mask", "1234", 0, false, false}, - {"invalid CIDR use of letters instead of numbers and / mask", "abc", 0, false, false}, - {"invalid IPv4 address provided instead of CIDR representation", "1.2.3.4", 0, false, false}, - {"invalid IPv6 address provided instead of CIDR representation", "2001:db8::1", 0, false, false}, - {"invalid multiple CIDR provided in a single stack cluster", "2001:db8::1/64,1.2.3.4/24", 0, false, false}, - {"invalid multiple CIDR provided in a single stack cluster and one invalid subnet", "2001:db8::1/64,a", 0, false, false}, - {"valid, but IPv4 CIDR too small. At least 10 addresses needed", "10.0.0.16/29", 10, false, false}, - {"valid, but IPv6 CIDR too small. At least 10 addresses needed", "2001:db8::/125", 10, false, false}, - {"valid IPv4 CIDR", "10.0.0.16/12", 10, false, true}, - {"valid IPv6 CIDR", "2001:db8::/98", 10, false, true}, - // dual-stack: - {"invalid missing CIDR", "", 0, true, false}, - {"valid dual-stack enabled but only an IPv4 CIDR specified", "10.0.0.16/12", 10, true, true}, - {"valid dual-stack enabled but only an IPv6 CIDR specified", "2001:db8::/98", 10, true, true}, - {"invalid IPv4 address provided instead of CIDR representation", "1.2.3.4,2001:db8::/98", 0, true, false}, - {"invalid IPv6 address provided instead of CIDR representation", "2001:db8::1,10.0.0.16/12", 0, true, false}, - {"valid, but IPv4 CIDR too small. At least 10 addresses needed", "10.0.0.16/29,2001:db8::/98", 10, true, false}, - {"valid, but IPv6 CIDR too small. At least 10 addresses needed", "10.0.0.16/12,2001:db8::/125", 10, true, false}, - {"valid, but only IPv4 family addresses specified. IPv6 CIDR is necessary.", "10.0.0.16/12,192.168.0.0/16", 10, true, false}, - {"valid, but only IPv6 family addresses specified. IPv4 CIDR is necessary.", "2001:db8::/98,2005:db8::/98", 10, true, false}, - {"valid IPv4 and IPv6 CIDR", "10.0.0.16/12,2001:db8::/98", 10, true, true}, - {"valid IPv6 and IPv4 CIDR", "10.0.0.16/12,2001:db8::/98", 10, true, true}, - {"invalid IPv6 and IPv4 CIDR with more than 2 subnets", "10.0.0.16/12,2001:db8::/98,192.168.0.0/16", 10, true, false}, - {"invalid IPv6 and IPv4 CIDR with more than 2 subnets", "10.0.0.16/12,2001:db8::/98,192.168.0.0/16,a.b.c.d/24", 10, true, false}, - } - for _, rt := range tests { - actual := ValidateIPNetFromString(rt.subnet, rt.minaddrs, rt.checkDualStack, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed :\n\texpected: %t\n\t actual: %t\n\t err(s): %v\n\t", - rt.name, - rt.expected, - (len(actual) == 0), - actual, - ) - } - } -} - -func TestValidatePodSubnetNodeMask(t *testing.T) { - var tests = []struct { - name string - subnet string - cmExtraArgs map[string]string - checkDualStack bool - expected bool - }{ - {"single IPv4, but mask too small. Default node-mask", "10.0.0.16/29", nil, false, false}, - {"single IPv4, but mask too small. Configured node-mask", "10.0.0.16/24", map[string]string{"node-cidr-mask-size": "23"}, false, false}, - {"single IPv6, but mask too small. Default node-mask", "2001:db8::1/112", nil, false, false}, - {"single IPv6, but mask too small. Configured node-mask", "2001:db8::1/64", map[string]string{"node-cidr-mask-size": "24"}, false, false}, - {"single IPv6, but mask difference greater than 16. Default node-mask", "2001:db8::1/12", nil, false, false}, - {"single IPv6, but mask difference greater than 16. Configured node-mask", "2001:db8::1/64", map[string]string{"node-cidr-mask-size": "120"}, false, false}, - {"single IPv4 CIDR", "10.0.0.16/12", nil, false, true}, - {"single IPv6 CIDR", "2001:db8::/48", nil, false, true}, - // dual-stack: - {"dual IPv4 only, but mask too small. Default node-mask", "10.0.0.16/29", nil, true, false}, - {"dual IPv4 only, but mask too small. Configured node-mask", "10.0.0.16/24", map[string]string{"node-cidr-mask-size-ipv4": "23"}, true, false}, - {"dual IPv6 only, but mask too small. Default node-mask", "2001:db8::1/112", nil, true, false}, - {"dual IPv6 only, but mask too small. Configured node-mask", "2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "24"}, true, false}, - {"dual IPv6 only, but mask difference greater than 16. Default node-mask", "2001:db8::1/12", nil, true, false}, - {"dual IPv6 only, but mask difference greater than 16. Configured node-mask", "2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "120"}, true, false}, - {"dual IPv4 only CIDR", "10.0.0.16/12", nil, true, true}, - {"dual IPv6 only CIDR", "2001:db8::/48", nil, true, true}, - {"dual, but IPv4 mask too small. Default node-mask", "10.0.0.16/29,2001:db8::/48", nil, true, false}, - {"dual, but IPv4 mask too small. Configured node-mask", "10.0.0.16/24,2001:db8::/48", map[string]string{"node-cidr-mask-size-ipv4": "23"}, true, false}, - {"dual, but IPv6 mask too small. Default node-mask", "2001:db8::1/112,10.0.0.16/16", nil, true, false}, - {"dual, but IPv6 mask too small. Configured node-mask", "10.0.0.16/16,2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "24"}, true, false}, - {"dual, but mask difference greater than 16. Default node-mask", "2001:db8::1/12,10.0.0.16/16", nil, true, false}, - {"dual, but mask difference greater than 16. Configured node-mask", "10.0.0.16/16,2001:db8::1/64", map[string]string{"node-cidr-mask-size-ipv6": "120"}, true, false}, - {"dual IPv4 IPv6", "2001:db8::/48,10.0.0.16/12", nil, true, true}, - {"dual IPv6 IPv4", "2001:db8::/48,10.0.0.16/12", nil, true, true}, - } - for _, rt := range tests { - cfg := &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: rt.checkDualStack}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: rt.cmExtraArgs, - }, - } - actual := ValidatePodSubnetNodeMask(rt.subnet, cfg, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed :\n\texpected: %t\n\t actual: %t\n\t err(s): %v\n\t", - rt.name, - rt.expected, - (len(actual) == 0), - actual, - ) - } - } -} - -func TestValidateServiceSubnetSize(t *testing.T) { - var tests = []struct { - name string - subnet string - expected bool - }{ - {"single IPv4, but mask too large.", "10.0.0.16/2", false}, - {"single IPv6, but mask too large.", "2001:db8::1/64", false}, - {"single IPv4 CIDR", "10.0.0.16/12", true}, - {"single IPv6 CIDR", "2001:db8::/112", true}, - // dual-stack: - {"dual, but IPv4 mask too large.", "2001:db8::1/112,10.0.0.16/6", false}, - {"dual, but IPv6 mask too large.", "2001:db8::1/12,10.0.0.16/16", false}, - {"dual IPv4 IPv6", "10.0.0.16/12,2001:db8::/112", true}, - {"dual IPv6 IPv4", "2001:db8::/112,10.0.0.16/12", true}, - } - for _, rt := range tests { - - actual := ValidateServiceSubnetSize(rt.subnet, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed :\n\texpected: %t\n\t actual: %t\n\t err(s): %v\n\t", - rt.name, - rt.expected, - (len(actual) == 0), - actual, - ) - } - } -} - -func TestValidateHostPort(t *testing.T) { - var tests = []struct { - name string - s string - expected bool - }{ - { - name: "Valid DNS address / port", - s: "cp.k8s.io:8081", - expected: true, - }, - { - name: "Valid DNS address", - s: "cp.k8s.io", - expected: true, - }, - { - name: "Valid IPv4 address / port", - s: "1.2.3.4:8081", - expected: true, - }, - { - name: "Valid IPv4 address", - s: "1.2.3.4", - expected: true, - }, - { - name: "Valid IPv6 address / port", - s: "[2001:db7::1]:8081", - expected: true, - }, - { - name: "Valid IPv6 address", - s: "2001:db7::1", - expected: true, - }, - { - name: "Invalid IPv4 address, but valid DNS", - s: "1.2.34", - expected: true, - }, - { - name: "Invalid DNS", - s: "a.B.c.d.e", - expected: false, - }, - { - name: "Invalid IPv6 address", - s: "2001:db7:1", - expected: false, - }, - { - name: "Invalid BindPort", - s: "1.2.3.4:0", - expected: false, - }, - } - for _, rt := range tests { - actual := ValidateHostPort(rt.s, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed:\n\texpected: %t\n\t actual: %t", - rt.name, - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateAPIEndpoint(t *testing.T) { - var tests = []struct { - name string - s *kubeadm.APIEndpoint - expected bool - }{ - { - name: "Valid IPv4 address / port", - s: &kubeadm.APIEndpoint{ - AdvertiseAddress: "4.5.6.7", - BindPort: 6443, - }, - expected: true, - }, - { - name: "Valid IPv6 address / port", - s: &kubeadm.APIEndpoint{ - AdvertiseAddress: "2001:db7::2", - BindPort: 6443, - }, - expected: true, - }, - { - name: "Invalid IPv4 address", - s: &kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.34", - BindPort: 6443, - }, - expected: false, - }, - { - name: "Invalid IPv6 address", - s: &kubeadm.APIEndpoint{ - AdvertiseAddress: "2001:db7:1", - BindPort: 6443, - }, - expected: false, - }, - { - name: "Invalid BindPort", - s: &kubeadm.APIEndpoint{ - AdvertiseAddress: "4.5.6.7", - BindPort: 0, - }, - expected: false, - }, - } - for _, rt := range tests { - actual := ValidateAPIEndpoint(rt.s, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed:\n\texpected: %t\n\t actual: %t", - rt.name, - rt.expected, - (len(actual) == 0), - ) - } - } -} - -//TODO: Create a separated test for ValidateClusterConfiguration -func TestValidateInitConfiguration(t *testing.T) { - nodename := "valid-nodename" - var tests = []struct { - name string - s *kubeadm.InitConfiguration - expected bool - }{ - {"invalid missing InitConfiguration", - &kubeadm.InitConfiguration{}, false}, - {"invalid missing token with IPv4 service subnet", - &kubeadm.InitConfiguration{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{ - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - }, - CertificatesDir: "/some/cert/dir", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{Name: nodename, CRISocket: "/some/path"}, - }, false}, - {"invalid missing token with IPv6 service subnet", - &kubeadm.InitConfiguration{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{ - Networking: kubeadm.Networking{ - ServiceSubnet: "2001:db8::1/98", - DNSDomain: "cluster.local", - }, - CertificatesDir: "/some/cert/dir", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{Name: nodename, CRISocket: "/some/path"}, - }, false}, - {"invalid missing node name", - &kubeadm.InitConfiguration{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{ - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - }, - CertificatesDir: "/some/other/cert/dir", - }, - }, false}, - {"valid InitConfiguration with incorrect IPv4 pod subnet", - &kubeadm.InitConfiguration{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{ - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - PodSubnet: "10.0.1.15", - }, - CertificatesDir: "/some/other/cert/dir", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{Name: nodename, CRISocket: "/some/path"}, - }, false}, - {"valid InitConfiguration with IPv4 service subnet", - &kubeadm.InitConfiguration{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 6443, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{ - Etcd: kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: "/some/path", - }, - }, - Networking: kubeadm.Networking{ - ServiceSubnet: "10.96.0.1/12", - DNSDomain: "cluster.local", - PodSubnet: "10.0.1.15/16", - }, - CertificatesDir: "/some/other/cert/dir", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{Name: nodename, CRISocket: "/some/path"}, - }, true}, - {"valid InitConfiguration using IPv6 service subnet", - &kubeadm.InitConfiguration{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1:2:3::4", - BindPort: 3446, - }, - ClusterConfiguration: kubeadm.ClusterConfiguration{ - Etcd: kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: "/some/path", - }, - }, - Networking: kubeadm.Networking{ - ServiceSubnet: "2001:db8::1/112", - DNSDomain: "cluster.local", - }, - CertificatesDir: "/some/other/cert/dir", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{Name: nodename, CRISocket: "/some/path"}, - }, true}, - } - for _, rt := range tests { - actual := ValidateInitConfiguration(rt.s) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%s test case failed:\n\texpected: %t\n\t actual: %t", - rt.name, - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateJoinConfiguration(t *testing.T) { - var tests = []struct { - s *kubeadm.JoinConfiguration - expected bool - }{ - {&kubeadm.JoinConfiguration{}, false}, - {&kubeadm.JoinConfiguration{ - CACertPath: "/some/cert.crt", - Discovery: kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456@foobar", - }, - File: &kubeadm.FileDiscovery{ - KubeConfigPath: "foo", - }, - }, - }, false}, - {&kubeadm.JoinConfiguration{ // Pass without JoinControlPlane - CACertPath: "/some/cert.crt", - Discovery: kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "1.2.3.4:6443", - CACertHashes: []string{"aaaa"}, - }, - TLSBootstrapToken: "abcdef.1234567890123456", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "aaa", - CRISocket: "/var/run/dockershim.sock", - }, - }, true}, - {&kubeadm.JoinConfiguration{ // Pass with JoinControlPlane - CACertPath: "/some/cert.crt", - Discovery: kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "1.2.3.4:6443", - CACertHashes: []string{"aaaa"}, - }, - TLSBootstrapToken: "abcdef.1234567890123456", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "aaa", - CRISocket: "/var/run/dockershim.sock", - }, - ControlPlane: &kubeadm.JoinControlPlane{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 1234, - }, - }, - }, true}, - {&kubeadm.JoinConfiguration{ // Fail JoinControlPlane.AdvertiseAddress validation - CACertPath: "/some/cert.crt", - Discovery: kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "1.2.3.4:6443", - CACertHashes: []string{"aaaa"}, - }, - TLSBootstrapToken: "abcdef.1234567890123456", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "aaa", - CRISocket: "/var/run/dockershim.sock", - }, - ControlPlane: &kubeadm.JoinControlPlane{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "aaa", - BindPort: 1234, - }, - }, - }, false}, - {&kubeadm.JoinConfiguration{ // Fail JoinControlPlane.BindPort validation - CACertPath: "/some/cert.crt", - Discovery: kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "1.2.3.4:6443", - CACertHashes: []string{"aaaa"}, - }, - TLSBootstrapToken: "abcdef.1234567890123456", - }, - NodeRegistration: kubeadm.NodeRegistrationOptions{ - Name: "aaa", - CRISocket: "/var/run/dockershim.sock", - }, - ControlPlane: &kubeadm.JoinControlPlane{ - LocalAPIEndpoint: kubeadm.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: -1, - }, - }, - }, false}, - } - for _, rt := range tests { - actual := ValidateJoinConfiguration(rt.s) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "failed ValidateJoinConfiguration:\n\texpected: %t\n\t actual: %t", - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateMixedArguments(t *testing.T) { - var tests = []struct { - args []string - expected bool - }{ - // Expected to succeed, --config is mixed with skip-* flags only or no other flags - {[]string{"--foo=bar"}, true}, - {[]string{"--config=hello"}, true}, - {[]string{"--config=hello", "--ignore-preflight-errors=all"}, true}, - {[]string{"--config=hello", "--skip-token-print=true"}, true}, - {[]string{"--config=hello", "--ignore-preflight-errors=baz", "--skip-token-print"}, true}, - // Expected to fail, --config is mixed with the --foo flag - {[]string{"--config=hello", "--ignore-preflight-errors=baz", "--foo=bar"}, false}, - {[]string{"--config=hello", "--foo=bar"}, false}, - } - - var cfgPath string - var ignorePreflightErrors []string - for _, rt := range tests { - f := pflag.NewFlagSet("test", pflag.ContinueOnError) - if f.Parsed() { - t.Error("f.Parse() = true before Parse") - } - f.String("foo", "", "flag bound to config object") - f.StringSliceVar(&ignorePreflightErrors, "ignore-preflight-errors", ignorePreflightErrors, "flag not bound to config object") - f.Bool("skip-token-print", false, "flag not bound to config object") - f.StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file") - if err := f.Parse(rt.args); err != nil { - t.Fatal(err) - } - - actual := ValidateMixedArguments(f) - if (actual == nil) != rt.expected { - t.Errorf( - "failed ValidateMixedArguments:\n\texpected: %t\n\t actual: %t testdata: %v", - rt.expected, - (actual == nil), - rt.args, - ) - } - } -} - -func TestValidateFeatureGates(t *testing.T) { - type featureFlag map[string]bool - var tests = []struct { - featureGates featureFlag - expected bool - }{ - {featureFlag{"Unknown": true}, false}, - {featureFlag{"Unknown": false}, false}, - } - for _, rt := range tests { - actual := ValidateFeatureGates(rt.featureGates, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "failed featureGates %v:\n\texpected: %t\n\t actual: %t", - rt.featureGates, - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateIgnorePreflightErrors(t *testing.T) { - var tests = []struct { - ignorePreflightErrorsFromCLI []string - ignorePreflightErrorsFromConfigFile []string - expectedSet sets.String - expectedError bool - }{ - { // empty lists in CLI and config file - []string{}, - []string{}, - sets.NewString(), - false, - }, - { // empty list in CLI only - []string{}, - []string{"a"}, - sets.NewString("a"), - false, - }, - { // empty list in config file only - []string{"a"}, - []string{}, - sets.NewString("a"), - false, - }, - { // no duplicates, no overlap - []string{"a", "b"}, - []string{"c", "d"}, - sets.NewString("a", "b", "c", "d"), - false, - }, - { // some duplicates, with some overlapping duplicates - []string{"a", "b", "a"}, - []string{"c", "b"}, - sets.NewString("a", "b", "c"), - false, - }, - { // non-duplicate, but 'all' present together with individual checks in CLI - []string{"a", "b", "all"}, - []string{}, - sets.NewString(), - true, - }, - { // empty list in CLI, but 'all' present in config file, which is forbidden - []string{}, - []string{"all"}, - sets.NewString(), - true, - }, - { // non-duplicate, but 'all' present in config file, which is forbidden - []string{"a", "b"}, - []string{"all"}, - sets.NewString(), - true, - }, - { // non-duplicate, but 'all' present in CLI, while values are in config file, which is forbidden - []string{"all"}, - []string{"a", "b"}, - sets.NewString(), - true, - }, - { // skip all checks - []string{"all"}, - []string{}, - sets.NewString("all"), - false, - }, - } - for _, rt := range tests { - result, err := ValidateIgnorePreflightErrors(rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile) - switch { - case err != nil && !rt.expectedError: - t.Errorf("ValidateIgnorePreflightErrors: unexpected error for input (%s, %s), error: %v", rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, err) - case err == nil && rt.expectedError: - t.Errorf("ValidateIgnorePreflightErrors: expected error for input (%s, %s) but got: %v", rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, result) - case err == nil && !result.Equal(rt.expectedSet): - t.Errorf("ValidateIgnorePreflightErrors: expected (%v) for input (%s, %s) but got: %v", rt.expectedSet, rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, result) - } - } -} - -func TestValidateDiscovery(t *testing.T) { - var tests = []struct { - name string - d *kubeadm.Discovery - expected bool - }{ - { - "invalid: .BootstrapToken and .File cannot both be set", - &kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - }, - File: &kubeadm.FileDiscovery{ - KubeConfigPath: "https://url/file.conf", - }, - }, - false, - }, - { - "invalid: .BootstrapToken or .File must be set", - &kubeadm.Discovery{}, - false, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - err := ValidateDiscovery(rt.d, nil).ToAggregate() - if (err == nil) != rt.expected { - t.Errorf( - "test case failed: ValidateDiscovery:\n\texpected: %t\n\t actual: %t", - rt.expected, - (err == nil), - ) - } - }) - } -} - -func TestValidateDiscoveryBootstrapToken(t *testing.T) { - var tests = []struct { - name string - btd *kubeadm.BootstrapTokenDiscovery - expected bool - }{ - { - "invalid: .APIServerEndpoint not set", - &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - }, - false, - }, - { - "invalid: using token-based discovery without .BootstrapToken.CACertHashes and .BootstrapToken.UnsafeSkipCAVerification", - &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "192.168.122.100:6443", - UnsafeSkipCAVerification: false, - }, - false, - }, - { - "valid: using token-based discovery with .BootstrapToken.CACertHashes", - &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "192.168.122.100:6443", - CACertHashes: []string{"sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc"}, - UnsafeSkipCAVerification: false, - }, - true, - }, - { - "valid: using token-based discovery with .BootstrapToken.CACertHashe but skip ca verification", - &kubeadm.BootstrapTokenDiscovery{ - Token: "abcdef.1234567890123456", - APIServerEndpoint: "192.168.122.100:6443", - CACertHashes: []string{"sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc"}, - UnsafeSkipCAVerification: true, - }, - true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - err := ValidateDiscoveryBootstrapToken(rt.btd, nil).ToAggregate() - if (err == nil) != rt.expected { - t.Errorf( - "test case failed: ValidateDiscoveryBootstrapToken:\n\texpected: %t\n\t actual: %t", - rt.expected, - (err == nil), - ) - } - }) - } -} - -func TestValidateDiscoveryTokenAPIServer(t *testing.T) { - var tests = []struct { - apiServerEndpoint string - expected bool - }{ - { - "192.168.122.100", - false, - }, - { - "192.168.122.100:6443", - true, - }, - } - for _, rt := range tests { - actual := ValidateDiscoveryTokenAPIServer(rt.apiServerEndpoint, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "failed ValidateDiscoveryTokenAPIServer:\n\texpected: %t\n\t actual: %t", - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateDiscoveryKubeConfigPath(t *testing.T) { - tmpfile, err := ioutil.TempFile("/tmp", "test_discovery_file") - if err != nil { - t.Errorf("Error creating temporary file: %v", err) - } - defer os.Remove(tmpfile.Name()) - - var tests = []struct { - s string - expected bool - }{ - {"foo", false}, - {"/foo/bar/file_which_i_believe_not_existing.conf", false}, - {tmpfile.Name(), true}, - {"http://[::1]a", false}, - {"http://url/file.conf", false}, - {"https://u r l/file.conf", false}, - {"https://url/file.conf", true}, - } - for i, rt := range tests { - actual := ValidateDiscoveryKubeConfigPath(rt.s, nil) - if (len(actual) == 0) != rt.expected { - t.Errorf( - "%d: failed ValidateDiscoveryKubeConfigPath:\n\texpected: %t\n\t actual: %t", - i, - rt.expected, - (len(actual) == 0), - ) - } - } -} - -func TestValidateSocketPath(t *testing.T) { - var tests = []struct { - name string - criSocket string - expectedErrors bool - }{ - {name: "valid path", criSocket: "/some/path", expectedErrors: false}, - {name: "valid socket url", criSocket: kubeadmapiv1beta2.DefaultUrlScheme + "://" + "/some/path", expectedErrors: false}, - {name: "unsupported url scheme", criSocket: "bla:///some/path", expectedErrors: true}, - {name: "unparseable url", criSocket: ":::", expectedErrors: true}, - {name: "invalid CRISocket (path is not absolute)", criSocket: "some/path", expectedErrors: true}, - {name: "empty CRISocket (path is not absolute)", criSocket: "", expectedErrors: true}, - } - for _, tc := range tests { - actual := ValidateSocketPath(tc.criSocket, field.NewPath("criSocket")) - actualErrors := len(actual) > 0 - if actualErrors != tc.expectedErrors { - t.Errorf("error: socket path: %q\n\texpected: %t\n\t actual: %t", tc.criSocket, tc.expectedErrors, actualErrors) - } - } -} - -func TestValidateURLs(t *testing.T) { - var tests = []struct { - name string - urls []string - requireHTTPS bool - expectedErrors bool - }{ - { - name: "valid urls (https not required)", - urls: []string{"http://example.com", "https://example.org"}, - requireHTTPS: false, - expectedErrors: false, - }, - { - name: "valid urls (https required)", - urls: []string{"https://example.com", "https://example.org"}, - requireHTTPS: true, - expectedErrors: false, - }, - { - name: "invalid url (https required)", - urls: []string{"http://example.com", "https://example.org"}, - requireHTTPS: true, - expectedErrors: true, - }, - { - name: "URL parse error", - urls: []string{"::://example.com"}, - requireHTTPS: false, - expectedErrors: true, - }, - { - name: "URL without scheme", - urls: []string{"example.com"}, - requireHTTPS: false, - expectedErrors: true, - }, - } - for _, tc := range tests { - actual := ValidateURLs(tc.urls, tc.requireHTTPS, nil) - actualErrors := len(actual) > 0 - if actualErrors != tc.expectedErrors { - t.Errorf("error:\n\texpected: %t\n\t actual: %t", tc.expectedErrors, actualErrors) - } - } -} - -func TestValidateEtcd(t *testing.T) { - var tests = []struct { - name string - etcd *kubeadm.Etcd - expectedErrors bool - }{ - { - name: "either .Etcd.Local or .Etcd.External is required", - etcd: &kubeadm.Etcd{}, - expectedErrors: true, - }, - { - name: ".Etcd.Local and .Etcd.External are mutually exclusive", - etcd: &kubeadm.Etcd{ - Local: &kubeadm.LocalEtcd{ - DataDir: "/some/path", - }, - External: &kubeadm.ExternalEtcd{ - Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379"}, - }, - }, - expectedErrors: true, - }, - { - name: "either both or none of .Etcd.External.CertFile and .Etcd.External.KeyFile must be set", - etcd: &kubeadm.Etcd{ - External: &kubeadm.ExternalEtcd{ - Endpoints: []string{"https://external.etcd1:2379", "https://external.etcd2:2379"}, - CertFile: "/some/file.crt", - }, - }, - expectedErrors: true, - }, - { - name: "setting .Etcd.External.CertFile and .Etcd.External.KeyFile requires .Etcd.External.CAFile", - etcd: &kubeadm.Etcd{ - External: &kubeadm.ExternalEtcd{ - Endpoints: []string{"https://external.etcd1:2379", "https://external.etcd2:2379"}, - CertFile: "/some/file.crt", - KeyFile: "/some/file.key", - }, - }, - expectedErrors: true, - }, - { - name: "valid external etcd", - etcd: &kubeadm.Etcd{ - External: &kubeadm.ExternalEtcd{ - Endpoints: []string{"https://external.etcd1:2379", "https://external.etcd2:2379"}, - CertFile: "/etcd.crt", - KeyFile: "/etcd.key", - CAFile: "/etcd-ca.crt", - }, - }, - expectedErrors: false, - }, - { - name: "valid external etcd (no TLS)", - etcd: &kubeadm.Etcd{ - External: &kubeadm.ExternalEtcd{ - Endpoints: []string{"http://10.100.0.1:2379", "http://10.100.0.2:2379"}, - }, - }, - expectedErrors: false, - }, - } - - for _, tc := range tests { - actual := ValidateEtcd(tc.etcd, field.NewPath("etcd")) - actualErrors := len(actual) > 0 - if actualErrors != tc.expectedErrors { - t.Errorf("Error: \n\texpected: %t\n\t actual: %t", - tc.expectedErrors, - actualErrors, - ) - } - } -} - -func TestGetClusterNodeMask(t *testing.T) { - tests := []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - isIPv6 bool - expectedMask int - expectedError bool - }{ - { - name: "ipv4 default mask", - cfg: &kubeadmapi.ClusterConfiguration{}, - isIPv6: false, - expectedMask: 24, - }, - { - name: "ipv4 custom mask", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23"}, - }, - }, - isIPv6: false, - expectedMask: 23, - }, - { - name: "ipv4 wrong mask", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "aa23"}, - }, - }, - isIPv6: false, - expectedError: true, - }, - { - name: "ipv6 default mask", - cfg: &kubeadmapi.ClusterConfiguration{}, - isIPv6: true, - expectedMask: 64, - }, - { - name: "ipv6 custom mask", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "83"}, - }, - }, - isIPv6: true, - expectedMask: 83, - }, - { - name: "dual ipv4 default mask", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - }, - isIPv6: false, - expectedMask: 24, - }, - { - name: "dual ipv4 custom mask", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "21", "node-cidr-mask-size-ipv4": "23"}, - }, - }, - isIPv6: false, - expectedMask: 23, - }, - { - name: "dual ipv6 default mask", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - }, - isIPv6: true, - expectedMask: 64, - }, - { - name: "dual ipv6 custom mask", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv6": "83"}, - }, - }, - isIPv6: true, - expectedMask: 83, - }, - { - name: "dual ipv4 custom mask", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "23"}, - }, - }, - isIPv6: false, - expectedMask: 23, - }, - { - name: "dual ipv4 wrong mask", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "aa"}, - }, - }, - isIPv6: false, - expectedError: true, - }, - { - name: "dual ipv6 default mask and legacy flag", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23"}, - }, - }, - isIPv6: true, - expectedMask: 64, - }, - { - name: "dual ipv6 custom mask and legacy flag", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23", "node-cidr-mask-size-ipv6": "83"}, - }, - }, - isIPv6: true, - expectedMask: 83, - }, - { - name: "dual ipv6 custom mask and wrong flag", - cfg: &kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "23", "node-cidr-mask-size-ipv6": "a83"}, - }, - }, - isIPv6: true, - expectedError: true, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - mask, err := getClusterNodeMask(test.cfg, test.isIPv6) - if (err == nil) == test.expectedError { - t.Errorf("expected error: %v, got %v", test.expectedError, err) - } - if mask != test.expectedMask { - t.Errorf("expected mask: %d, got %d", test.expectedMask, mask) - } - }) - } -} diff --git a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go deleted file mode 100644 index 76bb6958ea9b4..0000000000000 --- a/cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go +++ /dev/null @@ -1,586 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package kubeadm - -import ( - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIEndpoint) DeepCopyInto(out *APIEndpoint) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIEndpoint. -func (in *APIEndpoint) DeepCopy() *APIEndpoint { - if in == nil { - return nil - } - out := new(APIEndpoint) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIServer) DeepCopyInto(out *APIServer) { - *out = *in - in.ControlPlaneComponent.DeepCopyInto(&out.ControlPlaneComponent) - if in.CertSANs != nil { - in, out := &in.CertSANs, &out.CertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.TimeoutForControlPlane != nil { - in, out := &in.TimeoutForControlPlane, &out.TimeoutForControlPlane - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServer. -func (in *APIServer) DeepCopy() *APIServer { - if in == nil { - return nil - } - out := new(APIServer) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { - *out = *in - if in.Token != nil { - in, out := &in.Token, &out.Token - *out = new(BootstrapTokenString) - **out = **in - } - if in.TTL != nil { - in, out := &in.TTL, &out.TTL - *out = new(v1.Duration) - **out = **in - } - if in.Expires != nil { - in, out := &in.Expires, &out.Expires - *out = (*in).DeepCopy() - } - if in.Usages != nil { - in, out := &in.Usages, &out.Usages - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Groups != nil { - in, out := &in.Groups, &out.Groups - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. -func (in *BootstrapToken) DeepCopy() *BootstrapToken { - if in == nil { - return nil - } - out := new(BootstrapToken) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapTokenDiscovery) DeepCopyInto(out *BootstrapTokenDiscovery) { - *out = *in - if in.CACertHashes != nil { - in, out := &in.CACertHashes, &out.CACertHashes - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapTokenDiscovery. -func (in *BootstrapTokenDiscovery) DeepCopy() *BootstrapTokenDiscovery { - if in == nil { - return nil - } - out := new(BootstrapTokenDiscovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapTokenString) DeepCopyInto(out *BootstrapTokenString) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapTokenString. -func (in *BootstrapTokenString) DeepCopy() *BootstrapTokenString { - if in == nil { - return nil - } - out := new(BootstrapTokenString) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.ComponentConfigs != nil { - in, out := &in.ComponentConfigs, &out.ComponentConfigs - *out = make(ComponentConfigMap, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - in.Etcd.DeepCopyInto(&out.Etcd) - out.Networking = in.Networking - in.APIServer.DeepCopyInto(&out.APIServer) - in.ControllerManager.DeepCopyInto(&out.ControllerManager) - in.Scheduler.DeepCopyInto(&out.Scheduler) - out.DNS = in.DNS - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterConfiguration. -func (in *ClusterConfiguration) DeepCopy() *ClusterConfiguration { - if in == nil { - return nil - } - out := new(ClusterConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.APIEndpoints != nil { - in, out := &in.APIEndpoints, &out.APIEndpoints - *out = make(map[string]APIEndpoint, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. -func (in *ClusterStatus) DeepCopy() *ClusterStatus { - if in == nil { - return nil - } - out := new(ClusterStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterStatus) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in ComponentConfigMap) DeepCopyInto(out *ComponentConfigMap) { - { - in := &in - *out = make(ComponentConfigMap, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - return - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentConfigMap. -func (in ComponentConfigMap) DeepCopy() ComponentConfigMap { - if in == nil { - return nil - } - out := new(ComponentConfigMap) - in.DeepCopyInto(out) - return *out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControlPlaneComponent) DeepCopyInto(out *ControlPlaneComponent) { - *out = *in - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ExtraVolumes != nil { - in, out := &in.ExtraVolumes, &out.ExtraVolumes - *out = make([]HostPathMount, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneComponent. -func (in *ControlPlaneComponent) DeepCopy() *ControlPlaneComponent { - if in == nil { - return nil - } - out := new(ControlPlaneComponent) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DNS) DeepCopyInto(out *DNS) { - *out = *in - out.ImageMeta = in.ImageMeta - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNS. -func (in *DNS) DeepCopy() *DNS { - if in == nil { - return nil - } - out := new(DNS) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Discovery) DeepCopyInto(out *Discovery) { - *out = *in - if in.BootstrapToken != nil { - in, out := &in.BootstrapToken, &out.BootstrapToken - *out = new(BootstrapTokenDiscovery) - (*in).DeepCopyInto(*out) - } - if in.File != nil { - in, out := &in.File, &out.File - *out = new(FileDiscovery) - **out = **in - } - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Discovery. -func (in *Discovery) DeepCopy() *Discovery { - if in == nil { - return nil - } - out := new(Discovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Etcd) DeepCopyInto(out *Etcd) { - *out = *in - if in.Local != nil { - in, out := &in.Local, &out.Local - *out = new(LocalEtcd) - (*in).DeepCopyInto(*out) - } - if in.External != nil { - in, out := &in.External, &out.External - *out = new(ExternalEtcd) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Etcd. -func (in *Etcd) DeepCopy() *Etcd { - if in == nil { - return nil - } - out := new(Etcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExternalEtcd) DeepCopyInto(out *ExternalEtcd) { - *out = *in - if in.Endpoints != nil { - in, out := &in.Endpoints, &out.Endpoints - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalEtcd. -func (in *ExternalEtcd) DeepCopy() *ExternalEtcd { - if in == nil { - return nil - } - out := new(ExternalEtcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileDiscovery) DeepCopyInto(out *FileDiscovery) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileDiscovery. -func (in *FileDiscovery) DeepCopy() *FileDiscovery { - if in == nil { - return nil - } - out := new(FileDiscovery) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HostPathMount) DeepCopyInto(out *HostPathMount) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathMount. -func (in *HostPathMount) DeepCopy() *HostPathMount { - if in == nil { - return nil - } - out := new(HostPathMount) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageMeta) DeepCopyInto(out *ImageMeta) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageMeta. -func (in *ImageMeta) DeepCopy() *ImageMeta { - if in == nil { - return nil - } - out := new(ImageMeta) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InitConfiguration) DeepCopyInto(out *InitConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ClusterConfiguration.DeepCopyInto(&out.ClusterConfiguration) - if in.BootstrapTokens != nil { - in, out := &in.BootstrapTokens, &out.BootstrapTokens - *out = make([]BootstrapToken, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - out.LocalAPIEndpoint = in.LocalAPIEndpoint - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InitConfiguration. -func (in *InitConfiguration) DeepCopy() *InitConfiguration { - if in == nil { - return nil - } - out := new(InitConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *InitConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JoinConfiguration) DeepCopyInto(out *JoinConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) - in.Discovery.DeepCopyInto(&out.Discovery) - if in.ControlPlane != nil { - in, out := &in.ControlPlane, &out.ControlPlane - *out = new(JoinControlPlane) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinConfiguration. -func (in *JoinConfiguration) DeepCopy() *JoinConfiguration { - if in == nil { - return nil - } - out := new(JoinConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *JoinConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *JoinControlPlane) DeepCopyInto(out *JoinControlPlane) { - *out = *in - out.LocalAPIEndpoint = in.LocalAPIEndpoint - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JoinControlPlane. -func (in *JoinControlPlane) DeepCopy() *JoinControlPlane { - if in == nil { - return nil - } - out := new(JoinControlPlane) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LocalEtcd) DeepCopyInto(out *LocalEtcd) { - *out = *in - out.ImageMeta = in.ImageMeta - if in.ExtraArgs != nil { - in, out := &in.ExtraArgs, &out.ExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ServerCertSANs != nil { - in, out := &in.ServerCertSANs, &out.ServerCertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.PeerCertSANs != nil { - in, out := &in.PeerCertSANs, &out.PeerCertSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalEtcd. -func (in *LocalEtcd) DeepCopy() *LocalEtcd { - if in == nil { - return nil - } - out := new(LocalEtcd) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Networking) DeepCopyInto(out *Networking) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Networking. -func (in *Networking) DeepCopy() *Networking { - if in == nil { - return nil - } - out := new(Networking) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeRegistrationOptions) DeepCopyInto(out *NodeRegistrationOptions) { - *out = *in - if in.Taints != nil { - in, out := &in.Taints, &out.Taints - *out = make([]corev1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.KubeletExtraArgs != nil { - in, out := &in.KubeletExtraArgs, &out.KubeletExtraArgs - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.IgnorePreflightErrors != nil { - in, out := &in.IgnorePreflightErrors, &out.IgnorePreflightErrors - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeRegistrationOptions. -func (in *NodeRegistrationOptions) DeepCopy() *NodeRegistrationOptions { - if in == nil { - return nil - } - out := new(NodeRegistrationOptions) - in.DeepCopyInto(out) - return out -} diff --git a/cmd/kubeadm/app/apis/output/BUILD b/cmd/kubeadm/app/apis/output/BUILD deleted file mode 100644 index a8122ef7eff8f..0000000000000 --- a/cmd/kubeadm/app/apis/output/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = [ - "doc.go", - "register.go", - "types.go", - "zz_generated.deepcopy.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/output", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/apis/output/fuzzer:all-srcs", - "//cmd/kubeadm/app/apis/output/scheme:all-srcs", - "//cmd/kubeadm/app/apis/output/v1alpha1:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/output/doc.go b/cmd/kubeadm/app/apis/output/doc.go deleted file mode 100644 index eaca8994d2a7d..0000000000000 --- a/cmd/kubeadm/app/apis/output/doc.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:deepcopy-gen=package -// +groupName=output.kubeadm.k8s.io - -// Package output implements the kubeadm structured output -// The purpose of the kubeadm structured output is to have a well -// defined versioned output format that other software that uses -// kubeadm for cluster deployments can use and rely on. -package output // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" diff --git a/cmd/kubeadm/app/apis/output/fuzzer/BUILD b/cmd/kubeadm/app/apis/output/fuzzer/BUILD deleted file mode 100644 index ec666ca0101a7..0000000000000 --- a/cmd/kubeadm/app/apis/output/fuzzer/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["fuzzer.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/fuzzer", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/output:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//vendor/github.com/google/gofuzz:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["fuzzer_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/output/scheme:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/apis/output/fuzzer/fuzzer.go b/cmd/kubeadm/app/apis/output/fuzzer/fuzzer.go deleted file mode 100644 index 6c3cc37772515..0000000000000 --- a/cmd/kubeadm/app/apis/output/fuzzer/fuzzer.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fuzzer - -import ( - "time" - - fuzz "github.com/google/gofuzz" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" -) - -// Funcs returns the fuzzer functions for the kubeadm apis. -func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { - return []interface{}{ - fuzzBootstrapToken, - } -} - -func fuzzBootstrapToken(obj *output.BootstrapToken, c fuzz.Continue) { - c.FuzzNoCustom(obj) - - obj.Token = &kubeadmapiv1beta2.BootstrapTokenString{ID: "uvxdac", Secret: "fq35fuyue3kd4gda"} - obj.Description = "" - obj.TTL = &metav1.Duration{Duration: time.Hour * 24} - obj.Usages = []string{"authentication", "signing"} - obj.Groups = []string{"system:bootstrappers:kubeadm:default-node-token"} -} diff --git a/cmd/kubeadm/app/apis/output/fuzzer/fuzzer_test.go b/cmd/kubeadm/app/apis/output/fuzzer/fuzzer_test.go deleted file mode 100644 index 0afa00fb8026f..0000000000000 --- a/cmd/kubeadm/app/apis/output/fuzzer/fuzzer_test.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fuzzer - -import ( - "testing" - - "k8s.io/apimachinery/pkg/api/apitesting/roundtrip" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" -) - -func TestRoundTripTypes(t *testing.T) { - roundtrip.RoundTripTestForAPIGroup(t, scheme.AddToScheme, Funcs) -} diff --git a/cmd/kubeadm/app/apis/output/register.go b/cmd/kubeadm/app/apis/output/register.go deleted file mode 100644 index d7b7fcdd8e9f0..0000000000000 --- a/cmd/kubeadm/app/apis/output/register.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package output - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupName is the group name use in this package -const GroupName = "output.kubeadm.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} - -var ( - // SchemeBuilder points to a list of functions added to Scheme. - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme applies all the stored functions to the scheme. - AddToScheme = SchemeBuilder.AddToScheme -) - -// Kind takes an unqualified kind and returns a Group qualified GroupKind -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &BootstrapToken{}, - &Images{}, - &UpgradePlan{}, - ) - return nil -} diff --git a/cmd/kubeadm/app/apis/output/scheme/BUILD b/cmd/kubeadm/app/apis/output/scheme/BUILD deleted file mode 100644 index ab0202408018c..0000000000000 --- a/cmd/kubeadm/app/apis/output/scheme/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["scheme.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/output:go_default_library", - "//cmd/kubeadm/app/apis/output/v1alpha1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/output/scheme/scheme.go b/cmd/kubeadm/app/apis/output/scheme/scheme.go deleted file mode 100644 index 053dc1bc1a3be..0000000000000 --- a/cmd/kubeadm/app/apis/output/scheme/scheme.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package scheme - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha1" -) - -// Scheme is the runtime.Scheme to which all kubeadm api types are registered. -var Scheme = runtime.NewScheme() - -// Codecs provides access to encoding and decoding for the scheme. -var Codecs = serializer.NewCodecFactory(Scheme) - -func init() { - metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - AddToScheme(Scheme) -} - -// AddToScheme builds the kubeadm scheme using all known versions of the kubeadm api. -func AddToScheme(scheme *runtime.Scheme) { - utilruntime.Must(output.AddToScheme(scheme)) - utilruntime.Must(v1alpha1.AddToScheme(scheme)) - utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion)) -} diff --git a/cmd/kubeadm/app/apis/output/types.go b/cmd/kubeadm/app/apis/output/types.go deleted file mode 100644 index cff41df51ecc4..0000000000000 --- a/cmd/kubeadm/app/apis/output/types.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package output - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// BootstrapToken represents information for the output produced by 'kubeadm token list' -// This is a copy of BoostrapToken struct from ../kubeadm/types.go with 2 additions: -// metav1.TypeMeta and metav1.ObjectMeta -type BootstrapToken struct { - metav1.TypeMeta - - kubeadmapiv1beta2.BootstrapToken -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// Images represents information for the output produced by 'kubeadm config images list' -type Images struct { - metav1.TypeMeta - - Images []string -} - -// ComponentUpgradePlan represents information about upgrade plan for one component -type ComponentUpgradePlan struct { - Name string - CurrentVersion string - NewVersion string -} - -// ComponentConfigVersionState describes the current and desired version of a component config -type ComponentConfigVersionState struct { - // Group points to the Kubernetes API group that covers the config - Group string - - // CurrentVersion is the currently active component config version - // NOTE: This can be empty in case the config was not found on the cluster or it was unsupported - // kubeadm generated version - CurrentVersion string - - // PreferredVersion is the component config version that is currently preferred by kubeadm for use. - // NOTE: As of today, this is the only version supported by kubeadm. - PreferredVersion string - - // ManualUpgradeRequired indicates if users need to manually upgrade their component config versions. This happens if - // the CurrentVersion of the config is user supplied (or modified) and no longer supported. Users should upgrade - // their component configs to PreferredVersion or any other supported component config version. - ManualUpgradeRequired bool -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// UpgradePlan represents information about upgrade plan for the output -// produced by 'kubeadm upgrade plan' -type UpgradePlan struct { - metav1.TypeMeta - - Components []ComponentUpgradePlan - - ConfigVersions []ComponentConfigVersionState -} diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/BUILD b/cmd/kubeadm/app/apis/output/v1alpha1/BUILD deleted file mode 100644 index 4d78b59c96da5..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = [ - "defaults.go", - "doc.go", - "register.go", - "types.go", - "zz_generated.conversion.go", - "zz_generated.deepcopy.go", - "zz_generated.defaults.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha1", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/output:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/defaults.go b/cmd/kubeadm/app/apis/output/v1alpha1/defaults.go deleted file mode 100644 index 720ecdfb5e5b6..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/defaults.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime" -) - -func addDefaultingFuncs(scheme *runtime.Scheme) error { - return RegisterDefaults(scheme) -} diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/doc.go b/cmd/kubeadm/app/apis/output/v1alpha1/doc.go deleted file mode 100644 index 875ac45e5838b..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/doc.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:defaulter-gen=TypeMeta -// +groupName=output.kubeadm.k8s.io -// +k8s:deepcopy-gen=package -// +k8s:conversion-gen=k8s.io/kubernetes/cmd/kubeadm/app/apis/output - -// Package v1alpha1 defines the v1alpha1 version of the kubeadm data structures -// related to structured output -// The purpose of the kubeadm structured output is to have a well -// defined versioned output format that other software that uses -// kubeadm for cluster deployments can use and rely on. -package v1alpha1 // import "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha1" diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/register.go b/cmd/kubeadm/app/apis/output/v1alpha1/register.go deleted file mode 100644 index a112685921059..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/register.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// GroupName is the group name use in this package -const GroupName = "output.kubeadm.k8s.io" - -// SchemeGroupVersion is group version used to register these objects -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} - -var ( - // TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api. - // localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes. - - // SchemeBuilder points to a list of functions added to Scheme. - SchemeBuilder runtime.SchemeBuilder - localSchemeBuilder = &SchemeBuilder - // AddToScheme applies all the stored functions to the scheme. - AddToScheme = localSchemeBuilder.AddToScheme -) - -func init() { - // We only register manually written functions here. The registration of the - // generated functions takes place in the generated files. The separation - // makes the code compile even when the generated files are missing. - localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs) -} - -// Kind takes an unqualified kind and returns a Group qualified GroupKind -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &BootstrapToken{}, - &Images{}, - &UpgradePlan{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/types.go b/cmd/kubeadm/app/apis/output/v1alpha1/types.go deleted file mode 100644 index 134dfee80fd2d..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/types.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" -) - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// BootstrapToken represents information for the bootstrap token output produced by kubeadm -// This is a copy of BoostrapToken struct from ../kubeadm/types.go with 2 additions: -// metav1.TypeMeta and metav1.ObjectMeta -type BootstrapToken struct { - metav1.TypeMeta `json:",inline"` - - kubeadmapiv1beta2.BootstrapToken -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// Images represents information for the output produced by 'kubeadm config images list' -type Images struct { - metav1.TypeMeta `json:",inline"` - - Images []string `json:"images"` -} - -// ComponentUpgradePlan represents information about upgrade plan for one component -type ComponentUpgradePlan struct { - Name string `json:"name"` - CurrentVersion string `json:"currentVersion"` - NewVersion string `json:"newVersion"` -} - -// ComponentConfigVersionState describes the current and desired version of a component config -type ComponentConfigVersionState struct { - // Group points to the Kubernetes API group that covers the config - Group string `json:"group"` - - // CurrentVersion is the currently active component config version - // NOTE: This can be empty in case the config was not found on the cluster or it was unsupported - // kubeadm generated version - CurrentVersion string `json:"currentVersion"` - - // PreferredVersion is the component config version that is currently preferred by kubeadm for use. - // NOTE: As of today, this is the only version supported by kubeadm. - PreferredVersion string `json:"preferredVersion"` - - // ManualUpgradeRequired indicates if users need to manually upgrade their component config versions. This happens if - // the CurrentVersion of the config is user supplied (or modified) and no longer supported. Users should upgrade - // their component configs to PreferredVersion or any other supported component config version. - ManualUpgradeRequired bool `json:"manualUpgradeRequired"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// UpgradePlan represents information about upgrade plan for the output -// produced by 'kubeadm upgrade plan' -type UpgradePlan struct { - metav1.TypeMeta - - Components []ComponentUpgradePlan `json:"components"` - - ConfigVersions []ComponentConfigVersionState `json:"configVersions"` -} diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.conversion.go deleted file mode 100644 index 0e23d2546ca1f..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.conversion.go +++ /dev/null @@ -1,201 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by conversion-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - unsafe "unsafe" - - conversion "k8s.io/apimachinery/pkg/conversion" - runtime "k8s.io/apimachinery/pkg/runtime" - output "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" -) - -func init() { - localSchemeBuilder.Register(RegisterConversions) -} - -// RegisterConversions adds conversion functions to the given scheme. -// Public to allow building arbitrary schemes. -func RegisterConversions(s *runtime.Scheme) error { - if err := s.AddGeneratedConversionFunc((*BootstrapToken)(nil), (*output.BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_BootstrapToken_To_output_BootstrapToken(a.(*BootstrapToken), b.(*output.BootstrapToken), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*output.BootstrapToken)(nil), (*BootstrapToken)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_output_BootstrapToken_To_v1alpha1_BootstrapToken(a.(*output.BootstrapToken), b.(*BootstrapToken), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ComponentConfigVersionState)(nil), (*output.ComponentConfigVersionState)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_ComponentConfigVersionState_To_output_ComponentConfigVersionState(a.(*ComponentConfigVersionState), b.(*output.ComponentConfigVersionState), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*output.ComponentConfigVersionState)(nil), (*ComponentConfigVersionState)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_output_ComponentConfigVersionState_To_v1alpha1_ComponentConfigVersionState(a.(*output.ComponentConfigVersionState), b.(*ComponentConfigVersionState), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ComponentUpgradePlan)(nil), (*output.ComponentUpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_ComponentUpgradePlan_To_output_ComponentUpgradePlan(a.(*ComponentUpgradePlan), b.(*output.ComponentUpgradePlan), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*output.ComponentUpgradePlan)(nil), (*ComponentUpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_output_ComponentUpgradePlan_To_v1alpha1_ComponentUpgradePlan(a.(*output.ComponentUpgradePlan), b.(*ComponentUpgradePlan), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Images)(nil), (*output.Images)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_Images_To_output_Images(a.(*Images), b.(*output.Images), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*output.Images)(nil), (*Images)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_output_Images_To_v1alpha1_Images(a.(*output.Images), b.(*Images), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*UpgradePlan)(nil), (*output.UpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha1_UpgradePlan_To_output_UpgradePlan(a.(*UpgradePlan), b.(*output.UpgradePlan), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*output.UpgradePlan)(nil), (*UpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_output_UpgradePlan_To_v1alpha1_UpgradePlan(a.(*output.UpgradePlan), b.(*UpgradePlan), scope) - }); err != nil { - return err - } - return nil -} - -func autoConvert_v1alpha1_BootstrapToken_To_output_BootstrapToken(in *BootstrapToken, out *output.BootstrapToken, s conversion.Scope) error { - out.BootstrapToken = in.BootstrapToken - return nil -} - -// Convert_v1alpha1_BootstrapToken_To_output_BootstrapToken is an autogenerated conversion function. -func Convert_v1alpha1_BootstrapToken_To_output_BootstrapToken(in *BootstrapToken, out *output.BootstrapToken, s conversion.Scope) error { - return autoConvert_v1alpha1_BootstrapToken_To_output_BootstrapToken(in, out, s) -} - -func autoConvert_output_BootstrapToken_To_v1alpha1_BootstrapToken(in *output.BootstrapToken, out *BootstrapToken, s conversion.Scope) error { - out.BootstrapToken = in.BootstrapToken - return nil -} - -// Convert_output_BootstrapToken_To_v1alpha1_BootstrapToken is an autogenerated conversion function. -func Convert_output_BootstrapToken_To_v1alpha1_BootstrapToken(in *output.BootstrapToken, out *BootstrapToken, s conversion.Scope) error { - return autoConvert_output_BootstrapToken_To_v1alpha1_BootstrapToken(in, out, s) -} - -func autoConvert_v1alpha1_ComponentConfigVersionState_To_output_ComponentConfigVersionState(in *ComponentConfigVersionState, out *output.ComponentConfigVersionState, s conversion.Scope) error { - out.Group = in.Group - out.CurrentVersion = in.CurrentVersion - out.PreferredVersion = in.PreferredVersion - out.ManualUpgradeRequired = in.ManualUpgradeRequired - return nil -} - -// Convert_v1alpha1_ComponentConfigVersionState_To_output_ComponentConfigVersionState is an autogenerated conversion function. -func Convert_v1alpha1_ComponentConfigVersionState_To_output_ComponentConfigVersionState(in *ComponentConfigVersionState, out *output.ComponentConfigVersionState, s conversion.Scope) error { - return autoConvert_v1alpha1_ComponentConfigVersionState_To_output_ComponentConfigVersionState(in, out, s) -} - -func autoConvert_output_ComponentConfigVersionState_To_v1alpha1_ComponentConfigVersionState(in *output.ComponentConfigVersionState, out *ComponentConfigVersionState, s conversion.Scope) error { - out.Group = in.Group - out.CurrentVersion = in.CurrentVersion - out.PreferredVersion = in.PreferredVersion - out.ManualUpgradeRequired = in.ManualUpgradeRequired - return nil -} - -// Convert_output_ComponentConfigVersionState_To_v1alpha1_ComponentConfigVersionState is an autogenerated conversion function. -func Convert_output_ComponentConfigVersionState_To_v1alpha1_ComponentConfigVersionState(in *output.ComponentConfigVersionState, out *ComponentConfigVersionState, s conversion.Scope) error { - return autoConvert_output_ComponentConfigVersionState_To_v1alpha1_ComponentConfigVersionState(in, out, s) -} - -func autoConvert_v1alpha1_ComponentUpgradePlan_To_output_ComponentUpgradePlan(in *ComponentUpgradePlan, out *output.ComponentUpgradePlan, s conversion.Scope) error { - out.Name = in.Name - out.CurrentVersion = in.CurrentVersion - out.NewVersion = in.NewVersion - return nil -} - -// Convert_v1alpha1_ComponentUpgradePlan_To_output_ComponentUpgradePlan is an autogenerated conversion function. -func Convert_v1alpha1_ComponentUpgradePlan_To_output_ComponentUpgradePlan(in *ComponentUpgradePlan, out *output.ComponentUpgradePlan, s conversion.Scope) error { - return autoConvert_v1alpha1_ComponentUpgradePlan_To_output_ComponentUpgradePlan(in, out, s) -} - -func autoConvert_output_ComponentUpgradePlan_To_v1alpha1_ComponentUpgradePlan(in *output.ComponentUpgradePlan, out *ComponentUpgradePlan, s conversion.Scope) error { - out.Name = in.Name - out.CurrentVersion = in.CurrentVersion - out.NewVersion = in.NewVersion - return nil -} - -// Convert_output_ComponentUpgradePlan_To_v1alpha1_ComponentUpgradePlan is an autogenerated conversion function. -func Convert_output_ComponentUpgradePlan_To_v1alpha1_ComponentUpgradePlan(in *output.ComponentUpgradePlan, out *ComponentUpgradePlan, s conversion.Scope) error { - return autoConvert_output_ComponentUpgradePlan_To_v1alpha1_ComponentUpgradePlan(in, out, s) -} - -func autoConvert_v1alpha1_Images_To_output_Images(in *Images, out *output.Images, s conversion.Scope) error { - out.Images = *(*[]string)(unsafe.Pointer(&in.Images)) - return nil -} - -// Convert_v1alpha1_Images_To_output_Images is an autogenerated conversion function. -func Convert_v1alpha1_Images_To_output_Images(in *Images, out *output.Images, s conversion.Scope) error { - return autoConvert_v1alpha1_Images_To_output_Images(in, out, s) -} - -func autoConvert_output_Images_To_v1alpha1_Images(in *output.Images, out *Images, s conversion.Scope) error { - out.Images = *(*[]string)(unsafe.Pointer(&in.Images)) - return nil -} - -// Convert_output_Images_To_v1alpha1_Images is an autogenerated conversion function. -func Convert_output_Images_To_v1alpha1_Images(in *output.Images, out *Images, s conversion.Scope) error { - return autoConvert_output_Images_To_v1alpha1_Images(in, out, s) -} - -func autoConvert_v1alpha1_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *output.UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]output.ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) - out.ConfigVersions = *(*[]output.ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) - return nil -} - -// Convert_v1alpha1_UpgradePlan_To_output_UpgradePlan is an autogenerated conversion function. -func Convert_v1alpha1_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *output.UpgradePlan, s conversion.Scope) error { - return autoConvert_v1alpha1_UpgradePlan_To_output_UpgradePlan(in, out, s) -} - -func autoConvert_output_UpgradePlan_To_v1alpha1_UpgradePlan(in *output.UpgradePlan, out *UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) - out.ConfigVersions = *(*[]ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) - return nil -} - -// Convert_output_UpgradePlan_To_v1alpha1_UpgradePlan is an autogenerated conversion function. -func Convert_output_UpgradePlan_To_v1alpha1_UpgradePlan(in *output.UpgradePlan, out *UpgradePlan, s conversion.Scope) error { - return autoConvert_output_UpgradePlan_To_v1alpha1_UpgradePlan(in, out, s) -} diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index c11c44675cb98..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,148 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { - *out = *in - out.TypeMeta = in.TypeMeta - in.BootstrapToken.DeepCopyInto(&out.BootstrapToken) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. -func (in *BootstrapToken) DeepCopy() *BootstrapToken { - if in == nil { - return nil - } - out := new(BootstrapToken) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *BootstrapToken) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ComponentConfigVersionState) DeepCopyInto(out *ComponentConfigVersionState) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentConfigVersionState. -func (in *ComponentConfigVersionState) DeepCopy() *ComponentConfigVersionState { - if in == nil { - return nil - } - out := new(ComponentConfigVersionState) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ComponentUpgradePlan) DeepCopyInto(out *ComponentUpgradePlan) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentUpgradePlan. -func (in *ComponentUpgradePlan) DeepCopy() *ComponentUpgradePlan { - if in == nil { - return nil - } - out := new(ComponentUpgradePlan) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Images) DeepCopyInto(out *Images) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.Images != nil { - in, out := &in.Images, &out.Images - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. -func (in *Images) DeepCopy() *Images { - if in == nil { - return nil - } - out := new(Images) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Images) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *UpgradePlan) DeepCopyInto(out *UpgradePlan) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.Components != nil { - in, out := &in.Components, &out.Components - *out = make([]ComponentUpgradePlan, len(*in)) - copy(*out, *in) - } - if in.ConfigVersions != nil { - in, out := &in.ConfigVersions, &out.ConfigVersions - *out = make([]ComponentConfigVersionState, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpgradePlan. -func (in *UpgradePlan) DeepCopy() *UpgradePlan { - if in == nil { - return nil - } - out := new(UpgradePlan) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *UpgradePlan) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} diff --git a/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.defaults.go b/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.defaults.go deleted file mode 100644 index 8cdf1335c1711..0000000000000 --- a/cmd/kubeadm/app/apis/output/v1alpha1/zz_generated.defaults.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by defaulter-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" - v1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" -) - -// RegisterDefaults adds defaulters functions to the given scheme. -// Public to allow building arbitrary schemes. -// All generated defaulters are covering - they call all nested defaulters. -func RegisterDefaults(scheme *runtime.Scheme) error { - scheme.AddTypeDefaultingFunc(&BootstrapToken{}, func(obj interface{}) { SetObjectDefaults_BootstrapToken(obj.(*BootstrapToken)) }) - return nil -} - -func SetObjectDefaults_BootstrapToken(in *BootstrapToken) { - v1beta2.SetDefaults_BootstrapToken(&in.BootstrapToken) -} diff --git a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go deleted file mode 100644 index 49788197571ee..0000000000000 --- a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go +++ /dev/null @@ -1,148 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package output - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { - *out = *in - out.TypeMeta = in.TypeMeta - in.BootstrapToken.DeepCopyInto(&out.BootstrapToken) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapToken. -func (in *BootstrapToken) DeepCopy() *BootstrapToken { - if in == nil { - return nil - } - out := new(BootstrapToken) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *BootstrapToken) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ComponentConfigVersionState) DeepCopyInto(out *ComponentConfigVersionState) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentConfigVersionState. -func (in *ComponentConfigVersionState) DeepCopy() *ComponentConfigVersionState { - if in == nil { - return nil - } - out := new(ComponentConfigVersionState) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ComponentUpgradePlan) DeepCopyInto(out *ComponentUpgradePlan) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentUpgradePlan. -func (in *ComponentUpgradePlan) DeepCopy() *ComponentUpgradePlan { - if in == nil { - return nil - } - out := new(ComponentUpgradePlan) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Images) DeepCopyInto(out *Images) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.Images != nil { - in, out := &in.Images, &out.Images - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. -func (in *Images) DeepCopy() *Images { - if in == nil { - return nil - } - out := new(Images) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Images) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *UpgradePlan) DeepCopyInto(out *UpgradePlan) { - *out = *in - out.TypeMeta = in.TypeMeta - if in.Components != nil { - in, out := &in.Components, &out.Components - *out = make([]ComponentUpgradePlan, len(*in)) - copy(*out, *in) - } - if in.ConfigVersions != nil { - in, out := &in.ConfigVersions, &out.ConfigVersions - *out = make([]ComponentConfigVersionState, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpgradePlan. -func (in *UpgradePlan) DeepCopy() *UpgradePlan { - if in == nil { - return nil - } - out := new(UpgradePlan) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *UpgradePlan) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD deleted file mode 100644 index 2773bcee8f8ef..0000000000000 --- a/cmd/kubeadm/app/cmd/BUILD +++ /dev/null @@ -1,127 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "cmd.go", - "completion.go", - "config.go", - "init.go", - "join.go", - "reset.go", - "token.go", - "version.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", - "//cmd/kubeadm/app/apis/output/scheme:go_default_library", - "//cmd/kubeadm/app/apis/output/v1alpha1:go_default_library", - "//cmd/kubeadm/app/cmd/alpha:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases/init:go_default_library", - "//cmd/kubeadm/app/cmd/phases/join:go_default_library", - "//cmd/kubeadm/app/cmd/phases/reset:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/cmd/upgrade:go_default_library", - "//cmd/kubeadm/app/cmd/util:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/discovery:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/output:go_default_library", - "//cmd/kubeadm/app/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/version:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "completion_test.go", - "config_test.go", - "init_test.go", - "join_test.go", - "token_test.go", - "version_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/output/scheme:go_default_library", - "//cmd/kubeadm/app/apis/output/v1alpha1:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/output:go_default_library", - "//cmd/kubeadm/app/util/runtime:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/cmd/alpha:all-srcs", - "//cmd/kubeadm/app/cmd/options:all-srcs", - "//cmd/kubeadm/app/cmd/phases:all-srcs", - "//cmd/kubeadm/app/cmd/upgrade:all-srcs", - "//cmd/kubeadm/app/cmd/util:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/alpha/BUILD b/cmd/kubeadm/app/cmd/alpha/BUILD deleted file mode 100644 index 6aa1659f53616..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/BUILD +++ /dev/null @@ -1,78 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "alpha.go", - "certs.go", - "kubeconfig.go", - "selfhosting.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases:go_default_library", - "//cmd/kubeadm/app/cmd/util:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", - "//cmd/kubeadm/app/phases/copycerts:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/phases/selfhosting:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = [ - "certs_test.go", - "kubeconfig_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//cmd/kubeadm/test/cmd:go_default_library", - "//cmd/kubeadm/test/kubeconfig:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/github.com/stretchr/testify/require:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/cmd/alpha/alpha.go b/cmd/kubeadm/app/cmd/alpha/alpha.go deleted file mode 100644 index a8a980fb1458a..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/alpha.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "io" - - "github.com/spf13/cobra" -) - -// NewCmdAlpha returns "kubeadm alpha" command. -func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "alpha", - Short: "Kubeadm experimental sub-commands", - } - - cmd.AddCommand(newCmdKubeConfigUtility(out)) - - const shDeprecatedMessage = "self-hosting support in kubeadm is deprecated " + - "and will be removed in a future release" - shCommand := newCmdSelfhosting(in) - shCommand.Deprecated = shDeprecatedMessage - for _, cmd := range shCommand.Commands() { - cmd.Deprecated = shDeprecatedMessage - } - cmd.AddCommand(shCommand) - - certsCommand := NewCmdCertsUtility(out) - deprecateCertsCommand(certsCommand) - cmd.AddCommand(certsCommand) - - return cmd -} - -func deprecateCertsCommand(cmds ...*cobra.Command) { - const deprecatedMessage = "please use the same command under \"kubeadm certs\"" - - for _, cmd := range cmds { - cmd.Deprecated = deprecatedMessage - childCmds := cmd.Commands() - if len(childCmds) > 0 { - deprecateCertsCommand(childCmds...) - } - } -} diff --git a/cmd/kubeadm/app/cmd/alpha/certs.go b/cmd/kubeadm/app/cmd/alpha/certs.go deleted file mode 100644 index e1fdf80c70258..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/certs.go +++ /dev/null @@ -1,473 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "fmt" - "io" - "text/tabwriter" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/util/duration" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var ( - genericCertRenewLongDesc = cmdutil.LongDesc(` - Renew the %s. - - Renewals run unconditionally, regardless of certificate expiration date; extra attributes such as SANs will - be based on the existing file/certificates, there is no need to resupply them. - - Renewal by default tries to use the certificate authority in the local PKI managed by kubeadm; as alternative - it is possible to use K8s certificate API for certificate renewal, or as a last option, to generate a CSR request. - - After renewal, in order to make changes effective, is required to restart control-plane components and - eventually re-distribute the renewed certificate in case the file is used elsewhere. -`) - - allLongDesc = cmdutil.LongDesc(` - Renew all known certificates necessary to run the control plane. Renewals are run unconditionally, regardless - of expiration date. Renewals can also be run individually for more control. -`) - - expirationLongDesc = cmdutil.LongDesc(` - Checks expiration for the certificates in the local PKI managed by kubeadm. -`) - - certificateKeyLongDesc = dedent.Dedent(` - This command will print out a secure randomly-generated certificate key that can be used with - the "init" command. - - You can also use "kubeadm init --upload-certs" without specifying a certificate key and it will - generate and print one for you. -`) - generateCSRLongDesc = cmdutil.LongDesc(` - Generates keys and certificate signing requests (CSRs) for all the certificates required to run the control plane. - This command also generates partial kubeconfig files with private key data in the "users > user > client-key-data" field, - and for each kubeconfig file an accompanying ".csr" file is created. - - This command is designed for use in [Kubeadm External CA Mode](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/#external-ca-mode). - It generates CSRs which you can then submit to your external certificate authority for signing. - - The PEM encoded signed certificates should then be saved alongside the key files, using ".crt" as the file extension, - or in the case of kubeconfig files, the PEM encoded signed certificate should be base64 encoded - and added to the kubeconfig file in the "users > user > client-certificate-data" field. -`) - generateCSRExample = cmdutil.Examples(` - # The following command will generate keys and CSRs for all control-plane certificates and kubeconfig files: - kubeadm alpha certs generate-csr --kubeconfig-dir /tmp/etc-k8s --cert-dir /tmp/etc-k8s/pki -`) -) - -// NewCmdCertsUtility returns main command for certs phase -func NewCmdCertsUtility(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "certs", - Aliases: []string{"certificates"}, - Short: "Commands related to handling kubernetes certificates", - } - - cmd.AddCommand(newCmdCertsRenewal(out)) - cmd.AddCommand(newCmdCertsExpiration(out, constants.KubernetesDir)) - cmd.AddCommand(newCmdCertificateKey()) - cmd.AddCommand(newCmdGenCSR(out)) - return cmd -} - -// genCSRConfig is the configuration required by the gencsr command -type genCSRConfig struct { - kubeadmConfigPath string - certDir string - kubeConfigDir string - kubeadmConfig *kubeadmapi.InitConfiguration -} - -func newGenCSRConfig() *genCSRConfig { - return &genCSRConfig{ - kubeConfigDir: kubeadmconstants.KubernetesDir, - } -} - -func (o *genCSRConfig) addFlagSet(flagSet *pflag.FlagSet) { - options.AddConfigFlag(flagSet, &o.kubeadmConfigPath) - options.AddCertificateDirFlag(flagSet, &o.certDir) - options.AddKubeConfigDirFlag(flagSet, &o.kubeConfigDir) -} - -// load merges command line flag values into kubeadm's config. -// Reads Kubeadm config from a file (if present) -// else use dynamically generated default config. -// This configuration contains the DNS names and IP addresses which -// are encoded in the control-plane CSRs. -func (o *genCSRConfig) load() (err error) { - o.kubeadmConfig, err = configutil.LoadOrDefaultInitConfiguration( - o.kubeadmConfigPath, - &kubeadmapiv1beta2.InitConfiguration{}, - &kubeadmapiv1beta2.ClusterConfiguration{}, - ) - if err != nil { - return err - } - // --cert-dir takes priority over kubeadm config if set. - if o.certDir != "" { - o.kubeadmConfig.CertificatesDir = o.certDir - } - return nil -} - -// newCmdGenCSR returns cobra.Command for generating keys and CSRs -func newCmdGenCSR(out io.Writer) *cobra.Command { - config := newGenCSRConfig() - - cmd := &cobra.Command{ - Use: "generate-csr", - Short: "Generate keys and certificate signing requests", - Long: generateCSRLongDesc, - Example: generateCSRExample, - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - if err := config.load(); err != nil { - return err - } - return runGenCSR(out, config) - }, - } - config.addFlagSet(cmd.Flags()) - return cmd -} - -// runGenCSR contains the logic of the generate-csr sub-command. -func runGenCSR(out io.Writer, config *genCSRConfig) error { - if err := certsphase.CreateDefaultKeysAndCSRFiles(out, config.kubeadmConfig); err != nil { - return err - } - if err := kubeconfigphase.CreateDefaultKubeConfigsAndCSRFiles(out, config.kubeConfigDir, config.kubeadmConfig); err != nil { - return err - } - return nil -} - -// newCmdCertificateKey returns cobra.Command for certificate key generate -func newCmdCertificateKey() *cobra.Command { - return &cobra.Command{ - Use: "certificate-key", - Short: "Generate certificate keys", - Long: certificateKeyLongDesc, - - RunE: func(cmd *cobra.Command, args []string) error { - key, err := copycerts.CreateCertificateKey() - if err != nil { - return err - } - fmt.Println(key) - return nil - }, - Args: cobra.NoArgs, - } -} - -// newCmdCertsRenewal creates a new `cert renew` command. -func newCmdCertsRenewal(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "renew", - Short: "Renew certificates for a Kubernetes cluster", - Long: cmdutil.MacroCommandLongDescription, - RunE: cmdutil.SubCmdRunE("renew"), - } - - cmd.AddCommand(getRenewSubCommands(out, constants.KubernetesDir)...) - - return cmd -} - -type renewFlags struct { - cfgPath string - kubeconfigPath string - cfg kubeadmapiv1beta2.ClusterConfiguration - csrOnly bool - csrPath string -} - -func getRenewSubCommands(out io.Writer, kdir string) []*cobra.Command { - flags := &renewFlags{ - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - // Setting kubernetes version to a default value in order to allow a not necessary internet lookup - KubernetesVersion: constants.CurrentKubernetesVersion.String(), - }, - kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), - } - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(&flags.cfg) - - // Get a renewal manager for a generic Cluster configuration, that is used only for getting - // the list of certificates for building subcommands - rm, err := renewal.NewManager(&kubeadmapi.ClusterConfiguration{}, "") - if err != nil { - return nil - } - - cmdList := []*cobra.Command{} - for _, handler := range rm.Certificates() { - // get the cobra.Command skeleton for this command - cmd := &cobra.Command{ - Use: handler.Name, - Short: fmt.Sprintf("Renew the %s", handler.LongName), - Long: fmt.Sprintf(genericCertRenewLongDesc, handler.LongName), - } - addRenewFlags(cmd, flags) - // get the implementation of renewing this certificate - renewalFunc := func(handler *renewal.CertificateRenewHandler) func() error { - return func() error { - // Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback) - internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "renew") - if err != nil { - return err - } - - return renewCert(flags, kdir, internalcfg, handler) - } - }(handler) - // install the implementation into the command - cmd.RunE = func(*cobra.Command, []string) error { return renewalFunc() } - cmdList = append(cmdList, cmd) - } - - allCmd := &cobra.Command{ - Use: "all", - Short: "Renew all available certificates", - Long: allLongDesc, - RunE: func(*cobra.Command, []string) error { - // Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback) - internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "renew") - if err != nil { - return err - } - - // Get a renewal manager for a actual Cluster configuration - rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir) - if err != nil { - return nil - } - - // Renew certificates - for _, handler := range rm.Certificates() { - if err := renewCert(flags, kdir, internalcfg, handler); err != nil { - return err - } - } - fmt.Printf("\nDone renewing certificates. You must restart the kube-apiserver, kube-controller-manager, kube-scheduler and etcd, so that they can use the new certificates.\n") - return nil - }, - Args: cobra.NoArgs, - } - addRenewFlags(allCmd, flags) - - cmdList = append(cmdList, allCmd) - return cmdList -} - -func addRenewFlags(cmd *cobra.Command, flags *renewFlags) { - options.AddConfigFlag(cmd.Flags(), &flags.cfgPath) - options.AddCertificateDirFlag(cmd.Flags(), &flags.cfg.CertificatesDir) - options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeconfigPath) - options.AddCSRFlag(cmd.Flags(), &flags.csrOnly) - options.AddCSRDirFlag(cmd.Flags(), &flags.csrPath) -} - -func renewCert(flags *renewFlags, kdir string, internalcfg *kubeadmapi.InitConfiguration, handler *renewal.CertificateRenewHandler) error { - // Get a renewal manager for the given cluster configuration - rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir) - if err != nil { - return err - } - - if ok, _ := rm.CertificateExists(handler.Name); !ok { - fmt.Printf("MISSING! %s\n", handler.LongName) - return nil - } - - // if the renewal operation is set to generate CSR request only - if flags.csrOnly { - // checks a path for storing CSR request is given - if flags.csrPath == "" { - return errors.New("please provide a path where CSR request should be stored") - } - return rm.CreateRenewCSR(handler.Name, flags.csrPath) - } - - // otherwise, the renewal operation has to actually renew a certificate - - // renew using local certificate authorities. - // this operation can't complete in case the certificate key is not provided (external CA) - renewed, err := rm.RenewUsingLocalCA(handler.Name) - if err != nil { - return err - } - if !renewed { - fmt.Printf("Detected external %s, %s can't be renewed\n", handler.CABaseName, handler.LongName) - return nil - } - fmt.Printf("%s renewed\n", handler.LongName) - return nil -} - -func getInternalCfg(cfgPath string, kubeconfigPath string, cfg kubeadmapiv1beta2.ClusterConfiguration, out io.Writer, logPrefix string) (*kubeadmapi.InitConfiguration, error) { - // In case the user is not providing a custom config, try to get current config from the cluster. - // NB. this operation should not block, because we want to allow certificate renewal also in case of not-working clusters - if cfgPath == "" { - client, err := kubeconfigutil.ClientSetFromFile(kubeconfigPath) - if err == nil { - internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, out, logPrefix, false, false) - if err == nil { - fmt.Println() // add empty line to separate the FetchInitConfigurationFromCluster output from the command output - return internalcfg, nil - } - fmt.Printf("[%s] Error reading configuration from the Cluster. Falling back to default configuration\n\n", logPrefix) - } - } - - // Otherwise read config from --config if provided, otherwise use default configuration - return configutil.LoadOrDefaultInitConfiguration(cfgPath, &kubeadmapiv1beta2.InitConfiguration{}, &cfg) -} - -// newCmdCertsExpiration creates a new `cert check-expiration` command. -func newCmdCertsExpiration(out io.Writer, kdir string) *cobra.Command { - flags := &expirationFlags{ - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - // Setting kubernetes version to a default value in order to allow a not necessary internet lookup - KubernetesVersion: constants.CurrentKubernetesVersion.String(), - }, - kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), - } - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(&flags.cfg) - - cmd := &cobra.Command{ - Use: "check-expiration", - Short: "Check certificates expiration for a Kubernetes cluster", - Long: expirationLongDesc, - RunE: func(cmd *cobra.Command, args []string) error { - // Get cluster configuration (from --config, kubeadm-config ConfigMap, or default as a fallback) - internalcfg, err := getInternalCfg(flags.cfgPath, flags.kubeconfigPath, flags.cfg, out, "check-expiration") - if err != nil { - return err - } - - // Get a renewal manager for the given cluster configuration - rm, err := renewal.NewManager(&internalcfg.ClusterConfiguration, kdir) - if err != nil { - return err - } - - // Get all the certificate expiration info - yesNo := func(b bool) string { - if b { - return "yes" - } - return "no" - } - w := tabwriter.NewWriter(out, 10, 4, 3, ' ', 0) - fmt.Fprintln(w, "CERTIFICATE\tEXPIRES\tRESIDUAL TIME\tCERTIFICATE AUTHORITY\tEXTERNALLY MANAGED") - for _, handler := range rm.Certificates() { - if ok, _ := rm.CertificateExists(handler.Name); ok { - e, err := rm.GetCertificateExpirationInfo(handler.Name) - if err != nil { - return err - } - - s := fmt.Sprintf("%s\t%s\t%s\t%s\t%-8v", - e.Name, - e.ExpirationDate.Format("Jan 02, 2006 15:04 MST"), - duration.ShortHumanDuration(e.ResidualTime()), - handler.CAName, - yesNo(e.ExternallyManaged), - ) - - fmt.Fprintln(w, s) - continue - } - - // the certificate does not exist (for any reason) - s := fmt.Sprintf("!MISSING! %s\t\t\t\t", - handler.Name, - ) - fmt.Fprintln(w, s) - } - fmt.Fprintln(w) - fmt.Fprintln(w, "CERTIFICATE AUTHORITY\tEXPIRES\tRESIDUAL TIME\tEXTERNALLY MANAGED") - for _, handler := range rm.CAs() { - if ok, _ := rm.CAExists(handler.Name); ok { - e, err := rm.GetCAExpirationInfo(handler.Name) - if err != nil { - return err - } - - s := fmt.Sprintf("%s\t%s\t%s\t%-8v", - e.Name, - e.ExpirationDate.Format("Jan 02, 2006 15:04 MST"), - duration.ShortHumanDuration(e.ResidualTime()), - yesNo(e.ExternallyManaged), - ) - - fmt.Fprintln(w, s) - continue - } - - // the CA does not exist (for any reason) - s := fmt.Sprintf("!MISSING! %s\t\t\t", - handler.Name, - ) - fmt.Fprintln(w, s) - } - w.Flush() - return nil - }, - Args: cobra.NoArgs, - } - addExpirationFlags(cmd, flags) - - return cmd -} - -type expirationFlags struct { - cfgPath string - kubeconfigPath string - cfg kubeadmapiv1beta2.ClusterConfiguration -} - -func addExpirationFlags(cmd *cobra.Command, flags *expirationFlags) { - options.AddConfigFlag(cmd.Flags(), &flags.cfgPath) - options.AddCertificateDirFlag(cmd.Flags(), &flags.cfg.CertificatesDir) - options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeconfigPath) -} diff --git a/cmd/kubeadm/app/cmd/alpha/certs_test.go b/cmd/kubeadm/app/cmd/alpha/certs_test.go deleted file mode 100644 index 59e8f0f47b27e..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/certs_test.go +++ /dev/null @@ -1,476 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "crypto" - "crypto/x509" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "k8s.io/client-go/tools/clientcmd" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" - cmdtestutil "k8s.io/kubernetes/cmd/kubeadm/test/cmd" -) - -func TestCommandsGenerated(t *testing.T) { - expectedFlags := []string{ - "cert-dir", - "config", - } - - expectedCommands := []string{ - "renew all", - - "renew apiserver", - "renew apiserver-kubelet-client", - "renew apiserver-etcd-client", - - "renew front-proxy-client", - - "renew etcd-server", - "renew etcd-peer", - "renew etcd-healthcheck-client", - - "renew admin.conf", - "renew scheduler.conf", - "renew controller-manager.conf", - } - - renewCmd := newCmdCertsRenewal(os.Stdout) - - fakeRoot := &cobra.Command{} - fakeRoot.AddCommand(renewCmd) - - for _, cmdPath := range expectedCommands { - t.Run(cmdPath, func(t *testing.T) { - cmd, rem, _ := fakeRoot.Find(strings.Split(cmdPath, " ")) - if cmd == nil || len(rem) != 0 { - t.Fatalf("couldn't locate command %q (%v)", cmdPath, rem) - } - - for _, flag := range expectedFlags { - if cmd.Flags().Lookup(flag) == nil { - t.Errorf("couldn't find expected flag --%s", flag) - } - } - }) - } -} - -func TestRunRenewCommands(t *testing.T) { - tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) - - cfg := testutil.GetDefaultInternalConfig(t) - cfg.CertificatesDir = tmpDir - - // Generate all the CA - CACerts := map[string]*x509.Certificate{} - CAKeys := map[string]crypto.Signer{} - for _, ca := range []*certsphase.KubeadmCert{ - certsphase.KubeadmCertRootCA(), - certsphase.KubeadmCertFrontProxyCA(), - certsphase.KubeadmCertEtcdCA(), - } { - caCert, caKey, err := ca.CreateAsCA(cfg) - if err != nil { - t.Fatalf("couldn't write out CA %s: %v", ca.Name, err) - } - CACerts[ca.Name] = caCert - CAKeys[ca.Name] = caKey - } - - // Generate all the signed certificates - for _, cert := range []*certsphase.KubeadmCert{ - certsphase.KubeadmCertAPIServer(), - certsphase.KubeadmCertKubeletClient(), - certsphase.KubeadmCertFrontProxyClient(), - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertEtcdServer(), - certsphase.KubeadmCertEtcdPeer(), - certsphase.KubeadmCertEtcdHealthcheck(), - } { - caCert := CACerts[cert.CAName] - caKey := CAKeys[cert.CAName] - if err := cert.CreateFromCA(cfg, caCert, caKey); err != nil { - t.Fatalf("couldn't write certificate %s: %v", cert.Name, err) - } - } - - // Generate all the kubeconfig files with embedded certs - for _, kubeConfig := range []string{ - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.SchedulerKubeConfigFileName, - kubeadmconstants.ControllerManagerKubeConfigFileName, - } { - if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpDir, cfg); err != nil { - t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err) - } - } - - tests := []struct { - command string - Certs []*certsphase.KubeadmCert - KubeconfigFiles []string - }{ - { - command: "all", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertAPIServer(), - certsphase.KubeadmCertKubeletClient(), - certsphase.KubeadmCertFrontProxyClient(), - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertEtcdServer(), - certsphase.KubeadmCertEtcdPeer(), - certsphase.KubeadmCertEtcdHealthcheck(), - }, - KubeconfigFiles: []string{ - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.SchedulerKubeConfigFileName, - kubeadmconstants.ControllerManagerKubeConfigFileName, - }, - }, - { - command: "apiserver", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertAPIServer(), - }, - }, - { - command: "apiserver-kubelet-client", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertKubeletClient(), - }, - }, - { - command: "apiserver-etcd-client", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdAPIClient(), - }, - }, - { - command: "front-proxy-client", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertFrontProxyClient(), - }, - }, - { - command: "etcd-server", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdServer(), - }, - }, - { - command: "etcd-peer", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdPeer(), - }, - }, - { - command: "etcd-healthcheck-client", - Certs: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdHealthcheck(), - }, - }, - { - command: "admin.conf", - KubeconfigFiles: []string{ - kubeadmconstants.AdminKubeConfigFileName, - }, - }, - { - command: "scheduler.conf", - KubeconfigFiles: []string{ - kubeadmconstants.SchedulerKubeConfigFileName, - }, - }, - { - command: "controller-manager.conf", - KubeconfigFiles: []string{ - kubeadmconstants.ControllerManagerKubeConfigFileName, - }, - }, - } - - for _, test := range tests { - t.Run(test.command, func(t *testing.T) { - // Get file ModTime before renew - ModTime := map[string]time.Time{} - for _, cert := range test.Certs { - file, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("%s.crt", cert.BaseName))) - if err != nil { - t.Fatalf("couldn't get certificate %s: %v", cert.Name, err) - } - ModTime[cert.Name] = file.ModTime() - } - for _, kubeConfig := range test.KubeconfigFiles { - file, err := os.Stat(filepath.Join(tmpDir, kubeConfig)) - if err != nil { - t.Fatalf("couldn't get kubeconfig %s: %v", kubeConfig, err) - } - ModTime[kubeConfig] = file.ModTime() - } - - // exec renew - renewCmds := getRenewSubCommands(os.Stdout, tmpDir) - cmdtestutil.RunSubCommand(t, renewCmds, test.command, fmt.Sprintf("--cert-dir=%s", tmpDir)) - - // check the file is modified - for _, cert := range test.Certs { - file, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("%s.crt", cert.BaseName))) - if err != nil { - t.Fatalf("couldn't get certificate %s: %v", cert.Name, err) - } - if ModTime[cert.Name] == file.ModTime() { - t.Errorf("certificate %s was not renewed as expected", cert.Name) - } - } - for _, kubeConfig := range test.KubeconfigFiles { - file, err := os.Stat(filepath.Join(tmpDir, kubeConfig)) - if err != nil { - t.Fatalf("couldn't get kubeconfig %s: %v", kubeConfig, err) - } - if ModTime[kubeConfig] == file.ModTime() { - t.Errorf("kubeconfig %s was not renewed as expected", kubeConfig) - } - } - }) - } -} - -func TestRenewUsingCSR(t *testing.T) { - tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) - cert := certsphase.KubeadmCertEtcdServer() - - cfg := testutil.GetDefaultInternalConfig(t) - cfg.CertificatesDir = tmpDir - - caCert, caKey, err := certsphase.KubeadmCertEtcdCA().CreateAsCA(cfg) - if err != nil { - t.Fatalf("couldn't write out CA %s: %v", certsphase.KubeadmCertEtcdCA().Name, err) - } - - if err := cert.CreateFromCA(cfg, caCert, caKey); err != nil { - t.Fatalf("couldn't write certificate %s: %v", cert.Name, err) - } - - renewCmds := getRenewSubCommands(os.Stdout, tmpDir) - cmdtestutil.RunSubCommand(t, renewCmds, cert.Name, "--csr-only", "--csr-dir="+tmpDir, fmt.Sprintf("--cert-dir=%s", tmpDir)) - - if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(tmpDir, cert.Name); err != nil { - t.Fatalf("couldn't load certificate %q: %v", cert.Name, err) - } -} - -func TestRunGenCSR(t *testing.T) { - tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) - - kubeConfigDir := filepath.Join(tmpDir, "kubernetes") - certDir := kubeConfigDir + "/pki" - - expectedCertificates := []string{ - "apiserver", - "apiserver-etcd-client", - "apiserver-kubelet-client", - "front-proxy-client", - "etcd/healthcheck-client", - "etcd/peer", - "etcd/server", - } - - expectedKubeConfigs := []string{ - "admin", - "kubelet", - "controller-manager", - "scheduler", - } - - config := genCSRConfig{ - kubeConfigDir: kubeConfigDir, - kubeadmConfig: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "192.0.2.1", - BindPort: 443, - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - ServiceSubnet: "192.0.2.0/24", - }, - CertificatesDir: certDir, - KubernetesVersion: "v1.19.0", - }, - }, - } - - err := runGenCSR(nil, &config) - require.NoError(t, err, "expected runGenCSR to not fail") - - t.Log("The command generates key and CSR files in the configured --cert-dir") - for _, name := range expectedCertificates { - _, err = pkiutil.TryLoadKeyFromDisk(certDir, name) - assert.NoErrorf(t, err, "failed to load key file: %s", name) - - _, err = pkiutil.TryLoadCSRFromDisk(certDir, name) - assert.NoError(t, err, "failed to load CSR file: %s", name) - } - - t.Log("The command generates kubeconfig files in the configured --kubeconfig-dir") - for _, name := range expectedKubeConfigs { - _, err = clientcmd.LoadFromFile(kubeConfigDir + "/" + name + ".conf") - assert.NoErrorf(t, err, "failed to load kubeconfig file: %s", name) - - _, err = pkiutil.TryLoadCSRFromDisk(kubeConfigDir, name+".conf") - assert.NoError(t, err, "failed to load kubeconfig CSR file: %s", name) - } -} - -func TestGenCSRConfig(t *testing.T) { - type assertion func(*testing.T, *genCSRConfig) - - hasCertDir := func(expected string) assertion { - return func(t *testing.T, config *genCSRConfig) { - assert.Equal(t, expected, config.kubeadmConfig.CertificatesDir) - } - } - hasKubeConfigDir := func(expected string) assertion { - return func(t *testing.T, config *genCSRConfig) { - assert.Equal(t, expected, config.kubeConfigDir) - } - } - hasAdvertiseAddress := func(expected string) assertion { - return func(t *testing.T, config *genCSRConfig) { - assert.Equal(t, expected, config.kubeadmConfig.LocalAPIEndpoint.AdvertiseAddress) - } - } - - // A minimal kubeadm config with just enough values to avoid triggering - // auto-detection of config values at runtime. - const kubeadmConfig = ` -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: 192.0.2.1 -nodeRegistration: - criSocket: /path/to/dockershim.sock ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -certificatesDir: /custom/config/certificates-dir -kubernetesVersion: v1.19.0 -` - - tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) - - customConfigPath := tmpDir + "/kubeadm.conf" - - f, err := os.Create(customConfigPath) - require.NoError(t, err) - _, err = f.Write([]byte(kubeadmConfig)) - require.NoError(t, err) - - tests := []struct { - name string - flags []string - assertions []assertion - expectErr bool - }{ - { - name: "default", - assertions: []assertion{ - hasCertDir(kubeadmapiv1beta2.DefaultCertificatesDir), - hasKubeConfigDir(kubeadmconstants.KubernetesDir), - }, - }, - { - name: "--cert-dir overrides default", - flags: []string{"--cert-dir", "/foo/bar/pki"}, - assertions: []assertion{ - hasCertDir("/foo/bar/pki"), - }, - }, - { - name: "--config is loaded", - flags: []string{"--config", customConfigPath}, - assertions: []assertion{ - hasCertDir("/custom/config/certificates-dir"), - hasAdvertiseAddress("192.0.2.1"), - }, - }, - { - name: "--config not found", - flags: []string{"--config", "/does/not/exist"}, - expectErr: true, - }, - { - name: "--cert-dir overrides --config certificatesDir", - flags: []string{ - "--config", customConfigPath, - "--cert-dir", "/foo/bar/pki", - }, - assertions: []assertion{ - hasCertDir("/foo/bar/pki"), - hasAdvertiseAddress("192.0.2.1"), - }, - }, - { - name: "--kubeconfig-dir overrides default", - flags: []string{ - "--kubeconfig-dir", "/foo/bar/kubernetes", - }, - assertions: []assertion{ - hasKubeConfigDir("/foo/bar/kubernetes"), - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - flagset := pflag.NewFlagSet("flags-for-gencsr", pflag.ContinueOnError) - config := newGenCSRConfig() - config.addFlagSet(flagset) - require.NoError(t, flagset.Parse(test.flags)) - - err := config.load() - if test.expectErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - for _, assertFunc := range test.assertions { - assertFunc(t, config) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/alpha/kubeconfig.go b/cmd/kubeadm/app/cmd/alpha/kubeconfig.go deleted file mode 100644 index aa168b8585040..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/kubeconfig.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "io" - - "github.com/spf13/cobra" - - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -var ( - kubeconfigLongDesc = cmdutil.LongDesc(` - Kubeconfig file utilities. - ` + cmdutil.AlphaDisclaimer) - - userKubeconfigLongDesc = cmdutil.LongDesc(` - Output a kubeconfig file for an additional user. - ` + cmdutil.AlphaDisclaimer) - - userKubeconfigExample = cmdutil.Examples(` - # Output a kubeconfig file for an additional user named foo using a kubeadm config file bar - kubeadm alpha kubeconfig user --client-name=foo --config=bar - `) -) - -// newCmdKubeConfigUtility returns main command for kubeconfig phase -func newCmdKubeConfigUtility(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "kubeconfig", - Short: "Kubeconfig file utilities", - Long: kubeconfigLongDesc, - } - - cmd.AddCommand(newCmdUserKubeConfig(out)) - return cmd -} - -// newCmdUserKubeConfig returns sub commands for kubeconfig phase -func newCmdUserKubeConfig(out io.Writer) *cobra.Command { - - initCfg := &kubeadmapiv1beta2.InitConfiguration{} - clusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{} - - var ( - token, clientName, cfgPath string - organizations []string - ) - - // Creates the UX Command - cmd := &cobra.Command{ - Use: "user", - Short: "Output a kubeconfig file for an additional user", - Long: userKubeconfigLongDesc, - Example: userKubeconfigExample, - RunE: func(cmd *cobra.Command, args []string) error { - // This call returns the ready-to-use configuration based on the defaults populated by flags - internalCfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, initCfg, clusterCfg) - if err != nil { - return err - } - - // if the kubeconfig file for an additional user has to use a token, use it - if token != "" { - return kubeconfigphase.WriteKubeConfigWithToken(out, internalCfg, clientName, token) - } - - // Otherwise, write a kubeconfig file with a generate client cert - return kubeconfigphase.WriteKubeConfigWithClientCert(out, internalCfg, clientName, organizations) - }, - Args: cobra.NoArgs, - } - - options.AddConfigFlag(cmd.Flags(), &cfgPath) - - // Add command specific flags - cmd.Flags().StringVar(&token, options.TokenStr, token, "The token that should be used as the authentication mechanism for this kubeconfig, instead of client certificates") - cmd.Flags().StringVar(&clientName, "client-name", clientName, "The name of user. It will be used as the CN if client certificates are created") - cmd.Flags().StringSliceVar(&organizations, "org", organizations, "The orgnizations of the client certificate. It will be used as the O if client certificates are created") - - cmd.MarkFlagRequired(options.CfgPath) - cmd.MarkFlagRequired("client-name") - return cmd -} diff --git a/cmd/kubeadm/app/cmd/alpha/kubeconfig_test.go b/cmd/kubeadm/app/cmd/alpha/kubeconfig_test.go deleted file mode 100644 index 132ca498c90d2..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/kubeconfig_test.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/clientcmd" - "sigs.k8s.io/yaml" - - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" - kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig" -) - -func generateTestKubeadmConfig(dir, id, certDir, clusterName string) (string, error) { - cfgPath := filepath.Join(dir, id) - initCfg := kubeadmapiv1beta2.InitConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubeadm.k8s.io/v1beta2", - Kind: "InitConfiguration", - }, - LocalAPIEndpoint: kubeadmapiv1beta2.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 1234, - }, - } - clusterCfg := kubeadmapiv1beta2.ClusterConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubeadm.k8s.io/v1beta2", - Kind: "ClusterConfiguration", - }, - CertificatesDir: certDir, - ClusterName: clusterName, - KubernetesVersion: "v1.19.0", - } - - var buf bytes.Buffer - data, err := yaml.Marshal(&initCfg) - if err != nil { - return "", err - } - buf.Write(data) - buf.WriteString("---\n") - data, err = yaml.Marshal(&clusterCfg) - if err != nil { - return "", err - } - buf.Write(data) - - err = ioutil.WriteFile(cfgPath, buf.Bytes(), 0644) - return cfgPath, err -} - -func TestKubeConfigSubCommandsThatWritesToOut(t *testing.T) { - - // Temporary folders for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Adds a pki folder with a ca cert to the temp folder - pkidir := testutil.SetupPkiDirWithCertificateAuthority(t, tmpdir) - - // Retrieves ca cert for assertions - caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName) - if err != nil { - t.Fatalf("couldn't retrieve ca cert: %v", err) - } - - var tests = []struct { - name string - command string - clusterName string - withClientCert bool - withToken bool - additionalFlags []string - }{ - { - name: "user subCommand withClientCert", - command: "user", - withClientCert: true, - }, - { - name: "user subCommand withClientCert", - command: "user", - withClientCert: true, - clusterName: "my-cluster", - }, - { - name: "user subCommand withToken", - withToken: true, - command: "user", - additionalFlags: []string{"--token=123456"}, - }, - { - name: "user subCommand withToken", - withToken: true, - command: "user", - clusterName: "my-cluster-with-token", - additionalFlags: []string{"--token=123456"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - buf := new(bytes.Buffer) - - // Get subcommands working in the temporary directory - cmd := newCmdUserKubeConfig(buf) - - cfgPath, err := generateTestKubeadmConfig(tmpdir, test.name, pkidir, test.clusterName) - if err != nil { - t.Fatalf("Failed to generate kubeadm config: %v", err) - } - - commonFlags := []string{ - "--client-name=myUser", - fmt.Sprintf("--config=%s", cfgPath), - } - - // Execute the subcommand - allFlags := append(commonFlags, test.additionalFlags...) - cmd.SetArgs(allFlags) - if err := cmd.Execute(); err != nil { - t.Fatalf("Could not execute subcommand: %v", err) - } - - // reads kubeconfig written to stdout - config, err := clientcmd.Load(buf.Bytes()) - if err != nil { - t.Fatalf("couldn't read kubeconfig file from buffer: %v", err) - } - - // checks that CLI flags are properly propagated - kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) - - if test.withClientCert { - // checks that kubeconfig files have expected client cert - kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myUser") - } - - if test.withToken { - // checks that kubeconfig files have expected token - kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myUser", "123456") - } - - if len(test.clusterName) > 0 { - // checks that kubeconfig files have expected cluster name - kubeconfigtestutil.AssertKubeConfigCurrentContextWithClusterName(t, config, test.clusterName) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/alpha/selfhosting.go b/cmd/kubeadm/app/cmd/alpha/selfhosting.go deleted file mode 100644 index 5617b6abc0a83..0000000000000 --- a/cmd/kubeadm/app/cmd/alpha/selfhosting.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package alpha - -import ( - "bufio" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var ( - selfhostingLongDesc = cmdutil.LongDesc(` - Convert static Pod files for control plane components into self-hosted DaemonSets configured via the Kubernetes API. - - See the documentation for self-hosting limitations. - - ` + cmdutil.AlphaDisclaimer) - - selfhostingExample = cmdutil.Examples(` - # Convert a static Pod-hosted control plane into a self-hosted one. - - kubeadm alpha phase self-hosting convert-from-staticpods - `) -) - -// newCmdSelfhosting returns the self-hosting Cobra command -func newCmdSelfhosting(in io.Reader) *cobra.Command { - cmd := &cobra.Command{ - Use: "selfhosting", - Aliases: []string{"selfhosted", "self-hosting"}, - Short: "Make a kubeadm cluster self-hosted", - Long: cmdutil.MacroCommandLongDescription, - } - - cmd.AddCommand(getSelfhostingSubCommand(in)) - return cmd -} - -// getSelfhostingSubCommand returns sub commands for Self-hosting phase -func getSelfhostingSubCommand(in io.Reader) *cobra.Command { - - cfg := &kubeadmapiv1beta2.ClusterConfiguration{} - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath, featureGatesString, kubeConfigFile string - forcePivot, certsInSecrets := false, false - - // Creates the UX Command - cmd := &cobra.Command{ - Use: "pivot", - Aliases: []string{"from-staticpods"}, - Short: "Convert a static Pod-hosted control plane into a self-hosted one", - Long: selfhostingLongDesc, - Example: selfhostingExample, - RunE: func(cmd *cobra.Command, args []string) error { - - var err error - - if !forcePivot { - fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!") - fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ") - s := bufio.NewScanner(in) - s.Scan() - - if err = s.Err(); err != nil { - return err - } - - if strings.ToLower(s.Text()) != "y" { - return errors.New("aborted pivot operation") - } - } - - fmt.Println("[pivot] pivoting cluster to self-hosted") - - if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil { - return err - } - - if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { - return err - } - - // Gets the Kubernetes client - kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile) - client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile) - if err != nil { - return err - } - - // KubernetesVersion is not used, but we set it explicitly to avoid the lookup - // of the version from the internet when executing LoadOrDefaultInitConfiguration - phases.SetKubernetesVersion(cfg) - - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, &kubeadmapiv1beta2.InitConfiguration{}, cfg) - if err != nil { - return err - } - - // Converts the Static Pod-hosted control plane into a self-hosted one - waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout) - return selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets) - }, - Args: cobra.NoArgs, - } - - // Add flags to the command - // flags bound to the configuration object - cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`) - options.AddConfigFlag(cmd.Flags(), &cfgPath) - - cmd.Flags().BoolVarP( - &certsInSecrets, "store-certs-in-secrets", "s", - false, "Enable storing certs in secrets") - - cmd.Flags().BoolVarP( - &forcePivot, "force", "f", false, - "Pivot the cluster without prompting for confirmation", - ) - - // flags that are not bound to the configuration object - // Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go - options.AddKubeConfigFlag(cmd.Flags(), &kubeConfigFile) - - return cmd -} diff --git a/cmd/kubeadm/app/cmd/cmd.go b/cmd/kubeadm/app/cmd/cmd.go deleted file mode 100644 index 81a84c9460353..0000000000000 --- a/cmd/kubeadm/app/cmd/cmd.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "io" - - "github.com/lithammer/dedent" - "github.com/spf13/cobra" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/upgrade" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - // Register the kubeadm configuration types because CLI flag generation - // depends on the generated defaults. -) - -// NewKubeadmCommand returns cobra.Command to run kubeadm command -func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command { - var rootfsPath string - - cmds := &cobra.Command{ - Use: "kubeadm", - Short: "kubeadm: easily bootstrap a secure Kubernetes cluster", - Long: dedent.Dedent(` - - ┌──────────────────────────────────────────────────────────┐ - │ KUBEADM │ - │ Easily bootstrap a secure Kubernetes cluster │ - │ │ - │ Please give us feedback at: │ - │ https://github.com/kubernetes/kubeadm/issues │ - └──────────────────────────────────────────────────────────┘ - - Example usage: - - Create a two-machine cluster with one control-plane node - (which controls the cluster), and one worker node - (where your workloads, like Pods and Deployments run). - - ┌──────────────────────────────────────────────────────────┐ - │ On the first machine: │ - ├──────────────────────────────────────────────────────────┤ - │ control-plane# kubeadm init │ - └──────────────────────────────────────────────────────────┘ - - ┌──────────────────────────────────────────────────────────┐ - │ On the second machine: │ - ├──────────────────────────────────────────────────────────┤ - │ worker# kubeadm join │ - └──────────────────────────────────────────────────────────┘ - - You can then repeat the second step on as many other machines as you like. - - `), - SilenceErrors: true, - SilenceUsage: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if rootfsPath != "" { - if err := kubeadmutil.Chroot(rootfsPath); err != nil { - return err - } - } - return nil - }, - } - - cmds.ResetFlags() - - cmds.AddCommand(newCmdCompletion(out, "")) - cmds.AddCommand(newCmdConfig(out)) - cmds.AddCommand(newCmdInit(out, nil)) - cmds.AddCommand(newCmdJoin(out, nil)) - cmds.AddCommand(newCmdReset(in, out, nil)) - cmds.AddCommand(newCmdVersion(out)) - cmds.AddCommand(newCmdToken(out, err)) - cmds.AddCommand(upgrade.NewCmdUpgrade(out)) - cmds.AddCommand(alpha.NewCmdAlpha(in, out)) - options.AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath) - - // TODO: remove "certs" from "alpha" - // https://github.com/kubernetes/kubeadm/issues/2291 - cmds.AddCommand(alpha.NewCmdCertsUtility(out)) - - return cmds -} diff --git a/cmd/kubeadm/app/cmd/completion.go b/cmd/kubeadm/app/cmd/completion.go deleted file mode 100644 index 4ff1d63dbaaaf..0000000000000 --- a/cmd/kubeadm/app/cmd/completion.go +++ /dev/null @@ -1,299 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "io" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "k8s.io/klog/v2" -) - -const defaultBoilerPlate = ` -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -` - -var ( - completionLong = dedent.Dedent(` - Output shell completion code for the specified shell (bash or zsh). - The shell code must be evaluated to provide interactive - completion of kubeadm commands. This can be done by sourcing it from - the .bash_profile. - - Note: this requires the bash-completion framework. - - To install it on Mac use homebrew: - $ brew install bash-completion - Once installed, bash_completion must be evaluated. This can be done by adding the - following line to the .bash_profile - $ source $(brew --prefix)/etc/bash_completion - - If bash-completion is not installed on Linux, please install the 'bash-completion' package - via your distribution's package manager. - - Note for zsh users: [1] zsh completions are only supported in versions of zsh >= 5.2`) - - completionExample = dedent.Dedent(` - # Install bash completion on a Mac using homebrew - brew install bash-completion - printf "\n# Bash completion support\nsource $(brew --prefix)/etc/bash_completion\n" >> $HOME/.bash_profile - source $HOME/.bash_profile - - # Load the kubeadm completion code for bash into the current shell - source <(kubeadm completion bash) - - # Write bash completion code to a file and source it from .bash_profile - kubeadm completion bash > ~/.kube/kubeadm_completion.bash.inc - printf "\n# Kubeadm shell completion\nsource '$HOME/.kube/kubeadm_completion.bash.inc'\n" >> $HOME/.bash_profile - source $HOME/.bash_profile - - # Load the kubeadm completion code for zsh[1] into the current shell - source <(kubeadm completion zsh)`) -) - -var ( - completionShells = map[string]func(out io.Writer, cmd *cobra.Command) error{ - "bash": runCompletionBash, - "zsh": runCompletionZsh, - } -) - -// GetSupportedShells returns a list of supported shells -func GetSupportedShells() []string { - shells := []string{} - for s := range completionShells { - shells = append(shells, s) - } - return shells -} - -// newCmdCompletion returns the "kubeadm completion" command -func newCmdCompletion(out io.Writer, boilerPlate string) *cobra.Command { - cmd := &cobra.Command{ - Use: "completion SHELL", - Short: "Output shell completion code for the specified shell (bash or zsh)", - Long: completionLong, - Example: completionExample, - RunE: func(cmd *cobra.Command, args []string) error { - return RunCompletion(out, boilerPlate, cmd, args) - }, - ValidArgs: GetSupportedShells(), - } - - return cmd -} - -// RunCompletion checks given arguments and executes command -func RunCompletion(out io.Writer, boilerPlate string, cmd *cobra.Command, args []string) error { - if length := len(args); length == 0 { - return errors.New("shell not specified") - } else if length > 1 { - return errors.New("too many arguments. expected only the shell type") - } - run, found := completionShells[args[0]] - if !found { - return errors.Errorf("unsupported shell type %q", args[0]) - } - - if len(boilerPlate) == 0 { - boilerPlate = defaultBoilerPlate - } - if _, err := out.Write([]byte(boilerPlate)); err != nil { - return err - } - return run(out, cmd.Parent()) -} - -func runCompletionBash(out io.Writer, kubeadm *cobra.Command) error { - klog.V(1).Infoln("[completion] writing completion code for Bash") - return kubeadm.GenBashCompletion(out) -} - -func runCompletionZsh(out io.Writer, kubeadm *cobra.Command) error { - zshInitialization := ` -__kubeadm_bash_source() { - alias shopt=':' - alias _expand=_bash_expand - alias _complete=_bash_comp - emulate -L sh - setopt kshglob noshglob braceexpand - - source "$@" -} - -__kubeadm_type() { - # -t is not supported by zsh - if [ "$1" == "-t" ]; then - shift - - # fake Bash 4 to disable "complete -o nospace". Instead - # "compopt +-o nospace" is used in the code to toggle trailing - # spaces. We don't support that, but leave trailing spaces on - # all the time - if [ "$1" = "__kubeadm_compopt" ]; then - echo builtin - return 0 - fi - fi - type "$@" -} - -__kubeadm_compgen() { - local completions w - completions=( $(compgen "$@") ) || return $? - - # filter by given word as prefix - while [[ "$1" = -* && "$1" != -- ]]; do - shift - shift - done - if [[ "$1" == -- ]]; then - shift - fi - for w in "${completions[@]}"; do - if [[ "${w}" = "$1"* ]]; then - echo "${w}" - fi - done -} - -__kubeadm_compopt() { - true # don't do anything. Not supported by bashcompinit in zsh -} - -__kubeadm_ltrim_colon_completions() -{ - if [[ "$1" == *:* && "$COMP_WORDBREAKS" == *:* ]]; then - # Remove colon-word prefix from COMPREPLY items - local colon_word=${1%${1##*:}} - local i=${#COMPREPLY[*]} - while [[ $((--i)) -ge 0 ]]; do - COMPREPLY[$i]=${COMPREPLY[$i]#"$colon_word"} - done - fi -} - -__kubeadm_get_comp_words_by_ref() { - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[${COMP_CWORD}-1]}" - words=("${COMP_WORDS[@]}") - cword=("${COMP_CWORD[@]}") -} - -__kubeadm_filedir() { - local RET OLD_IFS w qw - - __kubectl_debug "_filedir $@ cur=$cur" - if [[ "$1" = \~* ]]; then - # somehow does not work. Maybe, zsh does not call this at all - eval echo "$1" - return 0 - fi - - OLD_IFS="$IFS" - IFS=$'\n' - if [ "$1" = "-d" ]; then - shift - RET=( $(compgen -d) ) - else - RET=( $(compgen -f) ) - fi - IFS="$OLD_IFS" - - IFS="," __kubectl_debug "RET=${RET[@]} len=${#RET[@]}" - - for w in ${RET[@]}; do - if [[ ! "${w}" = "${cur}"* ]]; then - continue - fi - if eval "[[ \"\${w}\" = *.$1 || -d \"\${w}\" ]]"; then - qw="$(__kubeadm_quote "${w}")" - if [ -d "${w}" ]; then - COMPREPLY+=("${qw}/") - else - COMPREPLY+=("${qw}") - fi - fi - done -} - -__kubeadm_quote() { - if [[ $1 == \'* || $1 == \"* ]]; then - # Leave out first character - printf %q "${1:1}" - else - printf %q "$1" - fi -} - -autoload -U +X bashcompinit && bashcompinit - -# use word boundary patterns for BSD or GNU sed -LWORD='[[:<:]]' -RWORD='[[:>:]]' -if sed --version 2>&1 | grep -q GNU; then - LWORD='\<' - RWORD='\>' -fi - -__kubeadm_convert_bash_to_zsh() { - sed \ - -e 's/declare -F/whence -w/' \ - -e 's/local \([a-zA-Z0-9_]*\)=/local \1; \1=/' \ - -e 's/flags+=("\(--.*\)=")/flags+=("\1"); two_word_flags+=("\1")/' \ - -e 's/must_have_one_flag+=("\(--.*\)=")/must_have_one_flag+=("\1")/' \ - -e "s/${LWORD}_filedir${RWORD}/__kubeadm_filedir/g" \ - -e "s/${LWORD}_get_comp_words_by_ref${RWORD}/__kubeadm_get_comp_words_by_ref/g" \ - -e "s/${LWORD}__ltrim_colon_completions${RWORD}/__kubeadm_ltrim_colon_completions/g" \ - -e "s/${LWORD}compgen${RWORD}/__kubeadm_compgen/g" \ - -e "s/${LWORD}compopt${RWORD}/__kubeadm_compopt/g" \ - -e "s/${LWORD}declare${RWORD}/builtin declare/g" \ - -e "s/\\\$(type${RWORD}/\$(__kubeadm_type/g" \ - <<'BASH_COMPLETION_EOF' -` - klog.V(1).Infoln("[completion] writing completion code for Zsh") - out.Write([]byte(zshInitialization)) - - buf := new(bytes.Buffer) - kubeadm.GenBashCompletion(buf) - klog.V(1).Infoln("[completion] writing completion code for Bash") - out.Write(buf.Bytes()) - - zshTail := ` -BASH_COMPLETION_EOF -} - -__kubeadm_bash_source <(__kubeadm_convert_bash_to_zsh) -` - out.Write([]byte(zshTail)) - return nil -} diff --git a/cmd/kubeadm/app/cmd/completion_test.go b/cmd/kubeadm/app/cmd/completion_test.go deleted file mode 100644 index 2955daa380ba6..0000000000000 --- a/cmd/kubeadm/app/cmd/completion_test.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "testing" - - "github.com/spf13/cobra" -) - -const shellsError = "Unexpected empty completion shells list" - -func TestNewCmdCompletion(t *testing.T) { - var out bytes.Buffer - shells := GetSupportedShells() - if len(shells) == 0 { - t.Errorf(shellsError) - } - // test newCmdCompletion with a valid shell. - // use a dummy parent command as newCmdCompletion needs it. - parentCmd := &cobra.Command{} - args := []string{"completion", shells[0]} - parentCmd.SetArgs(args) - cmd := newCmdCompletion(&out, "") - parentCmd.AddCommand(cmd) - if err := parentCmd.Execute(); err != nil { - t.Errorf("Cannot exectute newCmdCompletion: %v", err) - } -} - -func TestRunCompletion(t *testing.T) { - var out bytes.Buffer - type TestCase struct { - name string - args []string - expectedError bool - } - - testCases := []TestCase{ - { - name: "invalid: missing argument", - args: []string{}, - expectedError: true, - }, - { - name: "invalid: too many arguments", - args: []string{"", ""}, - expectedError: true, - }, - { - name: "invalid: unsupported shell name", - args: []string{"unsupported"}, - expectedError: true, - }, - } - - // test all supported shells - shells := GetSupportedShells() - if len(shells) == 0 { - t.Errorf(shellsError) - } - for _, shell := range shells { - test := TestCase{ - name: "valid: test shell " + shell, - args: []string{shell}, - } - testCases = append(testCases, test) - } - - // use dummy cobra commands - parentCmd := &cobra.Command{} - cmd := &cobra.Command{} - parentCmd.AddCommand(cmd) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - if err := RunCompletion(&out, "", cmd, tc.args); (err != nil) != tc.expectedError { - t.Errorf("Test case %q: TestRunCompletion expected error: %v, saw: %v", tc.name, tc.expectedError, (err != nil)) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go deleted file mode 100644 index 5f722836734d3..0000000000000 --- a/cmd/kubeadm/app/cmd/config.go +++ /dev/null @@ -1,524 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "sort" - "strings" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "k8s.io/klog/v2" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/cli-runtime/pkg/genericclioptions" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" - outputapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha1" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" - utilsexec "k8s.io/utils/exec" -) - -var ( - // placeholderToken is only set statically to make kubeadm not randomize the token on every run - placeholderToken = kubeadmapiv1beta2.BootstrapToken{ - Token: &kubeadmapiv1beta2.BootstrapTokenString{ - ID: "abcdef", - Secret: "0123456789abcdef", - }, - } -) - -// newCmdConfig returns cobra.Command for "kubeadm config" command -func newCmdConfig(out io.Writer) *cobra.Command { - var kubeConfigFile string - - cmd := &cobra.Command{ - Use: "config", - Short: "Manage configuration for a kubeadm cluster persisted in a ConfigMap in the cluster", - Long: fmt.Sprintf(dedent.Dedent(` - There is a ConfigMap in the %s namespace called %q that kubeadm uses to store internal configuration about the - cluster. kubeadm CLI v1.8.0+ automatically creates this ConfigMap with the config used with 'kubeadm init', but if you - initialized your cluster using kubeadm v1.7.x or lower, you must use the 'config upload' command to create this - ConfigMap. This is required so that 'kubeadm upgrade' can configure your upgraded cluster correctly. - `), metav1.NamespaceSystem, constants.KubeadmConfigConfigMap), - // Without this callback, if a user runs just the "upload" - // command without a subcommand, or with an invalid subcommand, - // cobra will print usage information, but still exit cleanly. - // We want to return an error code in these cases so that the - // user knows that their command was invalid. - RunE: cmdutil.SubCmdRunE("config"), - } - - options.AddKubeConfigFlag(cmd.PersistentFlags(), &kubeConfigFile) - - kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile) - cmd.AddCommand(newCmdConfigPrint(out)) - cmd.AddCommand(newCmdConfigMigrate(out)) - cmd.AddCommand(newCmdConfigView(out, &kubeConfigFile)) - cmd.AddCommand(newCmdConfigImages(out)) - return cmd -} - -// newCmdConfigPrint returns cobra.Command for "kubeadm config print" command -func newCmdConfigPrint(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "print", - Short: "Print configuration", - Long: dedent.Dedent(` - This command prints configurations for subcommands provided. - For details, see: https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2`), - RunE: cmdutil.SubCmdRunE("print"), - } - cmd.AddCommand(newCmdConfigPrintInitDefaults(out)) - cmd.AddCommand(newCmdConfigPrintJoinDefaults(out)) - return cmd -} - -// newCmdConfigPrintInitDefaults returns cobra.Command for "kubeadm config print init-defaults" command -func newCmdConfigPrintInitDefaults(out io.Writer) *cobra.Command { - return newCmdConfigPrintActionDefaults(out, "init", getDefaultInitConfigBytes) -} - -// newCmdConfigPrintJoinDefaults returns cobra.Command for "kubeadm config print join-defaults" command -func newCmdConfigPrintJoinDefaults(out io.Writer) *cobra.Command { - return newCmdConfigPrintActionDefaults(out, "join", getDefaultNodeConfigBytes) -} - -func newCmdConfigPrintActionDefaults(out io.Writer, action string, configBytesProc func() ([]byte, error)) *cobra.Command { - kinds := []string{} - cmd := &cobra.Command{ - Use: fmt.Sprintf("%s-defaults", action), - Short: fmt.Sprintf("Print default %s configuration, that can be used for 'kubeadm %s'", action, action), - Long: fmt.Sprintf(dedent.Dedent(` - This command prints objects such as the default %s configuration that is used for 'kubeadm %s'. - - Note that sensitive values like the Bootstrap Token fields are replaced with placeholder values like %q in order to pass validation but - not perform the real computation for creating a token. - `), action, action, placeholderToken), - RunE: func(cmd *cobra.Command, args []string) error { - groups, err := mapLegacyKindsToGroups(kinds) - if err != nil { - return err - } - return runConfigPrintActionDefaults(out, groups, configBytesProc) - }, - Args: cobra.NoArgs, - } - cmd.Flags().StringSliceVar(&kinds, "component-configs", kinds, - fmt.Sprintf("A comma-separated list for component config API objects to print the default values for. Available values: %v. If this flag is not set, no component configs will be printed.", getSupportedComponentConfigKinds())) - return cmd -} - -func runConfigPrintActionDefaults(out io.Writer, componentConfigs []string, configBytesProc func() ([]byte, error)) error { - initialConfig, err := configBytesProc() - if err != nil { - return err - } - - allBytes := [][]byte{initialConfig} - for _, componentConfig := range componentConfigs { - cfgBytes, err := getDefaultComponentConfigBytes(componentConfig) - if err != nil { - return err - } - allBytes = append(allBytes, cfgBytes) - } - - fmt.Fprint(out, string(bytes.Join(allBytes, []byte(constants.YAMLDocumentSeparator)))) - return nil -} - -func getDefaultComponentConfigBytes(group string) ([]byte, error) { - defaultedInitConfig, err := getDefaultedInitConfig() - if err != nil { - return []byte{}, err - } - - componentCfg, ok := defaultedInitConfig.ComponentConfigs[group] - if !ok { - return []byte{}, errors.Errorf("cannot get defaulted config for component group %q", group) - } - - return componentCfg.Marshal() -} - -// legacyKindToGroupMap maps between the old API object types and the new way of specifying component configs (by group) -var legacyKindToGroupMap = map[string]string{ - "KubeletConfiguration": componentconfigs.KubeletGroup, - "KubeProxyConfiguration": componentconfigs.KubeProxyGroup, -} - -// getSupportedComponentConfigKinds returns all currently supported component config API object names -func getSupportedComponentConfigKinds() []string { - objects := []string{} - for componentType := range legacyKindToGroupMap { - objects = append(objects, string(componentType)) - } - sort.Strings(objects) - return objects -} - -func mapLegacyKindsToGroups(kinds []string) ([]string, error) { - groups := []string{} - for _, kind := range kinds { - group, ok := legacyKindToGroupMap[kind] - if ok { - groups = append(groups, group) - } else { - return nil, errors.Errorf("--component-configs needs to contain some of %v", getSupportedComponentConfigKinds()) - } - } - return groups, nil -} - -func getDefaultedInitConfig() (*kubeadmapi.InitConfiguration, error) { - initCfg := &kubeadmapiv1beta2.InitConfiguration{ - LocalAPIEndpoint: kubeadmapiv1beta2.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - BootstrapTokens: []kubeadmapiv1beta2.BootstrapToken{placeholderToken}, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - CRISocket: constants.DefaultDockerCRISocket, // avoid CRI detection - }, - } - clusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: constants.CurrentKubernetesVersion.String(), // avoid going to the Internet for the current Kubernetes version - } - return configutil.DefaultedInitConfiguration(initCfg, clusterCfg) -} - -func getDefaultInitConfigBytes() ([]byte, error) { - internalcfg, err := getDefaultedInitConfig() - if err != nil { - return []byte{}, err - } - - return configutil.MarshalKubeadmConfigObject(internalcfg) -} - -func getDefaultNodeConfigBytes() ([]byte, error) { - internalcfg, err := configutil.DefaultedJoinConfiguration(&kubeadmapiv1beta2.JoinConfiguration{ - Discovery: kubeadmapiv1beta2.Discovery{ - BootstrapToken: &kubeadmapiv1beta2.BootstrapTokenDiscovery{ - Token: placeholderToken.Token.String(), - APIServerEndpoint: "kube-apiserver:6443", - UnsafeSkipCAVerification: true, // TODO: UnsafeSkipCAVerification: true needs to be set for validation to pass, but shouldn't be recommended as the default - }, - }, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - CRISocket: constants.DefaultDockerCRISocket, // avoid CRI detection - }, - }) - if err != nil { - return []byte{}, err - } - - return configutil.MarshalKubeadmConfigObject(internalcfg) -} - -// newCmdConfigMigrate returns cobra.Command for "kubeadm config migrate" command -func newCmdConfigMigrate(out io.Writer) *cobra.Command { - var oldCfgPath, newCfgPath string - cmd := &cobra.Command{ - Use: "migrate", - Short: "Read an older version of the kubeadm configuration API types from a file, and output the similar config object for the newer version", - Long: fmt.Sprintf(dedent.Dedent(` - This command lets you convert configuration objects of older versions to the latest supported version, - locally in the CLI tool without ever touching anything in the cluster. - In this version of kubeadm, the following API versions are supported: - - %s - - Further, kubeadm can only write out config of version %q, but read both types. - So regardless of what version you pass to the --old-config parameter here, the API object will be - read, deserialized, defaulted, converted, validated, and re-serialized when written to stdout or - --new-config if specified. - - In other words, the output of this command is what kubeadm actually would read internally if you - submitted this file to "kubeadm init" - `), kubeadmapiv1beta2.SchemeGroupVersion, kubeadmapiv1beta2.SchemeGroupVersion), - RunE: func(cmd *cobra.Command, args []string) error { - if len(oldCfgPath) == 0 { - return errors.New("the --old-config flag is mandatory") - } - - oldCfgBytes, err := ioutil.ReadFile(oldCfgPath) - if err != nil { - return err - } - - outputBytes, err := configutil.MigrateOldConfig(oldCfgBytes) - if err != nil { - return err - } - - if newCfgPath == "" { - fmt.Fprint(out, string(outputBytes)) - } else { - if err := ioutil.WriteFile(newCfgPath, outputBytes, 0644); err != nil { - return errors.Wrapf(err, "failed to write the new configuration to the file %q", newCfgPath) - } - } - return nil - }, - Args: cobra.NoArgs, - } - cmd.Flags().StringVar(&oldCfgPath, "old-config", "", "Path to the kubeadm config file that is using an old API version and should be converted. This flag is mandatory.") - cmd.Flags().StringVar(&newCfgPath, "new-config", "", "Path to the resulting equivalent kubeadm config file using the new API version. Optional, if not specified output will be sent to STDOUT.") - return cmd -} - -// newCmdConfigView returns cobra.Command for "kubeadm config view" command -func newCmdConfigView(out io.Writer, kubeConfigFile *string) *cobra.Command { - return &cobra.Command{ - Use: "view", - Short: "View the kubeadm configuration stored inside the cluster", - Deprecated: "This command is deprecated and will be removed in a future release, please use 'kubectl get cm -o yaml -n kube-system kubeadm-config' to get the kubeadm config directly.", - Long: fmt.Sprintf(dedent.Dedent(` - Using this command, you can view the ConfigMap in the cluster where the configuration for kubeadm is located. - - The configuration is located in the %q namespace in the %q ConfigMap. - `), metav1.NamespaceSystem, constants.KubeadmConfigConfigMap), - RunE: func(cmd *cobra.Command, args []string) error { - klog.V(1).Infoln("[config] retrieving ClientSet from file") - client, err := kubeconfigutil.ClientSetFromFile(*kubeConfigFile) - if err != nil { - return err - } - - return RunConfigView(out, client) - }, - Args: cobra.NoArgs, - } -} - -// RunConfigView gets the configuration persisted in the cluster -func RunConfigView(out io.Writer, client clientset.Interface) error { - - klog.V(1).Infoln("[config] getting the cluster configuration") - cfgConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), constants.KubeadmConfigConfigMap, metav1.GetOptions{}) - if err != nil { - return err - } - // No need to append \n as that already exists in the ConfigMap - fmt.Fprintf(out, "%s", cfgConfigMap.Data[constants.ClusterConfigurationConfigMapKey]) - return nil -} - -// newCmdConfigImages returns the "kubeadm config images" command -func newCmdConfigImages(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "images", - Short: "Interact with container images used by kubeadm", - RunE: cmdutil.SubCmdRunE("images"), - } - cmd.AddCommand(newCmdConfigImagesList(out, nil)) - cmd.AddCommand(newCmdConfigImagesPull()) - return cmd -} - -// newCmdConfigImagesPull returns the `kubeadm config images pull` command -func newCmdConfigImagesPull() *cobra.Command { - externalClusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{} - kubeadmscheme.Scheme.Default(externalClusterCfg) - externalInitCfg := &kubeadmapiv1beta2.InitConfiguration{} - kubeadmscheme.Scheme.Default(externalInitCfg) - var cfgPath, featureGatesString string - var err error - - cmd := &cobra.Command{ - Use: "pull", - Short: "Pull images used by kubeadm", - RunE: func(_ *cobra.Command, _ []string) error { - externalClusterCfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString) - if err != nil { - return err - } - internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, externalInitCfg, externalClusterCfg) - if err != nil { - return err - } - containerRuntime, err := utilruntime.NewContainerRuntime(utilsexec.New(), internalcfg.NodeRegistration.CRISocket) - if err != nil { - return err - } - return PullControlPlaneImages(containerRuntime, &internalcfg.ClusterConfiguration) - }, - Args: cobra.NoArgs, - } - AddImagesCommonConfigFlags(cmd.PersistentFlags(), externalClusterCfg, &cfgPath, &featureGatesString) - cmdutil.AddCRISocketFlag(cmd.PersistentFlags(), &externalInitCfg.NodeRegistration.CRISocket) - - return cmd -} - -// ImagesPull is the struct used to hold information relating to image pulling -type ImagesPull struct { - runtime utilruntime.ContainerRuntime - images []string -} - -// NewImagesPull initializes and returns the `kubeadm config images pull` command -func NewImagesPull(runtime utilruntime.ContainerRuntime, images []string) *ImagesPull { - return &ImagesPull{ - runtime: runtime, - images: images, - } -} - -// PullControlPlaneImages pulls all images that the ImagesPull knows about -func PullControlPlaneImages(runtime utilruntime.ContainerRuntime, cfg *kubeadmapi.ClusterConfiguration) error { - images := images.GetControlPlaneImages(cfg) - for _, image := range images { - if err := runtime.PullImage(image); err != nil { - return errors.Wrapf(err, "failed to pull image %q", image) - } - fmt.Printf("[config/images] Pulled %s\n", image) - } - return nil -} - -// newCmdConfigImagesList returns the "kubeadm config images list" command -func newCmdConfigImagesList(out io.Writer, mockK8sVersion *string) *cobra.Command { - externalcfg := &kubeadmapiv1beta2.ClusterConfiguration{} - kubeadmscheme.Scheme.Default(externalcfg) - var cfgPath, featureGatesString string - var err error - - // This just sets the Kubernetes version for unit testing so kubeadm won't try to - // lookup the latest release from the internet. - if mockK8sVersion != nil { - externalcfg.KubernetesVersion = *mockK8sVersion - } - - outputFlags := output.NewOutputFlags(&imageTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(output.TextOutput) - - cmd := &cobra.Command{ - Use: "list", - Short: "Print a list of images kubeadm will use. The configuration file is used in case any images or image repositories are customized", - RunE: func(_ *cobra.Command, _ []string) error { - externalcfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString) - if err != nil { - return err - } - - printer, err := outputFlags.ToPrinter() - if err != nil { - return err - } - - imagesList, err := NewImagesList(cfgPath, externalcfg) - if err != nil { - return err - } - - return imagesList.Run(out, printer) - }, - Args: cobra.NoArgs, - } - outputFlags.AddFlags(cmd) - AddImagesCommonConfigFlags(cmd.PersistentFlags(), externalcfg, &cfgPath, &featureGatesString) - return cmd -} - -// NewImagesList returns the underlying struct for the "kubeadm config images list" command -func NewImagesList(cfgPath string, cfg *kubeadmapiv1beta2.ClusterConfiguration) (*ImagesList, error) { - // Avoid running the CRI auto-detection code as we don't need it - versionedInitCfg := &kubeadmapiv1beta2.InitConfiguration{ - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - CRISocket: constants.DefaultDockerCRISocket, - }, - } - - initcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, versionedInitCfg, cfg) - if err != nil { - return nil, errors.Wrap(err, "could not convert cfg to an internal cfg") - } - - return &ImagesList{ - cfg: initcfg, - }, nil -} - -// ImagesList defines the struct used for "kubeadm config images list" -type ImagesList struct { - cfg *kubeadmapi.InitConfiguration -} - -// imageTextPrinter prints image info in a text form -type imageTextPrinter struct { - output.TextPrinter -} - -// PrintObj is an implementation of ResourcePrinter.PrintObj for plain text output -func (itp *imageTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { - var err error - if imgs, ok := obj.(*outputapiv1alpha1.Images); ok { - _, err = fmt.Fprintln(writer, strings.Join(imgs.Images, "\n")) - } else { - err = errors.New("unexpected object type") - } - return err -} - -// imageTextPrintFlags provides flags necessary for printing image in a text form. -type imageTextPrintFlags struct{} - -// ToPrinter returns kubeadm printer for the text output format -func (ipf *imageTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { - if outputFormat == output.TextOutput { - return &imageTextPrinter{}, nil - } - return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.TextOutput}} -} - -// Run runs the images command and writes the result to the io.Writer passed in -func (i *ImagesList) Run(out io.Writer, printer output.Printer) error { - imgs := images.GetControlPlaneImages(&i.cfg.ClusterConfiguration) - - if err := printer.PrintObj(&outputapiv1alpha1.Images{Images: imgs}, out); err != nil { - return errors.Wrap(err, "unable to print images") - } - - return nil -} - -// AddImagesCommonConfigFlags adds the flags that configure kubeadm (and affect the images kubeadm will use) -func AddImagesCommonConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.ClusterConfiguration, cfgPath *string, featureGatesString *string) { - options.AddKubernetesVersionFlag(flagSet, &cfg.KubernetesVersion) - options.AddFeatureGatesStringFlag(flagSet, featureGatesString) - options.AddImageMetaFlags(flagSet, &cfg.ImageRepository) - options.AddConfigFlag(flagSet, cfgPath) -} diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go deleted file mode 100644 index c40f9699703df..0000000000000 --- a/cmd/kubeadm/app/cmd/config_test.go +++ /dev/null @@ -1,530 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "sort" - "strings" - "testing" - "text/template" - - "github.com/lithammer/dedent" - "github.com/spf13/cobra" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -const ( - defaultNumberOfImages = 8 -) - -var ( - // dummyKubernetesVersion and dummyKubernetesVersionStr are just used for unit testing, in order to not make - // kubeadm lookup dl.k8s.io to resolve what the latest stable release is - dummyKubernetesVersion = constants.MinimumControlPlaneVersion - dummyKubernetesVersionStr = dummyKubernetesVersion.String() -) - -func TestNewCmdConfigImagesList(t *testing.T) { - var output bytes.Buffer - mockK8sVersion := dummyKubernetesVersionStr - images := newCmdConfigImagesList(&output, &mockK8sVersion) - if err := images.RunE(nil, nil); err != nil { - t.Fatalf("Error from running the images command: %v", err) - } - actual := strings.Split(output.String(), "\n") - if len(actual) != defaultNumberOfImages { - t.Fatalf("Expected %v but found %v images", defaultNumberOfImages, len(actual)) - } -} - -func TestImagesListRunWithCustomConfigPath(t *testing.T) { - testcases := []struct { - name string - expectedImageCount int - // each string provided here must appear in at least one image returned by Run - expectedImageSubstrings []string - configContents []byte - }{ - { - name: "set k8s version", - expectedImageCount: defaultNumberOfImages, - expectedImageSubstrings: []string{ - constants.CurrentKubernetesVersion.String(), - }, - configContents: []byte(dedent.Dedent(fmt.Sprintf(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: ClusterConfiguration - kubernetesVersion: %s - `, constants.CurrentKubernetesVersion))), - }, - { - name: "use coredns", - expectedImageCount: defaultNumberOfImages, - expectedImageSubstrings: []string{ - "coredns", - }, - configContents: []byte(dedent.Dedent(fmt.Sprintf(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: ClusterConfiguration - kubernetesVersion: %s - `, constants.MinimumControlPlaneVersion))), - }, - } - - outputFlags := output.NewOutputFlags(&imageTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(output.TextOutput) - printer, err := outputFlags.ToPrinter() - if err != nil { - t.Fatalf("can't create printer for the output format %s: %+v", output.TextOutput, err) - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "kubeadm-images-test") - if err != nil { - t.Fatalf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - - configFilePath := filepath.Join(tmpDir, "test-config-file") - if err := ioutil.WriteFile(configFilePath, tc.configContents, 0644); err != nil { - t.Fatalf("Failed writing a config file: %v", err) - } - - i, err := NewImagesList(configFilePath, &kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }) - if err != nil { - t.Fatalf("Failed getting the kubeadm images command: %v", err) - } - var output bytes.Buffer - if err = i.Run(&output, printer); err != nil { - t.Fatalf("Error from running the images command: %v", err) - } - actual := strings.Split(output.String(), "\n") - if len(actual) != tc.expectedImageCount { - t.Fatalf("did not get the same number of images: actual: %v expected: %v. Actual value: %v", len(actual), tc.expectedImageCount, actual) - } - - for _, substring := range tc.expectedImageSubstrings { - if !strings.Contains(output.String(), substring) { - t.Errorf("Expected to find %v but did not in this list of images: %v", substring, actual) - } - } - }) - } -} - -func TestConfigImagesListRunWithoutPath(t *testing.T) { - testcases := []struct { - name string - cfg kubeadmapiv1beta2.ClusterConfiguration - expectedImages int - }{ - { - name: "empty config", - expectedImages: defaultNumberOfImages, - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - }, - { - name: "external etcd configuration", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - Etcd: kubeadmapiv1beta2.Etcd{ - External: &kubeadmapiv1beta2.ExternalEtcd{ - Endpoints: []string{"https://some.etcd.com:2379"}, - }, - }, - KubernetesVersion: dummyKubernetesVersionStr, - }, - expectedImages: defaultNumberOfImages - 1, - }, - { - name: "coredns enabled", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - expectedImages: defaultNumberOfImages, - }, - { - name: "kube-dns enabled", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - DNS: kubeadmapiv1beta2.DNS{ - Type: kubeadmapiv1beta2.KubeDNS, - }, - }, - expectedImages: defaultNumberOfImages + 2, - }, - } - - outputFlags := output.NewOutputFlags(&imageTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(output.TextOutput) - printer, err := outputFlags.ToPrinter() - if err != nil { - t.Fatalf("can't create printer for the output format %s: %+v", output.TextOutput, err) - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - i, err := NewImagesList("", &tc.cfg) - if err != nil { - t.Fatalf("did not expect an error while creating the Images command: %v", err) - } - - var output bytes.Buffer - - if err = i.Run(&output, printer); err != nil { - t.Fatalf("did not expect an error running the Images command: %v", err) - } - - actual := strings.Split(output.String(), "\n") - if len(actual) != tc.expectedImages { - t.Fatalf("expected %v images but got %v", tc.expectedImages, actual) - } - }) - } -} - -func TestConfigImagesListOutput(t *testing.T) { - - etcdVersion, ok := constants.SupportedEtcdVersion[uint8(dummyKubernetesVersion.Minor())] - if !ok { - t.Fatalf("cannot determine etcd version for Kubernetes version %s", dummyKubernetesVersionStr) - } - versionMapping := struct { - EtcdVersion string - KubeVersion string - PauseVersion string - CoreDNSVersion string - }{ - EtcdVersion: etcdVersion, - KubeVersion: "v" + dummyKubernetesVersionStr, - PauseVersion: constants.PauseVersion, - CoreDNSVersion: constants.CoreDNSVersion, - } - - testcases := []struct { - name string - cfg kubeadmapiv1beta2.ClusterConfiguration - outputFormat string - expectedOutput string - }{ - { - name: "text output", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - outputFormat: "text", - expectedOutput: `k8s.gcr.io/kube-apiserver:{{.KubeVersion}} -k8s.gcr.io/kube-controller-manager:{{.KubeVersion}} -k8s.gcr.io/kube-scheduler:{{.KubeVersion}} -k8s.gcr.io/kube-proxy:{{.KubeVersion}} -k8s.gcr.io/pause:{{.PauseVersion}} -k8s.gcr.io/etcd:{{.EtcdVersion}} -k8s.gcr.io/coredns:{{.CoreDNSVersion}} -`, - }, - { - name: "JSON output", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - outputFormat: "json", - expectedOutput: `{ - "kind": "Images", - "apiVersion": "output.kubeadm.k8s.io/v1alpha1", - "images": [ - "k8s.gcr.io/kube-apiserver:{{.KubeVersion}}", - "k8s.gcr.io/kube-controller-manager:{{.KubeVersion}}", - "k8s.gcr.io/kube-scheduler:{{.KubeVersion}}", - "k8s.gcr.io/kube-proxy:{{.KubeVersion}}", - "k8s.gcr.io/pause:{{.PauseVersion}}", - "k8s.gcr.io/etcd:{{.EtcdVersion}}", - "k8s.gcr.io/coredns:{{.CoreDNSVersion}}" - ] -} -`, - }, - { - name: "YAML output", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - outputFormat: "yaml", - expectedOutput: `apiVersion: output.kubeadm.k8s.io/v1alpha1 -images: -- k8s.gcr.io/kube-apiserver:{{.KubeVersion}} -- k8s.gcr.io/kube-controller-manager:{{.KubeVersion}} -- k8s.gcr.io/kube-scheduler:{{.KubeVersion}} -- k8s.gcr.io/kube-proxy:{{.KubeVersion}} -- k8s.gcr.io/pause:{{.PauseVersion}} -- k8s.gcr.io/etcd:{{.EtcdVersion}} -- k8s.gcr.io/coredns:{{.CoreDNSVersion}} -kind: Images -`, - }, - { - name: "go-template output", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - outputFormat: `go-template={{range .images}}{{.}}{{"\n"}}{{end}}`, - expectedOutput: `k8s.gcr.io/kube-apiserver:{{.KubeVersion}} -k8s.gcr.io/kube-controller-manager:{{.KubeVersion}} -k8s.gcr.io/kube-scheduler:{{.KubeVersion}} -k8s.gcr.io/kube-proxy:{{.KubeVersion}} -k8s.gcr.io/pause:{{.PauseVersion}} -k8s.gcr.io/etcd:{{.EtcdVersion}} -k8s.gcr.io/coredns:{{.CoreDNSVersion}} -`, - }, - { - name: "JSONPATH output", - cfg: kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: dummyKubernetesVersionStr, - }, - outputFormat: `jsonpath={range.images[*]}{@} {end}`, - expectedOutput: "k8s.gcr.io/kube-apiserver:{{.KubeVersion}} k8s.gcr.io/kube-controller-manager:{{.KubeVersion}} k8s.gcr.io/kube-scheduler:{{.KubeVersion}} " + - "k8s.gcr.io/kube-proxy:{{.KubeVersion}} k8s.gcr.io/pause:{{.PauseVersion}} k8s.gcr.io/etcd:{{.EtcdVersion}} k8s.gcr.io/coredns:{{.CoreDNSVersion}} ", - }, - } - - for _, tc := range testcases { - outputFlags := output.NewOutputFlags(&imageTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(tc.outputFormat) - printer, err := outputFlags.ToPrinter() - if err != nil { - t.Fatalf("can't create printer for the output format %s: %+v", tc.outputFormat, err) - } - - t.Run(tc.name, func(t *testing.T) { - i, err := NewImagesList("", &tc.cfg) - if err != nil { - t.Fatalf("did not expect an error while creating the Images command: %v", err) - } - - var output, expectedOutput bytes.Buffer - - if err = i.Run(&output, printer); err != nil { - t.Fatalf("did not expect an error running the Images command: %v", err) - } - - tmpl, err := template.New("test").Parse(tc.expectedOutput) - if err != nil { - t.Fatalf("could not create template: %v", err) - } - if err = tmpl.Execute(&expectedOutput, versionMapping); err != nil { - t.Fatalf("could not execute template: %v", err) - } - - if output.String() != expectedOutput.String() { - t.Fatalf("unexpected output:\n|%s|\nexpected:\n|%s|\n", output.String(), tc.expectedOutput) - } - }) - } -} - -func TestImagesPull(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - }, - } - - fexec := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, - } - - containerRuntime, err := utilruntime.NewContainerRuntime(&fexec, constants.DefaultDockerCRISocket) - if err != nil { - t.Errorf("unexpected NewContainerRuntime error: %v", err) - } - - images := []string{"a", "b", "c", "d", "a"} - for _, image := range images { - if err := containerRuntime.PullImage(image); err != nil { - t.Fatalf("expected nil but found %v", err) - } - fmt.Printf("[config/images] Pulled %s\n", image) - } - - if fcmd.CombinedOutputCalls != len(images) { - t.Errorf("expected %d calls, got %d", len(images), fcmd.CombinedOutputCalls) - } -} - -func TestMigrate(t *testing.T) { - cfg := []byte(dedent.Dedent(` - # This is intentionally testing an old API version. Sometimes this may be the latest version (if no old configs are supported). - apiVersion: kubeadm.k8s.io/v1beta1 - kind: InitConfiguration - `)) - configFile, cleanup := tempConfig(t, cfg) - defer cleanup() - - var output bytes.Buffer - command := newCmdConfigMigrate(&output) - if err := command.Flags().Set("old-config", configFile); err != nil { - t.Fatalf("failed to set old-config flag") - } - newConfigPath := filepath.Join(filepath.Dir(configFile), "new-migrated-config") - if err := command.Flags().Set("new-config", newConfigPath); err != nil { - t.Fatalf("failed to set new-config flag") - } - if err := command.RunE(nil, nil); err != nil { - t.Fatalf("Error from running the migrate command: %v", err) - } - if _, err := configutil.LoadInitConfigurationFromFile(newConfigPath); err != nil { - t.Fatalf("Could not read output back into internal type: %v", err) - } -} - -// Returns the name of the file created and a cleanup callback -func tempConfig(t *testing.T, config []byte) (string, func()) { - t.Helper() - tmpDir, err := ioutil.TempDir("", "kubeadm-migration-test") - if err != nil { - t.Fatalf("Unable to create temporary directory: %v", err) - } - configFilePath := filepath.Join(tmpDir, "test-config-file") - if err := ioutil.WriteFile(configFilePath, config, 0644); err != nil { - os.RemoveAll(tmpDir) - t.Fatalf("Failed writing a config file: %v", err) - } - return configFilePath, func() { - os.RemoveAll(tmpDir) - } -} - -func TestNewCmdConfigPrintActionDefaults(t *testing.T) { - tests := []struct { - name string - expectedKinds []string // need to be sorted - componentConfigs string - cmdProc func(out io.Writer) *cobra.Command - }{ - { - name: "InitConfiguration: No component configs", - expectedKinds: []string{ - constants.ClusterConfigurationKind, - constants.InitConfigurationKind, - }, - cmdProc: newCmdConfigPrintInitDefaults, - }, - { - name: "InitConfiguration: KubeProxyConfiguration", - expectedKinds: []string{ - constants.ClusterConfigurationKind, - constants.InitConfigurationKind, - "KubeProxyConfiguration", - }, - componentConfigs: "KubeProxyConfiguration", - cmdProc: newCmdConfigPrintInitDefaults, - }, - { - name: "InitConfiguration: KubeProxyConfiguration and KubeletConfiguration", - expectedKinds: []string{ - constants.ClusterConfigurationKind, - constants.InitConfigurationKind, - "KubeProxyConfiguration", - "KubeletConfiguration", - }, - componentConfigs: "KubeProxyConfiguration,KubeletConfiguration", - cmdProc: newCmdConfigPrintInitDefaults, - }, - { - name: "JoinConfiguration: No component configs", - expectedKinds: []string{ - constants.JoinConfigurationKind, - }, - cmdProc: newCmdConfigPrintJoinDefaults, - }, - { - name: "JoinConfiguration: KubeProxyConfiguration", - expectedKinds: []string{ - constants.JoinConfigurationKind, - "KubeProxyConfiguration", - }, - componentConfigs: "KubeProxyConfiguration", - cmdProc: newCmdConfigPrintJoinDefaults, - }, - { - name: "JoinConfiguration: KubeProxyConfiguration and KubeletConfiguration", - expectedKinds: []string{ - constants.JoinConfigurationKind, - "KubeProxyConfiguration", - "KubeletConfiguration", - }, - componentConfigs: "KubeProxyConfiguration,KubeletConfiguration", - cmdProc: newCmdConfigPrintJoinDefaults, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var output bytes.Buffer - - command := test.cmdProc(&output) - if err := command.Flags().Set("component-configs", test.componentConfigs); err != nil { - t.Fatalf("failed to set component-configs flag") - } - if err := command.RunE(nil, nil); err != nil { - t.Fatalf("Error from running the print command: %v", err) - } - - gvkmap, err := kubeadmutil.SplitYAMLDocuments(output.Bytes()) - if err != nil { - t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err) - } - - gotKinds := []string{} - for gvk := range gvkmap { - gotKinds = append(gotKinds, gvk.Kind) - } - - sort.Strings(gotKinds) - - if !reflect.DeepEqual(gotKinds, test.expectedKinds) { - t.Fatalf("kinds not matching:\n\texpectedKinds: %v\n\tgotKinds: %v\n", test.expectedKinds, gotKinds) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go deleted file mode 100644 index c2a0457b6d353..0000000000000 --- a/cmd/kubeadm/app/cmd/init.go +++ /dev/null @@ -1,591 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "io" - "os" - "path/filepath" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var ( - initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(` - Your Kubernetes control-plane has initialized successfully! - - To start using your cluster, you need to run the following as a regular user: - - mkdir -p $HOME/.kube - sudo cp -i {{.KubeConfigPath}} $HOME/.kube/config - sudo chown $(id -u):$(id -g) $HOME/.kube/config - - Alternatively, if you are the root user, you can run: - - export KUBECONFIG=/etc/kubernetes/admin.conf - - You should now deploy a pod network to the cluster. - Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: - https://kubernetes.io/docs/concepts/cluster-administration/addons/ - - {{if .ControlPlaneEndpoint -}} - {{if .UploadCerts -}} - You can now join any number of the control-plane node running the following command on each as root: - - {{.joinControlPlaneCommand}} - - Please note that the certificate-key gives access to cluster sensitive data, keep it secret! - As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use - "kubeadm init phase upload-certs --upload-certs" to reload certs afterward. - - {{else -}} - You can now join any number of control-plane nodes by copying certificate authorities - and service account keys on each node and then running the following as root: - - {{.joinControlPlaneCommand}} - - {{end}}{{end}}Then you can join any number of worker nodes by running the following on each as root: - - {{.joinWorkerCommand}} - `))) -) - -// initOptions defines all the init options exposed via flags by kubeadm init. -// Please note that this structure includes the public kubeadm config API, but only a subset of the options -// supported by this api will be exposed as a flag. -type initOptions struct { - cfgPath string - skipTokenPrint bool - dryRun bool - kubeconfigDir string - kubeconfigPath string - featureGatesString string - ignorePreflightErrors []string - bto *options.BootstrapTokenOptions - externalInitCfg *kubeadmapiv1beta2.InitConfiguration - externalClusterCfg *kubeadmapiv1beta2.ClusterConfiguration - uploadCerts bool - skipCertificateKeyPrint bool - patchesDir string -} - -// compile-time assert that the local data object satisfies the phases data interface. -var _ phases.InitData = &initData{} - -// initData defines all the runtime information used when running the kubeadm init workflow; -// this data is shared across all the phases that are included in the workflow. -type initData struct { - cfg *kubeadmapi.InitConfiguration - skipTokenPrint bool - dryRun bool - kubeconfigDir string - kubeconfigPath string - ignorePreflightErrors sets.String - certificatesDir string - dryRunDir string - externalCA bool - client clientset.Interface - outputWriter io.Writer - uploadCerts bool - skipCertificateKeyPrint bool - patchesDir string -} - -// newCmdInit returns "kubeadm init" command. -// NB. initOptions is exposed as parameter for allowing unit testing of -// the newInitOptions method, that implements all the command options validation logic -func newCmdInit(out io.Writer, initOptions *initOptions) *cobra.Command { - if initOptions == nil { - initOptions = newInitOptions() - } - initRunner := workflow.NewRunner() - - cmd := &cobra.Command{ - Use: "init", - Short: "Run this command in order to set up the Kubernetes control plane", - RunE: func(cmd *cobra.Command, args []string) error { - c, err := initRunner.InitData(args) - if err != nil { - return err - } - - data := c.(*initData) - fmt.Printf("[init] Using Kubernetes version: %s\n", data.cfg.KubernetesVersion) - - if err := initRunner.Run(args); err != nil { - return err - } - - return showJoinCommand(data, out) - }, - Args: cobra.NoArgs, - } - - // adds flags to the init command - // init command local flags could be eventually inherited by the sub-commands automatically generated for phases - AddInitConfigFlags(cmd.Flags(), initOptions.externalInitCfg) - AddClusterConfigFlags(cmd.Flags(), initOptions.externalClusterCfg, &initOptions.featureGatesString) - AddInitOtherFlags(cmd.Flags(), initOptions) - initOptions.bto.AddTokenFlag(cmd.Flags()) - initOptions.bto.AddTTLFlag(cmd.Flags()) - options.AddImageMetaFlags(cmd.Flags(), &initOptions.externalClusterCfg.ImageRepository) - - // defines additional flag that are not used by the init command but that could be eventually used - // by the sub-commands automatically generated for phases - initRunner.SetAdditionalFlags(func(flags *flag.FlagSet) { - options.AddKubeConfigFlag(flags, &initOptions.kubeconfigPath) - options.AddKubeConfigDirFlag(flags, &initOptions.kubeconfigDir) - options.AddControlPlanExtraArgsFlags(flags, &initOptions.externalClusterCfg.APIServer.ExtraArgs, &initOptions.externalClusterCfg.ControllerManager.ExtraArgs, &initOptions.externalClusterCfg.Scheduler.ExtraArgs) - }) - - // initialize the workflow runner with the list of phases - initRunner.AppendPhase(phases.NewPreflightPhase()) - initRunner.AppendPhase(phases.NewCertsPhase()) - initRunner.AppendPhase(phases.NewKubeConfigPhase()) - initRunner.AppendPhase(phases.NewKubeletStartPhase()) - initRunner.AppendPhase(phases.NewControlPlanePhase()) - initRunner.AppendPhase(phases.NewEtcdPhase()) - initRunner.AppendPhase(phases.NewWaitControlPlanePhase()) - initRunner.AppendPhase(phases.NewUploadConfigPhase()) - initRunner.AppendPhase(phases.NewUploadCertsPhase()) - initRunner.AppendPhase(phases.NewMarkControlPlanePhase()) - initRunner.AppendPhase(phases.NewBootstrapTokenPhase()) - initRunner.AppendPhase(phases.NewKubeletFinalizePhase()) - initRunner.AppendPhase(phases.NewAddonPhase()) - - // sets the data builder function, that will be used by the runner - // both when running the entire workflow or single phases - initRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { - return newInitData(cmd, args, initOptions, out) - }) - - // binds the Runner to kubeadm init command by altering - // command help, adding --skip-phases flag and by adding phases subcommands - initRunner.BindToCommand(cmd) - - return cmd -} - -// AddInitConfigFlags adds init flags bound to the config to the specified flagset -func AddInitConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.InitConfiguration) { - flagSet.StringVar( - &cfg.LocalAPIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, cfg.LocalAPIEndpoint.AdvertiseAddress, - "The IP address the API Server will advertise it's listening on. If not set the default network interface will be used.", - ) - flagSet.Int32Var( - &cfg.LocalAPIEndpoint.BindPort, options.APIServerBindPort, cfg.LocalAPIEndpoint.BindPort, - "Port for the API Server to bind to.", - ) - flagSet.StringVar( - &cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name, - `Specify the node name.`, - ) - flagSet.StringVar( - &cfg.CertificateKey, options.CertificateKey, "", - "Key used to encrypt the control-plane certificates in the kubeadm-certs Secret.", - ) - cmdutil.AddCRISocketFlag(flagSet, &cfg.NodeRegistration.CRISocket) -} - -// AddClusterConfigFlags adds cluster flags bound to the config to the specified flagset -func AddClusterConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.ClusterConfiguration, featureGatesString *string) { - flagSet.StringVar( - &cfg.Networking.ServiceSubnet, options.NetworkingServiceSubnet, cfg.Networking.ServiceSubnet, - "Use alternative range of IP address for service VIPs.", - ) - flagSet.StringVar( - &cfg.Networking.PodSubnet, options.NetworkingPodSubnet, cfg.Networking.PodSubnet, - "Specify range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node.", - ) - flagSet.StringVar( - &cfg.Networking.DNSDomain, options.NetworkingDNSDomain, cfg.Networking.DNSDomain, - `Use alternative domain for services, e.g. "myorg.internal".`, - ) - - flagSet.StringVar( - &cfg.ControlPlaneEndpoint, options.ControlPlaneEndpoint, cfg.ControlPlaneEndpoint, - `Specify a stable IP address or DNS name for the control plane.`, - ) - - options.AddKubernetesVersionFlag(flagSet, &cfg.KubernetesVersion) - - flagSet.StringVar( - &cfg.CertificatesDir, options.CertificatesDir, cfg.CertificatesDir, - `The path where to save and store the certificates.`, - ) - flagSet.StringSliceVar( - &cfg.APIServer.CertSANs, options.APIServerCertSANs, cfg.APIServer.CertSANs, - `Optional extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names.`, - ) - options.AddFeatureGatesStringFlag(flagSet, featureGatesString) -} - -// AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset -// Note: All flags that are not bound to the cfg object should be allowed in cmd/kubeadm/app/apis/kubeadm/validation/validation.go -func AddInitOtherFlags(flagSet *flag.FlagSet, initOptions *initOptions) { - options.AddConfigFlag(flagSet, &initOptions.cfgPath) - flagSet.StringSliceVar( - &initOptions.ignorePreflightErrors, options.IgnorePreflightErrors, initOptions.ignorePreflightErrors, - "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", - ) - flagSet.BoolVar( - &initOptions.skipTokenPrint, options.SkipTokenPrint, initOptions.skipTokenPrint, - "Skip printing of the default bootstrap token generated by 'kubeadm init'.", - ) - flagSet.BoolVar( - &initOptions.dryRun, options.DryRun, initOptions.dryRun, - "Don't apply any changes; just output what would be done.", - ) - flagSet.BoolVar( - &initOptions.uploadCerts, options.UploadCerts, initOptions.uploadCerts, - "Upload control-plane certificates to the kubeadm-certs Secret.", - ) - flagSet.BoolVar( - &initOptions.skipCertificateKeyPrint, options.SkipCertificateKeyPrint, initOptions.skipCertificateKeyPrint, - "Don't print the key used to encrypt the control-plane certificates.", - ) - options.AddPatchesFlag(flagSet, &initOptions.patchesDir) -} - -// newInitOptions returns a struct ready for being used for creating cmd init flags. -func newInitOptions() *initOptions { - // initialize the public kubeadm config API by applying defaults - externalInitCfg := &kubeadmapiv1beta2.InitConfiguration{} - kubeadmscheme.Scheme.Default(externalInitCfg) - - externalClusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{} - kubeadmscheme.Scheme.Default(externalClusterCfg) - - // Create the options object for the bootstrap token-related flags, and override the default value for .Description - bto := options.NewBootstrapTokenOptions() - bto.Description = "The default bootstrap token generated by 'kubeadm init'." - - return &initOptions{ - externalInitCfg: externalInitCfg, - externalClusterCfg: externalClusterCfg, - bto: bto, - kubeconfigDir: kubeadmconstants.KubernetesDir, - kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), - uploadCerts: false, - } -} - -// newInitData returns a new initData struct to be used for the execution of the kubeadm init workflow. -// This func takes care of validating initOptions passed to the command, and then it converts -// options into the internal InitConfiguration type that is used as input all the phases in the kubeadm init workflow -func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io.Writer) (*initData, error) { - // Re-apply defaults to the public kubeadm API (this will set only values not exposed/not set as a flags) - kubeadmscheme.Scheme.Default(options.externalInitCfg) - kubeadmscheme.Scheme.Default(options.externalClusterCfg) - - // Validate standalone flags values and/or combination of flags and then assigns - // validated values to the public kubeadm config API when applicable - var err error - if options.externalClusterCfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, options.featureGatesString); err != nil { - return nil, err - } - - if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil { - return nil, err - } - - if err = options.bto.ApplyTo(options.externalInitCfg); err != nil { - return nil, err - } - - // Either use the config file if specified, or convert public kubeadm API to the internal InitConfiguration - // and validates InitConfiguration - cfg, err := configutil.LoadOrDefaultInitConfiguration(options.cfgPath, options.externalInitCfg, options.externalClusterCfg) - if err != nil { - return nil, err - } - - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) - if err != nil { - return nil, err - } - // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: - cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() - - // override node name and CRI socket from the command line options - if options.externalInitCfg.NodeRegistration.Name != "" { - cfg.NodeRegistration.Name = options.externalInitCfg.NodeRegistration.Name - } - if options.externalInitCfg.NodeRegistration.CRISocket != "" { - cfg.NodeRegistration.CRISocket = options.externalInitCfg.NodeRegistration.CRISocket - } - - if err := configutil.VerifyAPIServerBindAddress(cfg.LocalAPIEndpoint.AdvertiseAddress); err != nil { - return nil, err - } - if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { - return nil, err - } - - // if dry running creates a temporary folder for saving kubeadm generated files - dryRunDir := "" - if options.dryRun { - // the KUBEADM_INIT_DRYRUN_DIR environment variable allows overriding the dry-run temporary - // directory from the command line. This makes it possible to run "kubeadm init" integration - // tests without root. - if dryRunDir, err = kubeadmconstants.CreateTempDirForKubeadm(os.Getenv("KUBEADM_INIT_DRYRUN_DIR"), "kubeadm-init-dryrun"); err != nil { - return nil, errors.Wrap(err, "couldn't create a temporary directory") - } - } - - // Checks if an external CA is provided by the user (when the CA Cert is present but the CA Key is not) - externalCA, err := certsphase.UsingExternalCA(&cfg.ClusterConfiguration) - if externalCA { - // In case the certificates signed by CA (that should be provided by the user) are missing or invalid, - // returns, because kubeadm can't regenerate them without the CA Key - if err != nil { - return nil, errors.Wrapf(err, "invalid or incomplete external CA") - } - - // Validate that also the required kubeconfig files exists and are invalid, because - // kubeadm can't regenerate them without the CA Key - kubeconfigDir := options.kubeconfigDir - if options.dryRun { - kubeconfigDir = dryRunDir - } - if err := kubeconfigphase.ValidateKubeconfigsForExternalCA(kubeconfigDir, cfg); err != nil { - return nil, err - } - } - - // Checks if an external Front-Proxy CA is provided by the user (when the Front-Proxy CA Cert is present but the Front-Proxy CA Key is not) - externalFrontProxyCA, err := certsphase.UsingExternalFrontProxyCA(&cfg.ClusterConfiguration) - if externalFrontProxyCA { - // In case the certificates signed by Front-Proxy CA (that should be provided by the user) are missing or invalid, - // returns, because kubeadm can't regenerate them without the Front-Proxy CA Key - if err != nil { - return nil, errors.Wrapf(err, "invalid or incomplete external front-proxy CA") - } - } - - if options.uploadCerts && (externalCA || externalFrontProxyCA) { - return nil, errors.New("can't use upload-certs with an external CA or an external front-proxy CA") - } - - return &initData{ - cfg: cfg, - certificatesDir: cfg.CertificatesDir, - skipTokenPrint: options.skipTokenPrint, - dryRun: options.dryRun, - dryRunDir: dryRunDir, - kubeconfigDir: options.kubeconfigDir, - kubeconfigPath: options.kubeconfigPath, - ignorePreflightErrors: ignorePreflightErrorsSet, - externalCA: externalCA, - outputWriter: out, - uploadCerts: options.uploadCerts, - skipCertificateKeyPrint: options.skipCertificateKeyPrint, - patchesDir: options.patchesDir, - }, nil -} - -// UploadCerts returns Uploadcerts flag. -func (d *initData) UploadCerts() bool { - return d.uploadCerts -} - -// CertificateKey returns the key used to encrypt the certs. -func (d *initData) CertificateKey() string { - return d.cfg.CertificateKey -} - -// SetCertificateKey set the key used to encrypt the certs. -func (d *initData) SetCertificateKey(key string) { - d.cfg.CertificateKey = key -} - -// SkipCertificateKeyPrint returns the skipCertificateKeyPrint flag. -func (d *initData) SkipCertificateKeyPrint() bool { - return d.skipCertificateKeyPrint -} - -// Cfg returns initConfiguration. -func (d *initData) Cfg() *kubeadmapi.InitConfiguration { - return d.cfg -} - -// DryRun returns the DryRun flag. -func (d *initData) DryRun() bool { - return d.dryRun -} - -// SkipTokenPrint returns the SkipTokenPrint flag. -func (d *initData) SkipTokenPrint() bool { - return d.skipTokenPrint -} - -// IgnorePreflightErrors returns the IgnorePreflightErrors flag. -func (d *initData) IgnorePreflightErrors() sets.String { - return d.ignorePreflightErrors -} - -// CertificateWriteDir returns the path to the certificate folder or the temporary folder path in case of DryRun. -func (d *initData) CertificateWriteDir() string { - if d.dryRun { - return d.dryRunDir - } - return d.certificatesDir -} - -// CertificateDir returns the CertificateDir as originally specified by the user. -func (d *initData) CertificateDir() string { - return d.certificatesDir -} - -// KubeConfigDir returns the path of the Kubernetes configuration folder or the temporary folder path in case of DryRun. -func (d *initData) KubeConfigDir() string { - if d.dryRun { - return d.dryRunDir - } - return d.kubeconfigDir -} - -// KubeConfigPath returns the path to the kubeconfig file to use for connecting to Kubernetes -func (d *initData) KubeConfigPath() string { - if d.dryRun { - d.kubeconfigPath = filepath.Join(d.dryRunDir, kubeadmconstants.AdminKubeConfigFileName) - } - return d.kubeconfigPath -} - -// ManifestDir returns the path where manifest should be stored or the temporary folder path in case of DryRun. -func (d *initData) ManifestDir() string { - if d.dryRun { - return d.dryRunDir - } - return kubeadmconstants.GetStaticPodDirectory() -} - -// KubeletDir returns path of the kubelet configuration folder or the temporary folder in case of DryRun. -func (d *initData) KubeletDir() string { - if d.dryRun { - return d.dryRunDir - } - return kubeadmconstants.KubeletRunDirectory -} - -// ExternalCA returns true if an external CA is provided by the user. -func (d *initData) ExternalCA() bool { - return d.externalCA -} - -// OutputWriter returns the io.Writer used to write output to by this command. -func (d *initData) OutputWriter() io.Writer { - return d.outputWriter -} - -// Client returns a Kubernetes client to be used by kubeadm. -// This function is implemented as a singleton, thus avoiding to recreate the client when it is used by different phases. -// Important. This function must be called after the admin.conf kubeconfig file is created. -func (d *initData) Client() (clientset.Interface, error) { - if d.client == nil { - if d.dryRun { - svcSubnetCIDR, err := kubeadmconstants.GetKubernetesServiceCIDR(d.cfg.Networking.ServiceSubnet, features.Enabled(d.cfg.FeatureGates, features.IPv6DualStack)) - if err != nil { - return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", d.cfg.Networking.ServiceSubnet) - } - // If we're dry-running, we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests - dryRunGetter := apiclient.NewInitDryRunGetter(d.cfg.NodeRegistration.Name, svcSubnetCIDR.String()) - d.client = apiclient.NewDryRunClient(dryRunGetter, os.Stdout) - } else { - // If we're acting for real, we should create a connection to the API server and wait for it to come up - var err error - d.client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath()) - if err != nil { - return nil, err - } - } - } - return d.client, nil -} - -// Tokens returns an array of token strings. -func (d *initData) Tokens() []string { - tokens := []string{} - for _, bt := range d.cfg.BootstrapTokens { - tokens = append(tokens, bt.Token.String()) - } - return tokens -} - -// PatchesDir returns the folder where patches for components are stored -func (d *initData) PatchesDir() string { - return d.patchesDir -} - -func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, i *initData) error { - joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.CertificateKey(), i.skipTokenPrint, i.skipCertificateKeyPrint) - if err != nil { - return err - } - - joinWorkerCommand, err := cmdutil.GetJoinWorkerCommand(adminKubeConfigPath, token, i.skipTokenPrint) - if err != nil { - return err - } - - ctx := map[string]interface{}{ - "KubeConfigPath": adminKubeConfigPath, - "ControlPlaneEndpoint": i.Cfg().ControlPlaneEndpoint, - "UploadCerts": i.uploadCerts, - "joinControlPlaneCommand": joinControlPlaneCommand, - "joinWorkerCommand": joinWorkerCommand, - } - - return initDoneTempl.Execute(out, ctx) -} - -// showJoinCommand prints the join command after all the phases in init have finished -func showJoinCommand(i *initData, out io.Writer) error { - adminKubeConfigPath := i.KubeConfigPath() - - // Prints the join command, multiple times in case the user has multiple tokens - for _, token := range i.Tokens() { - if err := printJoinCommand(out, adminKubeConfigPath, token, i); err != nil { - return errors.Wrap(err, "failed to print join command") - } - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/init_test.go b/cmd/kubeadm/app/cmd/init_test.go deleted file mode 100644 index bb2e988ad8918..0000000000000 --- a/cmd/kubeadm/app/cmd/init_test.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" -) - -const ( - testInitConfig = `--- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: "1.2.3.4" -bootstrapTokens: -- token: "abcdef.0123456789abcdef" -nodeRegistration: - criSocket: /run/containerd/containerd.sock - name: someName - ignorePreflightErrors: - - c - - d ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -controlPlaneEndpoint: "3.4.5.6" -` -) - -func TestNewInitData(t *testing.T) { - // create temp directory - tmpDir, err := ioutil.TempDir("", "kubeadm-init-test") - if err != nil { - t.Errorf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - - // create config file - configFilePath := filepath.Join(tmpDir, "test-config-file") - cfgFile, err := os.Create(configFilePath) - if err != nil { - t.Errorf("Unable to create file %q: %v", configFilePath, err) - } - defer cfgFile.Close() - if _, err = cfgFile.WriteString(testInitConfig); err != nil { - t.Fatalf("Unable to write file %q: %v", configFilePath, err) - } - - testCases := []struct { - name string - args []string - flags map[string]string - validate func(*testing.T, *initData) - expectError bool - }{ - // Init data passed using flags - { - name: "pass without any flag (use defaults)", - }, - { - name: "fail if unknown feature gates flag are passed", - flags: map[string]string{ - options.FeatureGatesString: "unknown=true", - }, - expectError: true, - }, - { - name: "fails if invalid preflight checks are provided", - flags: map[string]string{ - options.IgnorePreflightErrors: "all,something-else", - }, - expectError: true, - }, - - // Init data passed using config file - { - name: "Pass with config from file", - flags: map[string]string{ - options.CfgPath: configFilePath, - }, - }, - { - name: "--cri-socket and --node-name flags override config from file", - flags: map[string]string{ - options.CfgPath: configFilePath, - options.NodeCRISocket: "/var/run/crio/crio.sock", - options.NodeName: "anotherName", - }, - validate: func(t *testing.T, data *initData) { - // validate that cri-socket and node-name are overwritten - if data.cfg.NodeRegistration.CRISocket != "/var/run/crio/crio.sock" { - t.Errorf("Invalid NodeRegistration.CRISocket") - } - if data.cfg.NodeRegistration.Name != "anotherName" { - t.Errorf("Invalid NodeRegistration.Name") - } - }, - }, - { - name: "fail if mixedArguments are passed", - flags: map[string]string{ - options.CfgPath: configFilePath, - options.APIServerAdvertiseAddress: "1.2.3.4", - }, - expectError: true, - }, - - // Pre-flight errors: - { - name: "pre-flights errors from CLI args only", - flags: map[string]string{ - options.IgnorePreflightErrors: "a,b", - }, - validate: expectedInitIgnorePreflightErrors("a", "b"), - }, - { - name: "pre-flights errors from InitConfiguration only", - flags: map[string]string{ - options.CfgPath: configFilePath, - }, - validate: expectedInitIgnorePreflightErrors("c", "d"), - }, - { - name: "pre-flights errors from both CLI args and InitConfiguration", - flags: map[string]string{ - options.CfgPath: configFilePath, - options.IgnorePreflightErrors: "a,b", - }, - validate: expectedInitIgnorePreflightErrors("a", "b", "c", "d"), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // initialize an external init option and inject it to the init cmd - initOptions := newInitOptions() - cmd := newCmdInit(nil, initOptions) - - // sets cmd flags (that will be reflected on the init options) - for f, v := range tc.flags { - cmd.Flags().Set(f, v) - } - - // test newInitData method - data, err := newInitData(cmd, tc.args, initOptions, nil) - if err != nil && !tc.expectError { - t.Fatalf("newInitData returned unexpected error: %v", err) - } - if err == nil && tc.expectError { - t.Fatalf("newInitData didn't return error when expected") - } - - // exec additional validation on the returned value - if tc.validate != nil { - tc.validate(t, data) - } - }) - } -} - -func expectedInitIgnorePreflightErrors(expectedItems ...string) func(t *testing.T, data *initData) { - expected := sets.NewString(expectedItems...) - return func(t *testing.T, data *initData) { - if !expected.Equal(data.ignorePreflightErrors) { - t.Errorf("Invalid ignore preflight errors. Expected: %v. Actual: %v", expected.List(), data.ignorePreflightErrors.List()) - } - if !expected.HasAll(data.cfg.NodeRegistration.IgnorePreflightErrors...) { - t.Errorf("Invalid ignore preflight errors in InitConfiguration. Expected: %v. Actual: %v", expected.List(), data.cfg.NodeRegistration.IgnorePreflightErrors) - } - } -} diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go deleted file mode 100644 index d489081d13725..0000000000000 --- a/cmd/kubeadm/app/cmd/join.go +++ /dev/null @@ -1,554 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "io" - "os" - "strings" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/discovery" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var ( - joinWorkerNodeDoneMsg = dedent.Dedent(` - This node has joined the cluster: - * Certificate signing request was sent to apiserver and a response was received. - * The Kubelet was informed of the new secure connection details. - - Run 'kubectl get nodes' on the control-plane to see this node join the cluster. - - `) - - joinControPlaneDoneTemp = template.Must(template.New("join").Parse(dedent.Dedent(` - This node has joined the cluster and a new control plane instance was created: - - * Certificate signing request was sent to apiserver and approval was received. - * The Kubelet was informed of the new secure connection details. - * Control plane (master) label and taint were applied to the new node. - * The Kubernetes control plane instances scaled up. - {{.etcdMessage}} - - To start administering your cluster from this node, you need to run the following as a regular user: - - mkdir -p $HOME/.kube - sudo cp -i {{.KubeConfigPath}} $HOME/.kube/config - sudo chown $(id -u):$(id -g) $HOME/.kube/config - - Run 'kubectl get nodes' to see this node join the cluster. - - `))) - - joinLongDescription = dedent.Dedent(` - When joining a kubeadm initialized cluster, we need to establish - bidirectional trust. This is split into discovery (having the Node - trust the Kubernetes Control Plane) and TLS bootstrap (having the - Kubernetes Control Plane trust the Node). - - There are 2 main schemes for discovery. The first is to use a shared - token along with the IP address of the API server. The second is to - provide a file - a subset of the standard kubeconfig file. This file - can be a local file or downloaded via an HTTPS URL. The forms are - kubeadm join --discovery-token abcdef.1234567890abcdef 1.2.3.4:6443, - kubeadm join --discovery-file path/to/file.conf, or kubeadm join - --discovery-file https://url/file.conf. Only one form can be used. If - the discovery information is loaded from a URL, HTTPS must be used. - Also, in that case the host installed CA bundle is used to verify - the connection. - - If you use a shared token for discovery, you should also pass the - --discovery-token-ca-cert-hash flag to validate the public key of the - root certificate authority (CA) presented by the Kubernetes Control Plane. - The value of this flag is specified as ":", - where the supported hash type is "sha256". The hash is calculated over - the bytes of the Subject Public Key Info (SPKI) object (as in RFC7469). - This value is available in the output of "kubeadm init" or can be - calculated using standard tools. The --discovery-token-ca-cert-hash flag - may be repeated multiple times to allow more than one public key. - - If you cannot know the CA public key hash ahead of time, you can pass - the --discovery-token-unsafe-skip-ca-verification flag to disable this - verification. This weakens the kubeadm security model since other nodes - can potentially impersonate the Kubernetes Control Plane. - - The TLS bootstrap mechanism is also driven via a shared token. This is - used to temporarily authenticate with the Kubernetes Control Plane to submit a - certificate signing request (CSR) for a locally created key pair. By - default, kubeadm will set up the Kubernetes Control Plane to automatically - approve these signing requests. This token is passed in with the - --tls-bootstrap-token abcdef.1234567890abcdef flag. - - Often times the same token is used for both parts. In this case, the - --token flag can be used instead of specifying each token individually. - `) -) - -// joinOptions defines all the options exposed via flags by kubeadm join. -// Please note that this structure includes the public kubeadm config API, but only a subset of the options -// supported by this api will be exposed as a flag. -type joinOptions struct { - cfgPath string - token string `datapolicy:"token"` - controlPlane bool - ignorePreflightErrors []string - externalcfg *kubeadmapiv1beta2.JoinConfiguration - joinControlPlane *kubeadmapiv1beta2.JoinControlPlane - patchesDir string -} - -// compile-time assert that the local data object satisfies the phases data interface. -var _ phases.JoinData = &joinData{} - -// joinData defines all the runtime information used when running the kubeadm join workflow; -// this data is shared across all the phases that are included in the workflow. -type joinData struct { - cfg *kubeadmapi.JoinConfiguration - initCfg *kubeadmapi.InitConfiguration - tlsBootstrapCfg *clientcmdapi.Config - clientSet *clientset.Clientset - ignorePreflightErrors sets.String - outputWriter io.Writer - patchesDir string -} - -// newCmdJoin returns "kubeadm join" command. -// NB. joinOptions is exposed as parameter for allowing unit testing of -// the newJoinData method, that implements all the command options validation logic -func newCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command { - if joinOptions == nil { - joinOptions = newJoinOptions() - } - joinRunner := workflow.NewRunner() - - cmd := &cobra.Command{ - Use: "join [api-server-endpoint]", - Short: "Run this on any machine you wish to join an existing cluster", - Long: joinLongDescription, - RunE: func(cmd *cobra.Command, args []string) error { - - c, err := joinRunner.InitData(args) - if err != nil { - return err - } - - data := c.(*joinData) - - if err := joinRunner.Run(args); err != nil { - return err - } - - // if the node is hosting a new control plane instance - if data.cfg.ControlPlane != nil { - // outputs the join control plane done message and exit - etcdMessage := "" - if data.initCfg.Etcd.External == nil { - etcdMessage = "* A new etcd member was added to the local/stacked etcd cluster." - } - - ctx := map[string]string{ - "KubeConfigPath": kubeadmconstants.GetAdminKubeConfigPath(), - "etcdMessage": etcdMessage, - } - if err := joinControPlaneDoneTemp.Execute(data.outputWriter, ctx); err != nil { - return err - } - - } else { - // otherwise, if the node joined as a worker node; - // outputs the join done message and exit - fmt.Fprint(data.outputWriter, joinWorkerNodeDoneMsg) - } - - return nil - }, - // We accept the control-plane location as an optional positional argument - Args: cobra.MaximumNArgs(1), - } - - addJoinConfigFlags(cmd.Flags(), joinOptions.externalcfg, joinOptions.joinControlPlane) - addJoinOtherFlags(cmd.Flags(), joinOptions) - - joinRunner.AppendPhase(phases.NewPreflightPhase()) - joinRunner.AppendPhase(phases.NewControlPlanePreparePhase()) - joinRunner.AppendPhase(phases.NewCheckEtcdPhase()) - joinRunner.AppendPhase(phases.NewKubeletStartPhase()) - joinRunner.AppendPhase(phases.NewControlPlaneJoinPhase()) - - // sets the data builder function, that will be used by the runner - // both when running the entire workflow or single phases - joinRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { - return newJoinData(cmd, args, joinOptions, out, kubeadmconstants.GetAdminKubeConfigPath()) - }) - - // binds the Runner to kubeadm join command by altering - // command help, adding --skip-phases flag and by adding phases subcommands - joinRunner.BindToCommand(cmd) - - return cmd -} - -// addJoinConfigFlags adds join flags bound to the config to the specified flagset -func addJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiv1beta2.JoinConfiguration, jcp *kubeadmapiv1beta2.JoinControlPlane) { - flagSet.StringVar( - &cfg.NodeRegistration.Name, options.NodeName, cfg.NodeRegistration.Name, - `Specify the node name.`, - ) - flagSet.StringVar( - &jcp.CertificateKey, options.CertificateKey, jcp.CertificateKey, - "Use this key to decrypt the certificate secrets uploaded by init.", - ) - // add control plane endpoint flags to the specified flagset - flagSet.StringVar( - &jcp.LocalAPIEndpoint.AdvertiseAddress, options.APIServerAdvertiseAddress, jcp.LocalAPIEndpoint.AdvertiseAddress, - "If the node should host a new control plane instance, the IP address the API Server will advertise it's listening on. If not set the default network interface will be used.", - ) - flagSet.Int32Var( - &jcp.LocalAPIEndpoint.BindPort, options.APIServerBindPort, jcp.LocalAPIEndpoint.BindPort, - "If the node should host a new control plane instance, the port for the API Server to bind to.", - ) - // adds bootstrap token specific discovery flags to the specified flagset - flagSet.StringVar( - &cfg.Discovery.BootstrapToken.Token, options.TokenDiscovery, "", - "For token-based discovery, the token used to validate cluster information fetched from the API server.", - ) - flagSet.StringSliceVar( - &cfg.Discovery.BootstrapToken.CACertHashes, options.TokenDiscoveryCAHash, []string{}, - "For token-based discovery, validate that the root CA public key matches this hash (format: \":\").", - ) - flagSet.BoolVar( - &cfg.Discovery.BootstrapToken.UnsafeSkipCAVerification, options.TokenDiscoverySkipCAHash, false, - "For token-based discovery, allow joining without --discovery-token-ca-cert-hash pinning.", - ) - // discovery via kube config file flag - flagSet.StringVar( - &cfg.Discovery.File.KubeConfigPath, options.FileDiscovery, "", - "For file-based discovery, a file or URL from which to load cluster information.", - ) - flagSet.StringVar( - &cfg.Discovery.TLSBootstrapToken, options.TLSBootstrapToken, cfg.Discovery.TLSBootstrapToken, - `Specify the token used to temporarily authenticate with the Kubernetes Control Plane while joining the node.`, - ) - cmdutil.AddCRISocketFlag(flagSet, &cfg.NodeRegistration.CRISocket) -} - -// addJoinOtherFlags adds join flags that are not bound to a configuration file to the given flagset -func addJoinOtherFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) { - flagSet.StringVar( - &joinOptions.cfgPath, options.CfgPath, joinOptions.cfgPath, - "Path to kubeadm config file.", - ) - flagSet.StringSliceVar( - &joinOptions.ignorePreflightErrors, options.IgnorePreflightErrors, joinOptions.ignorePreflightErrors, - "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", - ) - flagSet.StringVar( - &joinOptions.token, options.TokenStr, "", - "Use this token for both discovery-token and tls-bootstrap-token when those values are not provided.", - ) - flagSet.BoolVar( - &joinOptions.controlPlane, options.ControlPlane, joinOptions.controlPlane, - "Create a new control plane instance on this node", - ) - options.AddPatchesFlag(flagSet, &joinOptions.patchesDir) -} - -// newJoinOptions returns a struct ready for being used for creating cmd join flags. -func newJoinOptions() *joinOptions { - // initialize the public kubeadm config API by applying defaults - externalcfg := &kubeadmapiv1beta2.JoinConfiguration{} - - // Add optional config objects to host flags. - // un-set objects will be cleaned up afterwards (into newJoinData func) - externalcfg.Discovery.File = &kubeadmapiv1beta2.FileDiscovery{} - externalcfg.Discovery.BootstrapToken = &kubeadmapiv1beta2.BootstrapTokenDiscovery{} - externalcfg.ControlPlane = &kubeadmapiv1beta2.JoinControlPlane{} - - // This object is used for storage of control-plane flags. - joinControlPlane := &kubeadmapiv1beta2.JoinControlPlane{} - - // Apply defaults - kubeadmscheme.Scheme.Default(externalcfg) - kubeadmapiv1beta2.SetDefaults_JoinControlPlane(joinControlPlane) - - return &joinOptions{ - externalcfg: externalcfg, - joinControlPlane: joinControlPlane, - } -} - -// newJoinData returns a new joinData struct to be used for the execution of the kubeadm join workflow. -// This func takes care of validating joinOptions passed to the command, and then it converts -// options into the internal JoinConfiguration type that is used as input all the phases in the kubeadm join workflow -func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Writer, adminKubeConfigPath string) (*joinData, error) { - - // Validate the mixed arguments with --config and return early on errors - if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { - return nil, err - } - - // Re-apply defaults to the public kubeadm API (this will set only values not exposed/not set as a flags) - kubeadmscheme.Scheme.Default(opt.externalcfg) - - // Validate standalone flags values and/or combination of flags and then assigns - // validated values to the public kubeadm config API when applicable - - // if a token is provided, use this value for both discovery-token and tls-bootstrap-token when those values are not provided - if len(opt.token) > 0 { - if len(opt.externalcfg.Discovery.TLSBootstrapToken) == 0 { - opt.externalcfg.Discovery.TLSBootstrapToken = opt.token - } - if len(opt.externalcfg.Discovery.BootstrapToken.Token) == 0 { - opt.externalcfg.Discovery.BootstrapToken.Token = opt.token - } - } - - // if a file or URL from which to load cluster information was not provided, unset the Discovery.File object - if len(opt.externalcfg.Discovery.File.KubeConfigPath) == 0 { - opt.externalcfg.Discovery.File = nil - } - - // if an APIServerEndpoint from which to retrieve cluster information was not provided, unset the Discovery.BootstrapToken object - if len(args) == 0 { - opt.externalcfg.Discovery.BootstrapToken = nil - } else { - if len(opt.cfgPath) == 0 && len(args) > 1 { - klog.Warningf("[preflight] WARNING: More than one API server endpoint supplied on command line %v. Using the first one.", args) - } - opt.externalcfg.Discovery.BootstrapToken.APIServerEndpoint = args[0] - } - - // Include the JoinControlPlane with user flags - opt.externalcfg.ControlPlane = opt.joinControlPlane - - // If not passing --control-plane, unset the ControlPlane object - if !opt.controlPlane { - // Use a defaulted JoinControlPlane object to detect if the user has passed - // other control-plane related flags. - defaultJCP := &kubeadmapiv1beta2.JoinControlPlane{} - kubeadmapiv1beta2.SetDefaults_JoinControlPlane(defaultJCP) - - // This list must match the JCP flags in addJoinConfigFlags() - joinControlPlaneFlags := []string{ - options.CertificateKey, - options.APIServerAdvertiseAddress, - options.APIServerBindPort, - } - - if *opt.joinControlPlane != *defaultJCP { - klog.Warningf("[preflight] WARNING: --%s is also required when passing control-plane "+ - "related flags such as [%s]", options.ControlPlane, strings.Join(joinControlPlaneFlags, ", ")) - } - opt.externalcfg.ControlPlane = nil - } - - // if the admin.conf file already exists, use it for skipping the discovery process. - // NB. this case can happen when we are joining a control-plane node only (and phases are invoked atomically) - var tlsBootstrapCfg *clientcmdapi.Config - if _, err := os.Stat(adminKubeConfigPath); err == nil && opt.controlPlane { - // use the admin.conf as tlsBootstrapCfg, that is the kubeconfig file used for reading the kubeadm-config during discovery - klog.V(1).Infof("[preflight] found %s. Use it for skipping discovery", adminKubeConfigPath) - tlsBootstrapCfg, err = clientcmd.LoadFromFile(adminKubeConfigPath) - if err != nil { - return nil, errors.Wrapf(err, "Error loading %s", adminKubeConfigPath) - } - } - - // Either use the config file if specified, or convert public kubeadm API to the internal JoinConfiguration - // and validates JoinConfiguration - if opt.externalcfg.NodeRegistration.Name == "" { - klog.V(1).Infoln("[preflight] found NodeName empty; using OS hostname as NodeName") - } - - if opt.externalcfg.ControlPlane != nil && opt.externalcfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress == "" { - klog.V(1).Infoln("[preflight] found advertiseAddress empty; using default interface's IP address as advertiseAddress") - } - - // in case the command doesn't have flags for discovery, makes the join cfg validation pass checks on discovery - if cmd.Flags().Lookup(options.FileDiscovery) == nil { - if _, err := os.Stat(adminKubeConfigPath); os.IsNotExist(err) { - return nil, errors.Errorf("File %s does not exists. Please use 'kubeadm join phase control-plane-prepare' subcommands to generate it.", adminKubeConfigPath) - } - klog.V(1).Infof("[preflight] found discovery flags missing for this command. using FileDiscovery: %s", adminKubeConfigPath) - opt.externalcfg.Discovery.File = &kubeadmapiv1beta2.FileDiscovery{KubeConfigPath: adminKubeConfigPath} - opt.externalcfg.Discovery.BootstrapToken = nil //NB. this could be removed when we get better control on args (e.g. phases without discovery should have NoArgs ) - } - - cfg, err := configutil.LoadOrDefaultJoinConfiguration(opt.cfgPath, opt.externalcfg) - if err != nil { - return nil, err - } - - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opt.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) - if err != nil { - return nil, err - } - // Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration: - cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() - - // override node name and CRI socket from the command line opt - if opt.externalcfg.NodeRegistration.Name != "" { - cfg.NodeRegistration.Name = opt.externalcfg.NodeRegistration.Name - } - if opt.externalcfg.NodeRegistration.CRISocket != "" { - cfg.NodeRegistration.CRISocket = opt.externalcfg.NodeRegistration.CRISocket - } - - if cfg.ControlPlane != nil { - if err := configutil.VerifyAPIServerBindAddress(cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress); err != nil { - return nil, err - } - } - - return &joinData{ - cfg: cfg, - tlsBootstrapCfg: tlsBootstrapCfg, - ignorePreflightErrors: ignorePreflightErrorsSet, - outputWriter: out, - patchesDir: opt.patchesDir, - }, nil -} - -// CertificateKey returns the key used to encrypt the certs. -func (j *joinData) CertificateKey() string { - if j.cfg.ControlPlane != nil { - return j.cfg.ControlPlane.CertificateKey - } - return "" -} - -// Cfg returns the JoinConfiguration. -func (j *joinData) Cfg() *kubeadmapi.JoinConfiguration { - return j.cfg -} - -// TLSBootstrapCfg returns the cluster-info (kubeconfig). -func (j *joinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) { - if j.tlsBootstrapCfg != nil { - return j.tlsBootstrapCfg, nil - } - klog.V(1).Infoln("[preflight] Discovering cluster-info") - tlsBootstrapCfg, err := discovery.For(j.cfg) - j.tlsBootstrapCfg = tlsBootstrapCfg - return tlsBootstrapCfg, err -} - -// InitCfg returns the InitConfiguration. -func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { - if j.initCfg != nil { - return j.initCfg, nil - } - if _, err := j.TLSBootstrapCfg(); err != nil { - return nil, err - } - klog.V(1).Infoln("[preflight] Fetching init configuration") - initCfg, err := fetchInitConfigurationFromJoinConfiguration(j.cfg, j.tlsBootstrapCfg) - j.initCfg = initCfg - return initCfg, err -} - -// ClientSet returns the ClientSet for accessing the cluster with the identity defined in admin.conf. -func (j *joinData) ClientSet() (*clientset.Clientset, error) { - if j.clientSet != nil { - return j.clientSet, nil - } - path := kubeadmconstants.GetAdminKubeConfigPath() - client, err := kubeconfigutil.ClientSetFromFile(path) - if err != nil { - return nil, errors.Wrap(err, "[preflight] couldn't create Kubernetes client") - } - j.clientSet = client - return client, nil -} - -// IgnorePreflightErrors returns the list of preflight errors to ignore. -func (j *joinData) IgnorePreflightErrors() sets.String { - return j.ignorePreflightErrors -} - -// OutputWriter returns the io.Writer used to write messages such as the "join done" message. -func (j *joinData) OutputWriter() io.Writer { - return j.outputWriter -} - -// PatchesDir returns the folder where patches for components are stored -func (j *joinData) PatchesDir() string { - return j.patchesDir -} - -// fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery -func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { - // Retrieves the kubeadm configuration - klog.V(1).Infoln("[preflight] Retrieving KubeConfig objects") - initConfiguration, err := fetchInitConfiguration(tlsBootstrapCfg) - if err != nil { - return nil, err - } - - // Create the final KubeConfig file with the cluster name discovered after fetching the cluster configuration - clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) - tlsBootstrapCfg.Clusters = map[string]*clientcmdapi.Cluster{ - initConfiguration.ClusterName: clusterinfo, - } - tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster = initConfiguration.ClusterName - - // injects into the kubeadm configuration the information about the joining node - initConfiguration.NodeRegistration = cfg.NodeRegistration - if cfg.ControlPlane != nil { - initConfiguration.LocalAPIEndpoint = cfg.ControlPlane.LocalAPIEndpoint - } - - return initConfiguration, nil -} - -// fetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap -func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) { - // creates a client to access the cluster using the bootstrap token identity - tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) - if err != nil { - return nil, errors.Wrap(err, "unable to access the cluster") - } - - // Fetches the init configuration - initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, os.Stdout, "preflight", true, false) - if err != nil { - return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") - } - - return initConfiguration, nil -} diff --git a/cmd/kubeadm/app/cmd/join_test.go b/cmd/kubeadm/app/cmd/join_test.go deleted file mode 100644 index db08625741941..0000000000000 --- a/cmd/kubeadm/app/cmd/join_test.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -const ( - testJoinConfig = `apiVersion: kubeadm.k8s.io/v1beta2 -kind: JoinConfiguration -discovery: - bootstrapToken: - token: abcdef.0123456789abcdef - apiServerEndpoint: 1.2.3.4:6443 - unsafeSkipCAVerification: true -nodeRegistration: - criSocket: /run/containerd/containerd.sock - name: someName - ignorePreflightErrors: - - c - - d -` -) - -func TestNewJoinData(t *testing.T) { - // create temp directory - tmpDir, err := ioutil.TempDir("", "kubeadm-join-test") - if err != nil { - t.Errorf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - - // create kubeconfig - kubeconfigFilePath := filepath.Join(tmpDir, "test-kubeconfig-file") - kubeconfig := kubeconfigutil.CreateBasic("", "", "", []byte{}) - kubeconfigutil.WriteToDisk(kubeconfigFilePath, kubeconfig) - - // create config file - configFilePath := filepath.Join(tmpDir, "test-config-file") - cfgFile, err := os.Create(configFilePath) - if err != nil { - t.Errorf("Unable to create file %q: %v", configFilePath, err) - } - defer cfgFile.Close() - if _, err = cfgFile.WriteString(testJoinConfig); err != nil { - t.Fatalf("Unable to write file %q: %v", configFilePath, err) - } - - testCases := []struct { - name string - args []string - flags map[string]string - validate func(*testing.T, *joinData) - expectError bool - }{ - // Join data passed using flags - { - name: "fails if no discovery method set", - expectError: true, - }, - { - name: "fails if both file and bootstrap discovery methods set", - args: []string{"1.2.3.4:6443"}, - flags: map[string]string{ - options.FileDiscovery: "https://foo", - options.TokenDiscovery: "abcdef.0123456789abcdef", - options.TokenDiscoverySkipCAHash: "true", - }, - expectError: true, - }, - { - name: "pass if file discovery is set", - flags: map[string]string{ - options.FileDiscovery: "https://foo", - }, - validate: func(t *testing.T, data *joinData) { - // validate that file discovery settings are set into join data - if data.cfg.Discovery.File == nil || data.cfg.Discovery.File.KubeConfigPath != "https://foo" { - t.Errorf("Invalid data.cfg.Discovery.File") - } - }, - }, - { - name: "pass if bootstrap discovery is set", - args: []string{"1.2.3.4:6443", "5.6.7.8:6443"}, - flags: map[string]string{ - options.TokenDiscovery: "abcdef.0123456789abcdef", - options.TokenDiscoverySkipCAHash: "true", - }, - validate: func(t *testing.T, data *joinData) { - // validate that bootstrap discovery settings are set into join data - if data.cfg.Discovery.BootstrapToken == nil || - data.cfg.Discovery.BootstrapToken.APIServerEndpoint != "1.2.3.4:6443" || //only first arg should be kept as APIServerEndpoint - data.cfg.Discovery.BootstrapToken.Token != "abcdef.0123456789abcdef" || - data.cfg.Discovery.BootstrapToken.UnsafeSkipCAVerification != true { - t.Errorf("Invalid data.cfg.Discovery.BootstrapToken") - } - }, - }, - { - name: "--token sets TLSBootstrapToken and BootstrapToken.Token if unset", - args: []string{"1.2.3.4:6443"}, - flags: map[string]string{ - options.TokenStr: "abcdef.0123456789abcdef", - options.TokenDiscoverySkipCAHash: "true", - }, - validate: func(t *testing.T, data *joinData) { - // validate that token sets both TLSBootstrapToken and BootstrapToken.Token into join data - if data.cfg.Discovery.TLSBootstrapToken != "abcdef.0123456789abcdef" || - data.cfg.Discovery.BootstrapToken == nil || - data.cfg.Discovery.BootstrapToken.Token != "abcdef.0123456789abcdef" { - t.Errorf("Invalid TLSBootstrapToken or BootstrapToken.Token") - } - }, - }, - { - name: "--token doesn't override TLSBootstrapToken and BootstrapToken.Token if set", - args: []string{"1.2.3.4:6443"}, - flags: map[string]string{ - options.TokenStr: "aaaaaa.0123456789aaaaaa", - options.TLSBootstrapToken: "abcdef.0123456789abcdef", - options.TokenDiscovery: "defghi.0123456789defghi", - options.TokenDiscoverySkipCAHash: "true", - }, - validate: func(t *testing.T, data *joinData) { - // validate that TLSBootstrapToken and BootstrapToken.Token values are preserved into join data - if data.cfg.Discovery.TLSBootstrapToken != "abcdef.0123456789abcdef" || - data.cfg.Discovery.BootstrapToken == nil || - data.cfg.Discovery.BootstrapToken.Token != "defghi.0123456789defghi" { - t.Errorf("Invalid TLSBootstrapToken or BootstrapToken.Token") - } - }, - }, - { - name: "control plane setting are preserved if --control-plane flag is set", - flags: map[string]string{ - options.ControlPlane: "true", - options.APIServerAdvertiseAddress: "1.2.3.4", - options.APIServerBindPort: "1234", - options.FileDiscovery: "https://foo", //required only to pass discovery validation - }, - validate: func(t *testing.T, data *joinData) { - // validate that control plane attributes are set in join data - if data.cfg.ControlPlane == nil || - data.cfg.ControlPlane.LocalAPIEndpoint.AdvertiseAddress != "1.2.3.4" || - data.cfg.ControlPlane.LocalAPIEndpoint.BindPort != 1234 { - t.Errorf("Invalid ControlPlane") - } - }, - }, - { - name: "control plane setting are cleaned up if --control-plane flag is not set", - flags: map[string]string{ - options.ControlPlane: "false", - options.APIServerAdvertiseAddress: "1.2.3.4", - options.APIServerBindPort: "1.2.3.4", - options.FileDiscovery: "https://foo", //required only to pass discovery validation - }, - validate: func(t *testing.T, data *joinData) { - // validate that control plane attributes are unset in join data - if data.cfg.ControlPlane != nil { - t.Errorf("Invalid ControlPlane") - } - }, - }, - { - name: "fails if invalid preflight checks are provided", - flags: map[string]string{ - options.IgnorePreflightErrors: "all,something-else", - }, - expectError: true, - }, - - // Join data passed using config file - { - name: "Pass with config from file", - flags: map[string]string{ - options.CfgPath: configFilePath, - }, - }, - { - name: "--cri-socket and --node-name flags override config from file", - flags: map[string]string{ - options.CfgPath: configFilePath, - options.NodeCRISocket: "/var/run/crio/crio.sock", - options.NodeName: "anotherName", - }, - validate: func(t *testing.T, data *joinData) { - // validate that cri-socket and node-name are overwritten - if data.cfg.NodeRegistration.CRISocket != "/var/run/crio/crio.sock" { - t.Errorf("Invalid NodeRegistration.CRISocket") - } - if data.cfg.NodeRegistration.Name != "anotherName" { - t.Errorf("Invalid NodeRegistration.Name") - } - }, - }, - { - name: "fail if mixedArguments are passed", - flags: map[string]string{ - options.CfgPath: configFilePath, - options.APIServerAdvertiseAddress: "1.2.3.4", - }, - expectError: true, - }, - - // Pre-flight errors: - { - name: "pre-flights errors from CLI args only", - flags: map[string]string{ - options.IgnorePreflightErrors: "a,b", - options.FileDiscovery: "https://foo", //required only to pass discovery validation - }, - validate: expectedJoinIgnorePreflightErrors(sets.NewString("a", "b")), - }, - { - name: "pre-flights errors from JoinConfiguration only", - flags: map[string]string{ - options.CfgPath: configFilePath, - }, - validate: expectedJoinIgnorePreflightErrors(sets.NewString("c", "d")), - }, - { - name: "pre-flights errors from both CLI args and JoinConfiguration", - flags: map[string]string{ - options.CfgPath: configFilePath, - options.IgnorePreflightErrors: "a,b", - }, - validate: expectedJoinIgnorePreflightErrors(sets.NewString("a", "b", "c", "d")), - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // initialize an external join option and inject it to the join cmd - joinOptions := newJoinOptions() - cmd := newCmdJoin(nil, joinOptions) - - // sets cmd flags (that will be reflected on the join options) - for f, v := range tc.flags { - cmd.Flags().Set(f, v) - } - - // test newJoinData method - data, err := newJoinData(cmd, tc.args, joinOptions, nil, kubeconfigFilePath) - if err != nil && !tc.expectError { - t.Fatalf("newJoinData returned unexpected error: %v", err) - } - if err == nil && tc.expectError { - t.Fatalf("newJoinData didn't return error when expected") - } - - // exec additional validation on the returned value - if tc.validate != nil { - tc.validate(t, data) - } - }) - } -} - -func expectedJoinIgnorePreflightErrors(expected sets.String) func(t *testing.T, data *joinData) { - return func(t *testing.T, data *joinData) { - if !expected.Equal(data.ignorePreflightErrors) { - t.Errorf("Invalid ignore preflight errors. Expected: %v. Actual: %v", expected.List(), data.ignorePreflightErrors.List()) - } - if !expected.HasAll(data.cfg.NodeRegistration.IgnorePreflightErrors...) { - t.Errorf("Invalid ignore preflight errors in JoinConfiguration. Expected: %v. Actual: %v", expected.List(), data.cfg.NodeRegistration.IgnorePreflightErrors) - } - } -} diff --git a/cmd/kubeadm/app/cmd/options/BUILD b/cmd/kubeadm/app/cmd/options/BUILD deleted file mode 100644 index 78ae7b0809c2f..0000000000000 --- a/cmd/kubeadm/app/cmd/options/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = [ - "certs.go", - "constant.go", - "doc.go", - "generic.go", - "token.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/options/certs.go b/cmd/kubeadm/app/cmd/options/certs.go deleted file mode 100644 index 9d826dc8448b5..0000000000000 --- a/cmd/kubeadm/app/cmd/options/certs.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import "github.com/spf13/pflag" - -// AddCertificateDirFlag adds the --certs-dir flag to the given flagset -func AddCertificateDirFlag(fs *pflag.FlagSet, certsDir *string) { - fs.StringVar(certsDir, CertificatesDir, *certsDir, "The path where to save the certificates") -} - -// AddCSRFlag adds the --csr-only flag to the given flagset -func AddCSRFlag(fs *pflag.FlagSet, csr *bool) { - fs.BoolVar(csr, CSROnly, *csr, "Create CSRs instead of generating certificates") -} - -// AddCSRDirFlag adds the --csr-dir flag to the given flagset -func AddCSRDirFlag(fs *pflag.FlagSet, csrDir *string) { - fs.StringVar(csrDir, CSRDir, *csrDir, "The path to output the CSRs and private keys to") -} diff --git a/cmd/kubeadm/app/cmd/options/constant.go b/cmd/kubeadm/app/cmd/options/constant.go deleted file mode 100644 index 7d157f719b38e..0000000000000 --- a/cmd/kubeadm/app/cmd/options/constant.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -const ( - // APIServerAdvertiseAddress flag sets the IP address the API Server will advertise it's listening on. Specify '0.0.0.0' to use the address of the default network interface. - APIServerAdvertiseAddress = "apiserver-advertise-address" - - // APIServerBindPort flag sets the port for the API Server to bind to. - APIServerBindPort = "apiserver-bind-port" - - // APIServerCertSANs flag sets extra Subject Alternative Names (SANs) to use for the API Server serving certificate. Can be both IP addresses and DNS names. - APIServerCertSANs = "apiserver-cert-extra-sans" - - // APIServerExtraArgs flag sets a extra flags to pass to the API Server or override default ones in form of =. - APIServerExtraArgs = "apiserver-extra-args" - - // CertificatesDir flag sets the path where to save and read the certificates. - CertificatesDir = "cert-dir" - - // CfgPath flag sets the path to kubeadm config file. - CfgPath = "config" - - // ControllerManagerExtraArgs flag sets extra flags to pass to the Controller Manager or override default ones in form of =. - ControllerManagerExtraArgs = "controller-manager-extra-args" - - // ControlPlaneEndpoint flag sets a stable IP address or DNS name for the control plane. - ControlPlaneEndpoint = "control-plane-endpoint" - - // DryRun flag instruct kubeadm to don't apply any changes; just output what would be done. - DryRun = "dry-run" - - // FeatureGatesString flag sets key=value pairs that describe feature gates for various features. - FeatureGatesString = "feature-gates" - - // IgnorePreflightErrors sets the path a list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks. - IgnorePreflightErrors = "ignore-preflight-errors" - - // ImageRepository sets the container registry to pull control plane images from. - ImageRepository = "image-repository" - - // KubeconfigDir flag sets the path where to save the kubeconfig file. - KubeconfigDir = "kubeconfig-dir" - - // KubeconfigPath flag sets the kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations are searched for an existing KubeConfig file. - KubeconfigPath = "kubeconfig" - - // KubernetesVersion flag sets the Kubernetes version for the control plane. - KubernetesVersion = "kubernetes-version" - - // KubeletVersion flag sets the version for the kubelet config. - KubeletVersion = "kubelet-version" - - // NetworkingDNSDomain flag sets the domain for services, e.g. "myorg.internal". - NetworkingDNSDomain = "service-dns-domain" - - // NetworkingServiceSubnet flag sets the range of IP address for service VIPs. - NetworkingServiceSubnet = "service-cidr" - - // NetworkingPodSubnet flag sets the range of IP addresses for the pod network. If set, the control plane will automatically allocate CIDRs for every node. - NetworkingPodSubnet = "pod-network-cidr" - - // NodeCRISocket flag sets the CRI socket to connect to. - NodeCRISocket = "cri-socket" - - // NodeName flag sets the node name. - NodeName = "node-name" - - // SchedulerExtraArgs flag sets extra flags to pass to the Scheduler or override default ones in form of =". - SchedulerExtraArgs = "scheduler-extra-args" - - // SkipTokenPrint flag instruct kubeadm to skip printing of the default bootstrap token generated by 'kubeadm init'. - SkipTokenPrint = "skip-token-print" - - // CSROnly flag instructs kubeadm to create CSRs instead of automatically creating or renewing certs - CSROnly = "csr-only" - - // CSRDir flag sets the location for CSRs and flags to be output - CSRDir = "csr-dir" - - // TokenStr flags sets both the discovery-token and the tls-bootstrap-token when those values are not provided - TokenStr = "token" - - // TokenTTL flag sets the time to live for token - TokenTTL = "token-ttl" - - // TokenUsages flag sets the usages of the token - TokenUsages = "usages" - - // TokenGroups flag sets the authentication groups of the token - TokenGroups = "groups" - - // TokenDescription flag sets the description of the token - TokenDescription = "description" - - // TLSBootstrapToken flag sets the token used to temporarily authenticate with the Kubernetes Control Plane to submit a certificate signing request (CSR) for a locally created key pair - TLSBootstrapToken = "tls-bootstrap-token" - - // TokenDiscovery flag sets the token used to validate cluster information fetched from the API server (for token-based discovery) - TokenDiscovery = "discovery-token" - - // TokenDiscoveryCAHash flag instruct kubeadm to validate that the root CA public key matches this hash (for token-based discovery) - TokenDiscoveryCAHash = "discovery-token-ca-cert-hash" - - // TokenDiscoverySkipCAHash flag instruct kubeadm to skip CA hash verification (for token-based discovery) - TokenDiscoverySkipCAHash = "discovery-token-unsafe-skip-ca-verification" - - // FileDiscovery flag sets the file or URL from which to load cluster information (for file-based discovery) - FileDiscovery = "discovery-file" - - // ControlPlane flag instruct kubeadm to create a new control plane instance on this node - ControlPlane = "control-plane" - - // UploadCerts flag instruct kubeadm to upload certificates - UploadCerts = "upload-certs" - - // CertificateKey flag sets the key used to encrypt and decrypt certificate secrets - CertificateKey = "certificate-key" - - // SkipCertificateKeyPrint flag instruct kubeadm to skip printing certificate key used to encrypt certs by 'kubeadm init'. - SkipCertificateKeyPrint = "skip-certificate-key-print" - - // ForceReset flag instruct kubeadm to reset the node without prompting for confirmation - ForceReset = "force" - - // CertificateRenewal flag instruct kubeadm to execute certificate renewal during upgrades - CertificateRenewal = "certificate-renewal" - - // EtcdUpgrade flag instruct kubeadm to execute etcd upgrade during upgrades - EtcdUpgrade = "etcd-upgrade" - - // Patches flag sets the folder where kubeadm component patches are stored - Patches = "experimental-patches" -) diff --git a/cmd/kubeadm/app/cmd/options/doc.go b/cmd/kubeadm/app/cmd/options/doc.go deleted file mode 100644 index 8e04d08917e96..0000000000000 --- a/cmd/kubeadm/app/cmd/options/doc.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Package options provide a central point for defining flags for kubeadm cobra commands, -no matter if hard coded commands or autogenerated command for phases. - -New kubeadm flags should always be defined in this package as a constant before their usage, -in order to enforce naming consistency across different commands and to control flag proliferation. - -In addition to defining the flags, the package also contains set of utilities for flag management. - -For additional details about how flags are managed in phases, please refer to the -"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" package. -*/ -package options diff --git a/cmd/kubeadm/app/cmd/options/generic.go b/cmd/kubeadm/app/cmd/options/generic.go deleted file mode 100644 index fc9630dbc76be..0000000000000 --- a/cmd/kubeadm/app/cmd/options/generic.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "strings" - - "github.com/spf13/pflag" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" -) - -// AddKubeConfigFlag adds the --kubeconfig flag to the given flagset -func AddKubeConfigFlag(fs *pflag.FlagSet, kubeConfigFile *string) { - fs.StringVar(kubeConfigFile, KubeconfigPath, *kubeConfigFile, "The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations can be searched for an existing kubeconfig file.") - // Note that DefValue is the text shown in the terminal and not the default value assigned to the flag - fs.Lookup(KubeconfigPath).DefValue = constants.GetAdminKubeConfigPath() -} - -// AddKubeConfigDirFlag adds the --kubeconfig-dir flag to the given flagset -func AddKubeConfigDirFlag(fs *pflag.FlagSet, kubeConfigDir *string) { - fs.StringVar(kubeConfigDir, KubeconfigDir, *kubeConfigDir, "The path where to save the kubeconfig file.") -} - -// AddConfigFlag adds the --config flag to the given flagset -func AddConfigFlag(fs *pflag.FlagSet, cfgPath *string) { - fs.StringVar(cfgPath, CfgPath, *cfgPath, "Path to a kubeadm configuration file.") -} - -// AddIgnorePreflightErrorsFlag adds the --ignore-preflight-errors flag to the given flagset -func AddIgnorePreflightErrorsFlag(fs *pflag.FlagSet, ignorePreflightErrors *[]string) { - fs.StringSliceVar( - ignorePreflightErrors, IgnorePreflightErrors, *ignorePreflightErrors, - "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.", - ) -} - -// AddControlPlanExtraArgsFlags adds the ExtraArgs flags for control plane components -func AddControlPlanExtraArgsFlags(fs *pflag.FlagSet, apiServerExtraArgs, controllerManagerExtraArgs, schedulerExtraArgs *map[string]string) { - fs.Var(cliflag.NewMapStringString(apiServerExtraArgs), APIServerExtraArgs, "A set of extra flags to pass to the API Server or override default ones in form of =") - fs.Var(cliflag.NewMapStringString(controllerManagerExtraArgs), ControllerManagerExtraArgs, "A set of extra flags to pass to the Controller Manager or override default ones in form of =") - fs.Var(cliflag.NewMapStringString(schedulerExtraArgs), SchedulerExtraArgs, "A set of extra flags to pass to the Scheduler or override default ones in form of =") -} - -// AddImageMetaFlags adds the --image-repository flag to the given flagset -func AddImageMetaFlags(fs *pflag.FlagSet, imageRepository *string) { - fs.StringVar(imageRepository, ImageRepository, *imageRepository, "Choose a container registry to pull control plane images from") -} - -// AddFeatureGatesStringFlag adds the --feature-gates flag to the given flagset -func AddFeatureGatesStringFlag(fs *pflag.FlagSet, featureGatesString *string) { - if knownFeatures := features.KnownFeatures(&features.InitFeatureGates); len(knownFeatures) > 0 { - fs.StringVar(featureGatesString, FeatureGatesString, *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ - "Options are:\n"+strings.Join(knownFeatures, "\n")) - } else { - fs.StringVar(featureGatesString, FeatureGatesString, *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ - "No feature gates are available in this release.") - } -} - -// AddKubernetesVersionFlag adds the --kubernetes-version flag to the given flagset -func AddKubernetesVersionFlag(fs *pflag.FlagSet, kubernetesVersion *string) { - fs.StringVar( - kubernetesVersion, KubernetesVersion, *kubernetesVersion, - `Choose a specific Kubernetes version for the control plane.`, - ) -} - -// AddKubeadmOtherFlags adds flags that are not bound to a configuration file to the given flagset -func AddKubeadmOtherFlags(flagSet *pflag.FlagSet, rootfsPath *string) { - flagSet.StringVar( - rootfsPath, "rootfs", *rootfsPath, - "[EXPERIMENTAL] The path to the 'real' host root filesystem.", - ) -} - -// AddPatchesFlag adds the --patches flag to the given flagset -func AddPatchesFlag(fs *pflag.FlagSet, patchesDir *string) { - fs.StringVar(patchesDir, Patches, *patchesDir, `Path to a directory that contains files named `+ - `"target[suffix][+patchtype].extension". For example, `+ - `"kube-apiserver0+merge.yaml" or just "etcd.json". `+ - `"patchtype" can be one of "strategic", "merge" or "json" and they match the patch formats `+ - `supported by kubectl. The default "patchtype" is "strategic". "extension" must be either `+ - `"json" or "yaml". "suffix" is an optional string that can be used to determine `+ - `which patches are applied first alpha-numerically.`, - ) -} diff --git a/cmd/kubeadm/app/cmd/options/token.go b/cmd/kubeadm/app/cmd/options/token.go deleted file mode 100644 index ac043df315302..0000000000000 --- a/cmd/kubeadm/app/cmd/options/token.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "strings" - - "github.com/spf13/pflag" - - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// NewBootstrapTokenOptions creates a new BootstrapTokenOptions object with the default values -func NewBootstrapTokenOptions() *BootstrapTokenOptions { - bto := &BootstrapTokenOptions{&kubeadmapiv1beta2.BootstrapToken{}, ""} - kubeadmapiv1beta2.SetDefaults_BootstrapToken(bto.BootstrapToken) - return bto -} - -// BootstrapTokenOptions is a wrapper struct for adding bootstrap token-related flags to a FlagSet -// and applying the parsed flags to a InitConfiguration object later at runtime -// TODO: In the future, we might want to group the flags in a better way than adding them all individually like this -type BootstrapTokenOptions struct { - *kubeadmapiv1beta2.BootstrapToken - TokenStr string `datapolicy:"token"` -} - -// AddTokenFlag adds the --token flag to the given flagset -func (bto *BootstrapTokenOptions) AddTokenFlag(fs *pflag.FlagSet) { - fs.StringVar( - &bto.TokenStr, TokenStr, "", - "The token to use for establishing bidirectional trust between nodes and control-plane nodes. The format is [a-z0-9]{6}\\.[a-z0-9]{16} - e.g. abcdef.0123456789abcdef", - ) -} - -// AddTTLFlag adds the --token-ttl flag to the given flagset -func (bto *BootstrapTokenOptions) AddTTLFlag(fs *pflag.FlagSet) { - bto.AddTTLFlagWithName(fs, TokenTTL) -} - -// AddTTLFlagWithName adds the --token-ttl flag with a custom flag name given flagset -func (bto *BootstrapTokenOptions) AddTTLFlagWithName(fs *pflag.FlagSet, flagName string) { - fs.DurationVar( - &bto.TTL.Duration, flagName, bto.TTL.Duration, - "The duration before the token is automatically deleted (e.g. 1s, 2m, 3h). If set to '0', the token will never expire", - ) -} - -// AddUsagesFlag adds the --usages flag to the given flagset -func (bto *BootstrapTokenOptions) AddUsagesFlag(fs *pflag.FlagSet) { - fs.StringSliceVar( - &bto.Usages, TokenUsages, bto.Usages, - fmt.Sprintf("Describes the ways in which this token can be used. You can pass --usages multiple times or provide a comma separated list of options. Valid options: [%s]", strings.Join(kubeadmconstants.DefaultTokenUsages, ",")), - ) -} - -// AddGroupsFlag adds the --groups flag to the given flagset -func (bto *BootstrapTokenOptions) AddGroupsFlag(fs *pflag.FlagSet) { - fs.StringSliceVar( - &bto.Groups, TokenGroups, bto.Groups, - fmt.Sprintf("Extra groups that this token will authenticate as when used for authentication. Must match %q", bootstrapapi.BootstrapGroupPattern), - ) -} - -// AddDescriptionFlag adds the --description flag to the given flagset -func (bto *BootstrapTokenOptions) AddDescriptionFlag(fs *pflag.FlagSet) { - fs.StringVar( - &bto.Description, TokenDescription, bto.Description, - "A human friendly description of how this token is used.", - ) -} - -// ApplyTo applies the values set internally in the BootstrapTokenOptions object to a InitConfiguration object at runtime -// If --token was specified in the CLI (as a string), it's parsed and validated before it's added to the BootstrapToken object. -func (bto *BootstrapTokenOptions) ApplyTo(cfg *kubeadmapiv1beta2.InitConfiguration) error { - if len(bto.TokenStr) > 0 { - var err error - bto.Token, err = kubeadmapiv1beta2.NewBootstrapTokenString(bto.TokenStr) - if err != nil { - return err - } - } - - // Set the token specified by the flags as the first and only token to create in case --config is not specified - cfg.BootstrapTokens = []kubeadmapiv1beta2.BootstrapToken{*bto.BootstrapToken} - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/BUILD b/cmd/kubeadm/app/cmd/phases/BUILD deleted file mode 100644 index 974a28b372abd..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["util.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["util_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/cmd/phases/init:all-srcs", - "//cmd/kubeadm/app/cmd/phases/join:all-srcs", - "//cmd/kubeadm/app/cmd/phases/reset:all-srcs", - "//cmd/kubeadm/app/cmd/phases/upgrade/node:all-srcs", - "//cmd/kubeadm/app/cmd/phases/workflow:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/phases/init/BUILD b/cmd/kubeadm/app/cmd/phases/init/BUILD deleted file mode 100644 index 2891c075b9283..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/BUILD +++ /dev/null @@ -1,93 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "addons.go", - "bootstraptoken.go", - "certs.go", - "controlplane.go", - "data.go", - "etcd.go", - "kubeconfig.go", - "kubelet.go", - "kubeletfinalize.go", - "markcontrolplane.go", - "preflight.go", - "uploadcerts.go", - "uploadconfig.go", - "waitcontrolplane.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/init", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/cmd/util:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/addons/dns:go_default_library", - "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/controlplane:go_default_library", - "//cmd/kubeadm/app/phases/copycerts:go_default_library", - "//cmd/kubeadm/app/phases/etcd:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/phases/kubelet:go_default_library", - "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", - "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/dryrun:go_default_library", - "//cmd/kubeadm/app/util/etcd:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "certs_test.go", - "data_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util/certs:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/phases/init/addons.go b/cmd/kubeadm/app/cmd/phases/init/addons.go deleted file mode 100644 index 005d7a713de4d..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/addons.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "github.com/pkg/errors" - - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" - proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" -) - -var ( - coreDNSAddonLongDesc = cmdutil.LongDesc(` - Install the CoreDNS addon components via the API server. - Please note that although the DNS server is deployed, it will not be scheduled until CNI is installed. - `) - - kubeProxyAddonLongDesc = cmdutil.LongDesc(` - Install the kube-proxy addon components via the API server. - `) -) - -// NewAddonPhase returns the addon Cobra command -func NewAddonPhase() workflow.Phase { - return workflow.Phase{ - Name: "addon", - Short: "Install required addons for passing Conformance tests", - Long: cmdutil.MacroCommandLongDescription, - Phases: []workflow.Phase{ - { - Name: "all", - Short: "Install all the addons", - InheritFlags: getAddonPhaseFlags("all"), - RunAllSiblings: true, - }, - { - Name: "coredns", - Short: "Install the CoreDNS addon to a Kubernetes cluster", - Long: coreDNSAddonLongDesc, - InheritFlags: getAddonPhaseFlags("coredns"), - Run: runCoreDNSAddon, - }, - { - Name: "kube-proxy", - Short: "Install the kube-proxy addon to a Kubernetes cluster", - Long: kubeProxyAddonLongDesc, - InheritFlags: getAddonPhaseFlags("kube-proxy"), - Run: runKubeProxyAddon, - }, - }, - } -} - -func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) { - data, ok := c.(InitData) - if !ok { - return nil, nil, errors.New("addon phase invoked with an invalid data struct") - } - cfg := data.Cfg() - client, err := data.Client() - if err != nil { - return nil, nil, err - } - return cfg, client, err -} - -// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster -func runCoreDNSAddon(c workflow.RunData) error { - cfg, client, err := getInitData(c) - if err != nil { - return err - } - return dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client) -} - -// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster -func runKubeProxyAddon(c workflow.RunData) error { - cfg, client, err := getInitData(c) - if err != nil { - return err - } - return proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client) -} - -func getAddonPhaseFlags(name string) []string { - flags := []string{ - options.CfgPath, - options.KubeconfigPath, - options.KubernetesVersion, - options.ImageRepository, - } - if name == "all" || name == "kube-proxy" { - flags = append(flags, - options.APIServerAdvertiseAddress, - options.ControlPlaneEndpoint, - options.APIServerBindPort, - options.NetworkingPodSubnet, - ) - } - if name == "all" || name == "coredns" { - flags = append(flags, - options.FeatureGatesString, - options.NetworkingDNSDomain, - options.NetworkingServiceSubnet, - ) - } - return flags -} diff --git a/cmd/kubeadm/app/cmd/phases/init/bootstraptoken.go b/cmd/kubeadm/app/cmd/phases/init/bootstraptoken.go deleted file mode 100644 index b89001063adfc..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/bootstraptoken.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" - nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" -) - -var ( - bootstrapTokenLongDesc = cmdutil.LongDesc(` - Bootstrap tokens are used for establishing bidirectional trust between a node joining - the cluster and a control-plane node. - - This command makes all the configurations required to make bootstrap tokens works - and then creates an initial token. - `) - - bootstrapTokenExamples = cmdutil.Examples(` - # Make all the bootstrap token configurations and create an initial token, functionally - # equivalent to what generated by kubeadm init. - kubeadm init phase bootstrap-token - `) -) - -// NewBootstrapTokenPhase returns the phase to bootstrapToken -func NewBootstrapTokenPhase() workflow.Phase { - return workflow.Phase{ - Name: "bootstrap-token", - Aliases: []string{"bootstraptoken"}, - Short: "Generates bootstrap tokens used to join a node to a cluster", - Example: bootstrapTokenExamples, - Long: bootstrapTokenLongDesc, - InheritFlags: []string{ - options.CfgPath, - options.KubeconfigPath, - options.SkipTokenPrint, - }, - Run: runBootstrapToken, - } -} - -func runBootstrapToken(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("bootstrap-token phase invoked with an invalid data struct") - } - - client, err := data.Client() - if err != nil { - return err - } - - if !data.SkipTokenPrint() { - tokens := data.Tokens() - if len(tokens) == 1 { - fmt.Printf("[bootstrap-token] Using token: %s\n", tokens[0]) - } else if len(tokens) > 1 { - fmt.Printf("[bootstrap-token] Using tokens: %v\n", tokens) - } - } - - fmt.Println("[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles") - // Create the default node bootstrap token - if err := nodebootstraptokenphase.UpdateOrCreateTokens(client, false, data.Cfg().BootstrapTokens); err != nil { - return errors.Wrap(err, "error updating or creating token") - } - // Create RBAC rules that makes the bootstrap tokens able to get nodes - if err := nodebootstraptokenphase.AllowBoostrapTokensToGetNodes(client); err != nil { - return errors.Wrap(err, "error allowing bootstrap tokens to get Nodes") - } - // Create RBAC rules that makes the bootstrap tokens able to post CSRs - if err := nodebootstraptokenphase.AllowBootstrapTokensToPostCSRs(client); err != nil { - return errors.Wrap(err, "error allowing bootstrap tokens to post CSRs") - } - // Create RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically - if err := nodebootstraptokenphase.AutoApproveNodeBootstrapTokens(client); err != nil { - return errors.Wrap(err, "error auto-approving node bootstrap tokens") - } - - // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically - if err := nodebootstraptokenphase.AutoApproveNodeCertificateRotation(client); err != nil { - return err - } - - // Create the cluster-info ConfigMap with the associated RBAC rules - if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, data.KubeConfigPath()); err != nil { - return errors.Wrap(err, "error creating bootstrap ConfigMap") - } - if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil { - return errors.Wrap(err, "error creating clusterinfo RBAC rules") - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/init/certs.go b/cmd/kubeadm/app/cmd/phases/init/certs.go deleted file mode 100644 index 29676b661998b..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/certs.go +++ /dev/null @@ -1,301 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/pflag" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -var ( - saKeyLongDesc = fmt.Sprintf(cmdutil.LongDesc(` - Generate the private key for signing service account tokens along with its public key, and save them into - %s and %s files. - If both files already exist, kubeadm skips the generation step and existing files will be used. - `+cmdutil.AlphaDisclaimer), kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName) - - genericLongDesc = cmdutil.LongDesc(` - Generate the %[1]s, and save them into %[2]s.cert and %[2]s.key files.%[3]s - - If both files already exist, kubeadm skips the generation step and existing files will be used. - ` + cmdutil.AlphaDisclaimer) -) - -var ( - csrOnly bool - csrDir string -) - -// NewCertsPhase returns the phase for the certs -func NewCertsPhase() workflow.Phase { - return workflow.Phase{ - Name: "certs", - Short: "Certificate generation", - Phases: newCertSubPhases(), - Run: runCerts, - Long: cmdutil.MacroCommandLongDescription, - } -} - -func localFlags() *pflag.FlagSet { - set := pflag.NewFlagSet("csr", pflag.ExitOnError) - options.AddCSRFlag(set, &csrOnly) - set.MarkDeprecated(options.CSROnly, "This flag will be removed in a future version. Please use kubeadm alpha certs generate-csr instead.") - options.AddCSRDirFlag(set, &csrDir) - set.MarkDeprecated(options.CSRDir, "This flag will be removed in a future version. Please use kubeadm alpha certs generate-csr instead.") - return set -} - -// newCertSubPhases returns sub phases for certs phase -func newCertSubPhases() []workflow.Phase { - subPhases := []workflow.Phase{} - - // All subphase - allPhase := workflow.Phase{ - Name: "all", - Short: "Generate all certificates", - InheritFlags: getCertPhaseFlags("all"), - RunAllSiblings: true, - } - - subPhases = append(subPhases, allPhase) - - // This loop assumes that GetDefaultCertList() always returns a list of - // certificate that is preceded by the CAs that sign them. - var lastCACert *certsphase.KubeadmCert - for _, cert := range certsphase.GetDefaultCertList() { - var phase workflow.Phase - if cert.CAName == "" { - phase = newCertSubPhase(cert, runCAPhase(cert)) - lastCACert = cert - } else { - phase = newCertSubPhase(cert, runCertPhase(cert, lastCACert)) - phase.LocalFlags = localFlags() - } - subPhases = append(subPhases, phase) - } - - // SA creates the private/public key pair, which doesn't use x509 at all - saPhase := workflow.Phase{ - Name: "sa", - Short: "Generate a private key for signing service account tokens along with its public key", - Long: saKeyLongDesc, - Run: runCertsSa, - InheritFlags: []string{options.CertificatesDir}, - } - - subPhases = append(subPhases, saPhase) - - return subPhases -} - -func newCertSubPhase(certSpec *certsphase.KubeadmCert, run func(c workflow.RunData) error) workflow.Phase { - phase := workflow.Phase{ - Name: certSpec.Name, - Short: fmt.Sprintf("Generate the %s", certSpec.LongName), - Long: fmt.Sprintf( - genericLongDesc, - certSpec.LongName, - certSpec.BaseName, - getSANDescription(certSpec), - ), - Run: run, - InheritFlags: getCertPhaseFlags(certSpec.Name), - } - return phase -} - -func getCertPhaseFlags(name string) []string { - flags := []string{ - options.CertificatesDir, - options.CfgPath, - options.CSROnly, - options.CSRDir, - options.KubernetesVersion, - } - if name == "all" || name == "apiserver" { - flags = append(flags, - options.APIServerAdvertiseAddress, - options.ControlPlaneEndpoint, - options.APIServerCertSANs, - options.NetworkingDNSDomain, - options.NetworkingServiceSubnet, - ) - } - return flags -} - -func getSANDescription(certSpec *certsphase.KubeadmCert) string { - //Defaulted config we will use to get SAN certs - defaultConfig := &kubeadmapiv1beta2.InitConfiguration{ - LocalAPIEndpoint: kubeadmapiv1beta2.APIEndpoint{ - // GetAPIServerAltNames errors without an AdvertiseAddress; this is as good as any. - AdvertiseAddress: "127.0.0.1", - }, - } - defaultInternalConfig := &kubeadmapi.InitConfiguration{} - - kubeadmscheme.Scheme.Default(defaultConfig) - if err := kubeadmscheme.Scheme.Convert(defaultConfig, defaultInternalConfig, nil); err != nil { - return "" - } - - certConfig, err := certSpec.GetConfig(defaultInternalConfig) - if err != nil { - return "" - } - - if len(certConfig.AltNames.DNSNames) == 0 && len(certConfig.AltNames.IPs) == 0 { - return "" - } - // This mutates the certConfig, but we're throwing it after we construct the command anyway - sans := []string{} - - for _, dnsName := range certConfig.AltNames.DNSNames { - if dnsName != "" { - sans = append(sans, dnsName) - } - } - - for _, ip := range certConfig.AltNames.IPs { - sans = append(sans, ip.String()) - } - return fmt.Sprintf("\n\nDefault SANs are %s", strings.Join(sans, ", ")) -} - -func runCertsSa(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("certs phase invoked with an invalid data struct") - } - - // if external CA mode, skip service account key generation - if data.ExternalCA() { - fmt.Printf("[certs] Using existing sa keys\n") - return nil - } - - // create the new service account key (or use existing) - return certsphase.CreateServiceAccountKeyAndPublicKeyFiles(data.CertificateWriteDir(), data.Cfg().ClusterConfiguration.PublicKeyAlgorithm()) -} - -func runCerts(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("certs phase invoked with an invalid data struct") - } - - fmt.Printf("[certs] Using certificateDir folder %q\n", data.CertificateWriteDir()) - return nil -} - -func runCAPhase(ca *certsphase.KubeadmCert) func(c workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("certs phase invoked with an invalid data struct") - } - - // if using external etcd, skips etcd certificate authority generation - if data.Cfg().Etcd.External != nil && ca.Name == "etcd-ca" { - fmt.Printf("[certs] External etcd mode: Skipping %s certificate authority generation\n", ca.BaseName) - return nil - } - - if cert, err := pkiutil.TryLoadCertFromDisk(data.CertificateDir(), ca.BaseName); err == nil { - certsphase.CheckCertificatePeriodValidity(ca.BaseName, cert) - - if _, err := pkiutil.TryLoadKeyFromDisk(data.CertificateDir(), ca.BaseName); err == nil { - fmt.Printf("[certs] Using existing %s certificate authority\n", ca.BaseName) - return nil - } - fmt.Printf("[certs] Using existing %s keyless certificate authority\n", ca.BaseName) - return nil - } - - // if dryrunning, write certificates authority to a temporary folder (and defer restore to the path originally specified by the user) - cfg := data.Cfg() - cfg.CertificatesDir = data.CertificateWriteDir() - defer func() { cfg.CertificatesDir = data.CertificateDir() }() - - // create the new certificate authority (or use existing) - return certsphase.CreateCACertAndKeyFiles(ca, cfg) - } -} - -func runCertPhase(cert *certsphase.KubeadmCert, caCert *certsphase.KubeadmCert) func(c workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("certs phase invoked with an invalid data struct") - } - - // if using external etcd, skips etcd certificates generation - if data.Cfg().Etcd.External != nil && cert.CAName == "etcd-ca" { - fmt.Printf("[certs] External etcd mode: Skipping %s certificate generation\n", cert.BaseName) - return nil - } - - if certData, _, err := pkiutil.TryLoadCertAndKeyFromDisk(data.CertificateDir(), cert.BaseName); err == nil { - certsphase.CheckCertificatePeriodValidity(cert.BaseName, certData) - - caCertData, err := pkiutil.TryLoadCertFromDisk(data.CertificateDir(), caCert.BaseName) - if err != nil { - return errors.Wrapf(err, "couldn't load CA certificate %s", caCert.Name) - } - - certsphase.CheckCertificatePeriodValidity(caCert.BaseName, caCertData) - - if err := certData.CheckSignatureFrom(caCertData); err != nil { - return errors.Wrapf(err, "[certs] certificate %s not signed by CA certificate %s", cert.BaseName, caCert.BaseName) - } - - fmt.Printf("[certs] Using existing %s certificate and key on disk\n", cert.BaseName) - return nil - } - - if csrOnly { - fmt.Printf("[certs] Generating CSR for %s instead of certificate\n", cert.BaseName) - if csrDir == "" { - csrDir = data.CertificateWriteDir() - } - - return certsphase.CreateCSR(cert, data.Cfg(), csrDir) - } - - // if dryrunning, write certificates to a temporary folder (and defer restore to the path originally specified by the user) - cfg := data.Cfg() - cfg.CertificatesDir = data.CertificateWriteDir() - defer func() { cfg.CertificatesDir = data.CertificateDir() }() - - // create the new certificate (or use existing) - return certsphase.CreateCertAndKeyFilesWithCA(cert, caCert, cfg) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/init/certs_test.go b/cmd/kubeadm/app/cmd/phases/init/certs_test.go deleted file mode 100644 index 8cae580a2e124..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/certs_test.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "os" - "testing" - - "github.com/spf13/cobra" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -type testCertsData struct { - testInitData - cfg *kubeadmapi.InitConfiguration -} - -func (t *testCertsData) Cfg() *kubeadmapi.InitConfiguration { return t.cfg } -func (t *testCertsData) ExternalCA() bool { return false } -func (t *testCertsData) CertificateDir() string { return t.cfg.CertificatesDir } -func (t *testCertsData) CertificateWriteDir() string { return t.cfg.CertificatesDir } - -func TestCertsWithCSRs(t *testing.T) { - csrDir := testutil.SetupTempDir(t) - defer os.RemoveAll(csrDir) - certDir := testutil.SetupTempDir(t) - defer os.RemoveAll(certDir) - cert := certs.KubeadmCertAPIServer() - - certsData := &testCertsData{ - cfg: testutil.GetDefaultInternalConfig(t), - } - certsData.cfg.CertificatesDir = certDir - - // global vars - csrOnly = true - csrDir = certDir - defer func() { - csrOnly = false - }() - - phase := NewCertsPhase() - // find the api cert phase - var apiServerPhase *workflow.Phase - for _, phase := range phase.Phases { - if phase.Name == cert.Name { - apiServerPhase = &phase - break - } - } - - if apiServerPhase == nil { - t.Fatalf("couldn't find apiserver phase") - } - - err := apiServerPhase.Run(certsData) - if err != nil { - t.Fatalf("couldn't run API server phase: %v", err) - } - - if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(csrDir, cert.BaseName); err != nil { - t.Fatalf("couldn't load certificate %q: %v", cert.BaseName, err) - } -} - -func TestCreateSparseCerts(t *testing.T) { - for _, test := range certstestutil.GetSparseCertTestCases(t) { - t.Run(test.Name, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - certstestutil.WritePKIFiles(t, tmpdir, test.Files) - - r := workflow.NewRunner() - r.AppendPhase(NewCertsPhase()) - r.SetDataInitializer(func(*cobra.Command, []string) (workflow.RunData, error) { - certsData := &testCertsData{ - cfg: testutil.GetDefaultInternalConfig(t), - } - certsData.cfg.CertificatesDir = tmpdir - return certsData, nil - }) - - if err := r.Run([]string{}); (err != nil) != test.ExpectError { - t.Fatalf("expected error to be %t, got %t (%v)", test.ExpectError, (err != nil), err) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/init/controlplane.go b/cmd/kubeadm/app/cmd/phases/init/controlplane.go deleted file mode 100644 index 49dcf24e92fcf..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/controlplane.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" -) - -var ( - controlPlaneExample = cmdutil.Examples(` - # Generates all static Pod manifest files for control plane components, - # functionally equivalent to what is generated by kubeadm init. - kubeadm init phase control-plane all - - # Generates all static Pod manifest files using options read from a configuration file. - kubeadm init phase control-plane all --config config.yaml - `) - - controlPlanePhaseProperties = map[string]struct { - name string - short string - }{ - kubeadmconstants.KubeAPIServer: { - name: "apiserver", - short: getPhaseDescription(kubeadmconstants.KubeAPIServer), - }, - kubeadmconstants.KubeControllerManager: { - name: "controller-manager", - short: getPhaseDescription(kubeadmconstants.KubeControllerManager), - }, - kubeadmconstants.KubeScheduler: { - name: "scheduler", - short: getPhaseDescription(kubeadmconstants.KubeScheduler), - }, - } -) - -func getPhaseDescription(component string) string { - return fmt.Sprintf("Generates the %s static Pod manifest", component) -} - -// NewControlPlanePhase creates a kubeadm workflow phase that implements bootstrapping the control plane. -func NewControlPlanePhase() workflow.Phase { - phase := workflow.Phase{ - Name: "control-plane", - Short: "Generate all static Pod manifest files necessary to establish the control plane", - Long: cmdutil.MacroCommandLongDescription, - Phases: []workflow.Phase{ - { - Name: "all", - Short: "Generate all static Pod manifest files", - InheritFlags: getControlPlanePhaseFlags("all"), - Example: controlPlaneExample, - RunAllSiblings: true, - }, - newControlPlaneSubphase(kubeadmconstants.KubeAPIServer), - newControlPlaneSubphase(kubeadmconstants.KubeControllerManager), - newControlPlaneSubphase(kubeadmconstants.KubeScheduler), - }, - Run: runControlPlanePhase, - } - return phase -} - -func newControlPlaneSubphase(component string) workflow.Phase { - phase := workflow.Phase{ - Name: controlPlanePhaseProperties[component].name, - Short: controlPlanePhaseProperties[component].short, - Run: runControlPlaneSubphase(component), - InheritFlags: getControlPlanePhaseFlags(component), - } - return phase -} - -func getControlPlanePhaseFlags(name string) []string { - flags := []string{ - options.CfgPath, - options.CertificatesDir, - options.KubernetesVersion, - options.ImageRepository, - options.Patches, - } - if name == "all" || name == kubeadmconstants.KubeAPIServer { - flags = append(flags, - options.APIServerAdvertiseAddress, - options.ControlPlaneEndpoint, - options.APIServerBindPort, - options.APIServerExtraArgs, - options.FeatureGatesString, - options.NetworkingServiceSubnet, - ) - } - if name == "all" || name == kubeadmconstants.KubeControllerManager { - flags = append(flags, - options.ControllerManagerExtraArgs, - options.NetworkingPodSubnet, - ) - } - if name == "all" || name == kubeadmconstants.KubeScheduler { - flags = append(flags, - options.SchedulerExtraArgs, - ) - } - return flags -} - -func runControlPlanePhase(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("control-plane phase invoked with an invalid data struct") - } - - fmt.Printf("[control-plane] Using manifest folder %q\n", data.ManifestDir()) - return nil -} - -func runControlPlaneSubphase(component string) func(c workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("control-plane phase invoked with an invalid data struct") - } - cfg := data.Cfg() - - fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) - return controlplane.CreateStaticPodFiles(data.ManifestDir(), data.PatchesDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/init/data.go b/cmd/kubeadm/app/cmd/phases/init/data.go deleted file mode 100644 index 25b7454b5d2d1..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/data.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io" - - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -// InitData is the interface to use for init phases. -// The "initData" type from "cmd/init.go" must satisfy this interface. -type InitData interface { - UploadCerts() bool - CertificateKey() string - SetCertificateKey(key string) - SkipCertificateKeyPrint() bool - Cfg() *kubeadmapi.InitConfiguration - DryRun() bool - SkipTokenPrint() bool - IgnorePreflightErrors() sets.String - CertificateWriteDir() string - CertificateDir() string - KubeConfigDir() string - KubeConfigPath() string - ManifestDir() string - KubeletDir() string - ExternalCA() bool - OutputWriter() io.Writer - Client() (clientset.Interface, error) - Tokens() []string - PatchesDir() string -} diff --git a/cmd/kubeadm/app/cmd/phases/init/data_test.go b/cmd/kubeadm/app/cmd/phases/init/data_test.go deleted file mode 100644 index e0770b6221ce5..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/data_test.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io" - - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -// a package local type for testing purposes. -type testInitData struct{} - -// testInitData must satisfy InitData. -var _ InitData = &testInitData{} - -func (t *testInitData) UploadCerts() bool { return false } -func (t *testInitData) CertificateKey() string { return "" } -func (t *testInitData) SetCertificateKey(key string) {} -func (t *testInitData) SkipCertificateKeyPrint() bool { return false } -func (t *testInitData) Cfg() *kubeadmapi.InitConfiguration { return nil } -func (t *testInitData) DryRun() bool { return false } -func (t *testInitData) SkipTokenPrint() bool { return false } -func (t *testInitData) IgnorePreflightErrors() sets.String { return nil } -func (t *testInitData) CertificateWriteDir() string { return "" } -func (t *testInitData) CertificateDir() string { return "" } -func (t *testInitData) KubeConfigDir() string { return "" } -func (t *testInitData) KubeConfigPath() string { return "" } -func (t *testInitData) ManifestDir() string { return "" } -func (t *testInitData) KubeletDir() string { return "" } -func (t *testInitData) ExternalCA() bool { return false } -func (t *testInitData) OutputWriter() io.Writer { return nil } -func (t *testInitData) Client() (clientset.Interface, error) { return nil, nil } -func (t *testInitData) Tokens() []string { return nil } -func (t *testInitData) PatchesDir() string { return "" } diff --git a/cmd/kubeadm/app/cmd/phases/init/etcd.go b/cmd/kubeadm/app/cmd/phases/init/etcd.go deleted file mode 100644 index 54c5d926bf7b4..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/etcd.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" -) - -var ( - etcdLocalExample = cmdutil.Examples(` - # Generates the static Pod manifest file for etcd, functionally - # equivalent to what is generated by kubeadm init. - kubeadm init phase etcd local - - # Generates the static Pod manifest file for etcd using options - # read from a configuration file. - kubeadm init phase etcd local --config config.yaml - `) -) - -// NewEtcdPhase creates a kubeadm workflow phase that implements handling of etcd. -func NewEtcdPhase() workflow.Phase { - phase := workflow.Phase{ - Name: "etcd", - Short: "Generate static Pod manifest file for local etcd", - Long: cmdutil.MacroCommandLongDescription, - Phases: []workflow.Phase{ - newEtcdLocalSubPhase(), - }, - } - return phase -} - -func newEtcdLocalSubPhase() workflow.Phase { - phase := workflow.Phase{ - Name: "local", - Short: "Generate the static Pod manifest file for a local, single-node local etcd instance", - Example: etcdLocalExample, - Run: runEtcdPhaseLocal(), - InheritFlags: getEtcdPhaseFlags(), - } - return phase -} - -func getEtcdPhaseFlags() []string { - flags := []string{ - options.CertificatesDir, - options.CfgPath, - options.ImageRepository, - options.Patches, - } - return flags -} - -func runEtcdPhaseLocal() func(c workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("etcd phase invoked with an invalid data struct") - } - cfg := data.Cfg() - - // Add etcd static pod spec only if external etcd is not configured - if cfg.Etcd.External == nil { - // creates target folder if doesn't exist already - if !data.DryRun() { - // Create the etcd data directory - if err := etcdutil.CreateDataDirectory(cfg.Etcd.Local.DataDir); err != nil { - return err - } - } else { - fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir) - } - fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir()) - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { - return errors.Wrap(err, "error creating local etcd static pod manifest file") - } - } else { - klog.V(1).Infoln("[etcd] External etcd mode. Skipping the creation of a manifest for local etcd") - } - return nil - } -} diff --git a/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go deleted file mode 100644 index f2f6b7d3cb48c..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/kubeconfig.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" -) - -var ( - kubeconfigFilePhaseProperties = map[string]struct { - name string - short string - long string - }{ - kubeadmconstants.AdminKubeConfigFileName: { - name: "admin", - short: "Generate a kubeconfig file for the admin to use and for kubeadm itself", - long: "Generate the kubeconfig file for the admin and for kubeadm itself, and save it to %s file.", - }, - kubeadmconstants.KubeletKubeConfigFileName: { - name: "kubelet", - short: "Generate a kubeconfig file for the kubelet to use *only* for cluster bootstrapping purposes", - long: cmdutil.LongDesc(` - Generate the kubeconfig file for the kubelet to use and save it to %s file. - - Please note that this should *only* be used for cluster bootstrapping purposes. After your control plane is up, - you should request all kubelet credentials from the CSR API.`), - }, - kubeadmconstants.ControllerManagerKubeConfigFileName: { - name: "controller-manager", - short: "Generate a kubeconfig file for the controller manager to use", - long: "Generate the kubeconfig file for the controller manager to use and save it to %s file", - }, - kubeadmconstants.SchedulerKubeConfigFileName: { - name: "scheduler", - short: "Generate a kubeconfig file for the scheduler to use", - long: "Generate the kubeconfig file for the scheduler to use and save it to %s file.", - }, - } -) - -// NewKubeConfigPhase creates a kubeadm workflow phase that creates all kubeconfig files necessary to establish the control plane and the admin kubeconfig file. -func NewKubeConfigPhase() workflow.Phase { - return workflow.Phase{ - Name: "kubeconfig", - Short: "Generate all kubeconfig files necessary to establish the control plane and the admin kubeconfig file", - Long: cmdutil.MacroCommandLongDescription, - Phases: []workflow.Phase{ - { - Name: "all", - Short: "Generate all kubeconfig files", - InheritFlags: getKubeConfigPhaseFlags("all"), - RunAllSiblings: true, - }, - NewKubeConfigFilePhase(kubeadmconstants.AdminKubeConfigFileName), - NewKubeConfigFilePhase(kubeadmconstants.KubeletKubeConfigFileName), - NewKubeConfigFilePhase(kubeadmconstants.ControllerManagerKubeConfigFileName), - NewKubeConfigFilePhase(kubeadmconstants.SchedulerKubeConfigFileName), - }, - Run: runKubeConfig, - } -} - -// NewKubeConfigFilePhase creates a kubeadm workflow phase that creates a kubeconfig file. -func NewKubeConfigFilePhase(kubeConfigFileName string) workflow.Phase { - return workflow.Phase{ - Name: kubeconfigFilePhaseProperties[kubeConfigFileName].name, - Short: kubeconfigFilePhaseProperties[kubeConfigFileName].short, - Long: fmt.Sprintf(kubeconfigFilePhaseProperties[kubeConfigFileName].long, kubeConfigFileName), - Run: runKubeConfigFile(kubeConfigFileName), - InheritFlags: getKubeConfigPhaseFlags(kubeConfigFileName), - } -} - -func getKubeConfigPhaseFlags(name string) []string { - flags := []string{ - options.APIServerAdvertiseAddress, - options.ControlPlaneEndpoint, - options.APIServerBindPort, - options.CertificatesDir, - options.CfgPath, - options.KubeconfigDir, - options.KubernetesVersion, - } - if name == "all" || name == kubeadmconstants.KubeletKubeConfigFileName { - flags = append(flags, - options.NodeName, - ) - } - return flags -} - -func runKubeConfig(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("kubeconfig phase invoked with an invalid data struct") - } - - fmt.Printf("[kubeconfig] Using kubeconfig folder %q\n", data.KubeConfigDir()) - return nil -} - -// runKubeConfigFile executes kubeconfig creation logic. -func runKubeConfigFile(kubeConfigFileName string) func(workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("kubeconfig phase invoked with an invalid data struct") - } - - // if external CA mode, skip certificate authority generation - if data.ExternalCA() { - fmt.Printf("[kubeconfig] External CA mode: Using user provided %s\n", kubeConfigFileName) - return nil - } - - // if dryrunning, reads certificates from a temporary folder (and defer restore to the path originally specified by the user) - cfg := data.Cfg() - cfg.CertificatesDir = data.CertificateWriteDir() - defer func() { cfg.CertificatesDir = data.CertificateDir() }() - - // creates the KubeConfig file (or use existing) - return kubeconfigphase.CreateKubeConfigFile(kubeConfigFileName, data.KubeConfigDir(), data.Cfg()) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/init/kubelet.go b/cmd/kubeadm/app/cmd/phases/init/kubelet.go deleted file mode 100644 index 0db4fa16fe53d..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/kubelet.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" -) - -var ( - kubeletStartPhaseExample = cmdutil.Examples(` - # Writes a dynamic environment file with kubelet flags from a InitConfiguration file. - kubeadm init phase kubelet-start --config config.yaml - `) -) - -// NewKubeletStartPhase creates a kubeadm workflow phase that start kubelet on a node. -func NewKubeletStartPhase() workflow.Phase { - return workflow.Phase{ - Name: "kubelet-start", - Short: "Write kubelet settings and (re)start the kubelet", - Long: "Write a file with KubeletConfiguration and an environment file with node specific kubelet settings, and then (re)start kubelet.", - Example: kubeletStartPhaseExample, - Run: runKubeletStart, - InheritFlags: []string{ - options.CfgPath, - options.NodeCRISocket, - options.NodeName, - }, - } -} - -// runKubeletStart executes kubelet start logic. -func runKubeletStart(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("kubelet-start phase invoked with an invalid data struct") - } - - // First off, configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet - // Try to stop the kubelet service so no race conditions occur when configuring it - if !data.DryRun() { - klog.V(1).Infoln("Stopping the kubelet") - kubeletphase.TryStopKubelet() - } - - // Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the control-plane, - // as we handle that ourselves in the mark-control-plane phase - // TODO: Maybe we want to do that some time in the future, in order to remove some logic from the mark-control-plane phase? - if err := kubeletphase.WriteKubeletDynamicEnvFile(&data.Cfg().ClusterConfiguration, &data.Cfg().NodeRegistration, false, data.KubeletDir()); err != nil { - return errors.Wrap(err, "error writing a dynamic environment file for the kubelet") - } - - // Write the kubelet configuration file to disk. - if err := kubeletphase.WriteConfigToDisk(&data.Cfg().ClusterConfiguration, data.KubeletDir()); err != nil { - return errors.Wrap(err, "error writing kubelet configuration to disk") - } - - // Try to start the kubelet service in case it's inactive - if !data.DryRun() { - fmt.Println("[kubelet-start] Starting the kubelet") - kubeletphase.TryStartKubelet() - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go b/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go deleted file mode 100644 index 339ba7dc2f1c7..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/kubeletfinalize.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" - clientcmd "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" -) - -var ( - kubeletFinalizePhaseExample = cmdutil.Examples(` - # Updates settings relevant to the kubelet after TLS bootstrap" - kubeadm init phase kubelet-finalize all --config - `) -) - -// NewKubeletFinalizePhase creates a kubeadm workflow phase that updates settings -// relevant to the kubelet after TLS bootstrap. -func NewKubeletFinalizePhase() workflow.Phase { - return workflow.Phase{ - Name: "kubelet-finalize", - Short: "Updates settings relevant to the kubelet after TLS bootstrap", - Example: kubeletFinalizePhaseExample, - Phases: []workflow.Phase{ - { - Name: "all", - Short: "Run all kubelet-finalize phases", - InheritFlags: []string{options.CfgPath, options.CertificatesDir}, - Example: kubeletFinalizePhaseExample, - RunAllSiblings: true, - }, - { - Name: "experimental-cert-rotation", - Short: "Enable kubelet client certificate rotation", - InheritFlags: []string{options.CfgPath, options.CertificatesDir}, - Run: runKubeletFinalizeCertRotation, - }, - }, - } -} - -// runKubeletFinalizeCertRotation detects if the kubelet certificate rotation is enabled -// and updates the kubelet.conf file to point to a rotatable certificate and key for the -// Node user. -func runKubeletFinalizeCertRotation(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("kubelet-finalize phase invoked with an invalid data struct") - } - - // Check if the user has added the kubelet --cert-dir flag. - // If yes, use that path, else use the kubeadm provided value. - cfg := data.Cfg() - pkiPath := filepath.Join(data.KubeletDir(), "pki") - val, ok := cfg.NodeRegistration.KubeletExtraArgs["cert-dir"] - if ok { - pkiPath = val - } - - // Check for the existence of the kubelet-client-current.pem file in the kubelet certificate directory. - rotate := false - pemPath := filepath.Join(pkiPath, "kubelet-client-current.pem") - if _, err := os.Stat(pemPath); err == nil { - klog.V(1).Infof("[kubelet-finalize] Assuming that kubelet client certificate rotation is enabled: found %q", pemPath) - rotate = true - } else { - klog.V(1).Infof("[kubelet-finalize] Assuming that kubelet client certificate rotation is disabled: %v", err) - } - - // Exit early if rotation is disabled. - if !rotate { - return nil - } - - kubeconfigPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName) - fmt.Printf("[kubelet-finalize] Updating %q to point to a rotatable kubelet client certificate and key\n", kubeconfigPath) - - // Exit early if dry-running is enabled. - if data.DryRun() { - return nil - } - - // Load the kubeconfig from disk. - kubeconfig, err := clientcmd.LoadFromFile(kubeconfigPath) - if err != nil { - return errors.Wrapf(err, "could not load %q", kubeconfigPath) - } - - // Perform basic validation. The errors here can only happen if the kubelet.conf was corrupted. - userName := fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name) - info, ok := kubeconfig.AuthInfos[userName] - if !ok { - return errors.Errorf("the file %q does not contain authentication for user %q", kubeconfigPath, cfg.NodeRegistration.Name) - } - - // Update the client certificate and key of the node authorizer to point to the PEM symbolic link. - info.ClientKeyData = []byte{} - info.ClientCertificateData = []byte{} - info.ClientKey = pemPath - info.ClientCertificate = pemPath - - // Writes the kubeconfig back to disk. - if err = clientcmd.WriteToFile(*kubeconfig, kubeconfigPath); err != nil { - return errors.Wrapf(err, "failed to serialize %q", kubeconfigPath) - } - - // Restart the kubelet. - klog.V(1).Info("[kubelet-finalize] Restarting the kubelet to enable client certificate rotation") - kubeletphase.TryRestartKubelet() - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/init/markcontrolplane.go b/cmd/kubeadm/app/cmd/phases/init/markcontrolplane.go deleted file mode 100644 index ba05cb7f93824..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/markcontrolplane.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" -) - -var ( - markControlPlaneExample = cmdutil.Examples(` - # Applies control-plane label and taint to the current node, functionally equivalent to what executed by kubeadm init. - kubeadm init phase mark-control-plane --config config.yml - - # Applies control-plane label and taint to a specific node - kubeadm init phase mark-control-plane --node-name myNode - `) -) - -// NewMarkControlPlanePhase creates a kubeadm workflow phase that implements mark-controlplane checks. -func NewMarkControlPlanePhase() workflow.Phase { - return workflow.Phase{ - Name: "mark-control-plane", - Short: "Mark a node as a control-plane", - Example: markControlPlaneExample, - InheritFlags: []string{ - options.NodeName, - options.CfgPath, - }, - Run: runMarkControlPlane, - } -} - -// runMarkControlPlane executes mark-control-plane checks logic. -func runMarkControlPlane(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("mark-control-plane phase invoked with an invalid data struct") - } - - client, err := data.Client() - if err != nil { - return err - } - - nodeRegistration := data.Cfg().NodeRegistration - return markcontrolplanephase.MarkControlPlane(client, nodeRegistration.Name, nodeRegistration.Taints) -} diff --git a/cmd/kubeadm/app/cmd/phases/init/preflight.go b/cmd/kubeadm/app/cmd/phases/init/preflight.go deleted file mode 100644 index 0aab506708f5b..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/preflight.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - utilsexec "k8s.io/utils/exec" -) - -var ( - preflightExample = cmdutil.Examples(` - # Run pre-flight checks for kubeadm init using a config file. - kubeadm init phase preflight --config kubeadm-config.yml - `) -) - -// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new control-plane node. -func NewPreflightPhase() workflow.Phase { - return workflow.Phase{ - Name: "preflight", - Short: "Run pre-flight checks", - Long: "Run pre-flight checks for kubeadm init.", - Example: preflightExample, - Run: runPreflight, - InheritFlags: []string{ - options.CfgPath, - options.IgnorePreflightErrors, - }, - } -} - -// runPreflight executes preflight checks logic. -func runPreflight(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("preflight phase invoked with an invalid data struct") - } - - fmt.Println("[preflight] Running pre-flight checks") - if err := preflight.RunInitNodeChecks(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors(), false, false); err != nil { - return err - } - - if !data.DryRun() { - fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster") - fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection") - fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'") - if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil { - return err - } - } else { - fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')") - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go b/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go deleted file mode 100644 index 857fb5d00b46f..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/uploadcerts.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts" -) - -// NewUploadCertsPhase returns the uploadCerts phase -func NewUploadCertsPhase() workflow.Phase { - return workflow.Phase{ - Name: "upload-certs", - Short: fmt.Sprintf("Upload certificates to %s", kubeadmconstants.KubeadmCertsSecret), - Long: cmdutil.MacroCommandLongDescription, - Run: runUploadCerts, - InheritFlags: []string{ - options.CfgPath, - options.KubeconfigPath, - options.UploadCerts, - options.CertificateKey, - options.SkipCertificateKeyPrint, - }, - } -} - -func runUploadCerts(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("upload-certs phase invoked with an invalid data struct") - } - - if !data.UploadCerts() { - fmt.Printf("[upload-certs] Skipping phase. Please see --%s\n", options.UploadCerts) - return nil - } - client, err := data.Client() - if err != nil { - return err - } - - if len(data.CertificateKey()) == 0 { - certificateKey, err := copycerts.CreateCertificateKey() - if err != nil { - return err - } - data.SetCertificateKey(certificateKey) - } - - if err := copycerts.UploadCerts(client, data.Cfg(), data.CertificateKey()); err != nil { - return errors.Wrap(err, "error uploading certs") - } - if !data.SkipCertificateKeyPrint() { - fmt.Printf("[upload-certs] Using certificate key:\n%s\n", data.CertificateKey()) - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/init/uploadconfig.go b/cmd/kubeadm/app/cmd/phases/init/uploadconfig.go deleted file mode 100644 index 52c5a935223d1..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/uploadconfig.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" -) - -var ( - uploadKubeadmConfigLongDesc = fmt.Sprintf(cmdutil.LongDesc(` - Upload the kubeadm ClusterConfiguration to a ConfigMap called %s in the %s namespace. - This enables correct configuration of system components and a seamless user experience when upgrading. - - Alternatively, you can use kubeadm config. - `), kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - - uploadKubeadmConfigExample = cmdutil.Examples(` - # upload the configuration of your cluster - kubeadm init phase upload-config --config=myConfig.yaml - `) - - uploadKubeletConfigLongDesc = cmdutil.LongDesc(` - Upload kubelet configuration extracted from the kubeadm InitConfiguration object to a ConfigMap - of the form kubelet-config-1.X in the cluster, where X is the minor version of the current (API Server) Kubernetes version. - `) - - uploadKubeletConfigExample = cmdutil.Examples(` - # Upload the kubelet configuration from the kubeadm Config file to a ConfigMap in the cluster. - kubeadm init phase upload-config kubelet --config kubeadm.yaml - `) -) - -// NewUploadConfigPhase returns the phase to uploadConfig -func NewUploadConfigPhase() workflow.Phase { - return workflow.Phase{ - Name: "upload-config", - Aliases: []string{"uploadconfig"}, - Short: "Upload the kubeadm and kubelet configuration to a ConfigMap", - Long: cmdutil.MacroCommandLongDescription, - Phases: []workflow.Phase{ - { - Name: "all", - Short: "Upload all configuration to a config map", - RunAllSiblings: true, - InheritFlags: getUploadConfigPhaseFlags(), - }, - { - Name: "kubeadm", - Short: "Upload the kubeadm ClusterConfiguration to a ConfigMap", - Long: uploadKubeadmConfigLongDesc, - Example: uploadKubeadmConfigExample, - Run: runUploadKubeadmConfig, - InheritFlags: getUploadConfigPhaseFlags(), - }, - { - Name: "kubelet", - Short: "Upload the kubelet component config to a ConfigMap", - Long: uploadKubeletConfigLongDesc, - Example: uploadKubeletConfigExample, - Run: runUploadKubeletConfig, - InheritFlags: getUploadConfigPhaseFlags(), - }, - }, - } -} - -func getUploadConfigPhaseFlags() []string { - return []string{ - options.CfgPath, - options.KubeconfigPath, - } -} - -// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap -func runUploadKubeadmConfig(c workflow.RunData) error { - cfg, client, err := getUploadConfigData(c) - if err != nil { - return err - } - - klog.V(1).Infoln("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap") - if err := uploadconfig.UploadConfiguration(cfg, client); err != nil { - return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration") - } - return nil -} - -// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap -func runUploadKubeletConfig(c workflow.RunData) error { - cfg, client, err := getUploadConfigData(c) - if err != nil { - return err - } - - klog.V(1).Infoln("[upload-config] Uploading the kubelet component config to a ConfigMap") - if err = kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil { - return errors.Wrap(err, "error creating kubelet configuration ConfigMap") - } - - klog.V(1).Infoln("[upload-config] Preserving the CRISocket information for the control-plane node") - if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { - return errors.Wrap(err, "Error writing Crisocket information for the control-plane node") - } - return nil -} - -func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, error) { - data, ok := c.(InitData) - if !ok { - return nil, nil, errors.New("upload-config phase invoked with an invalid data struct") - } - cfg := data.Cfg() - client, err := data.Client() - if err != nil { - return nil, nil, err - } - return cfg, client, err -} diff --git a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go deleted file mode 100644 index 582b88a92fe04..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - "io" - "path/filepath" - "text/template" - "time" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" -) - -var ( - kubeletFailTempl = template.Must(template.New("init").Parse(dedent.Dedent(` - Unfortunately, an error has occurred: - {{ .Error }} - - This error is likely caused by: - - The kubelet is not running - - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled) - - If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands: - - 'systemctl status kubelet' - - 'journalctl -xeu kubelet' - - Additionally, a control plane component may have crashed or exited when started by the container runtime. - To troubleshoot, list all containers using your preferred container runtimes CLI. -{{ if .IsDocker }} - Here is one example how you may list all Kubernetes containers running in docker: - - 'docker ps -a | grep kube | grep -v pause' - Once you have found the failing container, you can inspect its logs with: - - 'docker logs CONTAINERID' -{{ else }} - Here is one example how you may list all Kubernetes containers running in cri-o/containerd using crictl: - - 'crictl --runtime-endpoint {{ .Socket }} ps -a | grep kube | grep -v pause' - Once you have found the failing container, you can inspect its logs with: - - 'crictl --runtime-endpoint {{ .Socket }} logs CONTAINERID' -{{ end }} - `))) -) - -// NewWaitControlPlanePhase is a hidden phase that runs after the control-plane and etcd phases -func NewWaitControlPlanePhase() workflow.Phase { - phase := workflow.Phase{ - Name: "wait-control-plane", - Run: runWaitControlPlanePhase, - Hidden: true, - } - return phase -} - -func runWaitControlPlanePhase(c workflow.RunData) error { - data, ok := c.(InitData) - if !ok { - return errors.New("wait-control-plane phase invoked with an invalid data struct") - } - - // If we're dry-running, print the generated manifests - if err := printFilesIfDryRunning(data); err != nil { - return errors.Wrap(err, "error printing files on dryrun") - } - - // waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled - klog.V(1).Infoln("[wait-control-plane] Waiting for the API server to be healthy") - - client, err := data.Client() - if err != nil { - return errors.Wrap(err, "cannot obtain client") - } - - timeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration - waiter, err := newControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter()) - if err != nil { - return errors.Wrap(err, "error creating waiter") - } - - fmt.Printf("[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory %q. This can take up to %v\n", data.ManifestDir(), timeout) - - if err := waiter.WaitForKubeletAndFunc(waiter.WaitForAPI); err != nil { - context := struct { - Error string - Socket string - IsDocker bool - }{ - Error: fmt.Sprintf("%v", err), - Socket: data.Cfg().NodeRegistration.CRISocket, - IsDocker: data.Cfg().NodeRegistration.CRISocket == kubeadmconstants.DefaultDockerCRISocket, - } - - kubeletFailTempl.Execute(data.OutputWriter(), context) - return errors.New("couldn't initialize a Kubernetes cluster") - } - - return nil -} - -// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup -func printFilesIfDryRunning(data InitData) error { - if !data.DryRun() { - return nil - } - manifestDir := data.ManifestDir() - - fmt.Printf("[dryrun] Wrote certificates, kubeconfig files and control plane manifests to the %q directory\n", manifestDir) - fmt.Println("[dryrun] The certificates or kubeconfig files would not be printed due to their sensitive nature") - fmt.Printf("[dryrun] Please examine the %q directory for details about what would be written\n", manifestDir) - - // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests - files := []dryrunutil.FileToPrint{} - // Print static pod manifests - for _, component := range kubeadmconstants.ControlPlaneComponents { - realPath := kubeadmconstants.GetStaticPodFilepath(component, manifestDir) - outputPath := kubeadmconstants.GetStaticPodFilepath(component, kubeadmconstants.GetStaticPodDirectory()) - files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) - } - // Print kubelet config manifests - kubeletConfigFiles := []string{kubeadmconstants.KubeletConfigurationFileName, kubeadmconstants.KubeletEnvFileName} - for _, filename := range kubeletConfigFiles { - realPath := filepath.Join(manifestDir, filename) - outputPath := filepath.Join(kubeadmconstants.KubeletRunDirectory, filename) - files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) - } - - return dryrunutil.PrintDryRunFiles(files, data.OutputWriter()) -} - -// newControlPlaneWaiter returns a new waiter that is used to wait on the control plane to boot up. -func newControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) { - if dryRun { - return dryrunutil.NewWaiter(), nil - } - - return apiclient.NewKubeWaiter(client, timeout, out), nil -} diff --git a/cmd/kubeadm/app/cmd/phases/join/BUILD b/cmd/kubeadm/app/cmd/phases/join/BUILD deleted file mode 100644 index df53cfb825380..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/BUILD +++ /dev/null @@ -1,74 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "checketcd.go", - "controlplanejoin.go", - "controlplaneprepare.go", - "data.go", - "kubelet.go", - "preflight.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/cmd/util:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/controlplane:go_default_library", - "//cmd/kubeadm/app/phases/copycerts:go_default_library", - "//cmd/kubeadm/app/phases/etcd:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/phases/kubelet:go_default_library", - "//cmd/kubeadm/app/phases/markcontrolplane:go_default_library", - "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/etcd:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = ["data_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/cmd/phases/join/checketcd.go b/cmd/kubeadm/app/cmd/phases/join/checketcd.go deleted file mode 100644 index 98156ec70f8f1..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/checketcd.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" -) - -// NewCheckEtcdPhase is a hidden phase that runs after the control-plane-prepare and -// before the bootstrap-kubelet phase that ensures etcd is healthy -func NewCheckEtcdPhase() workflow.Phase { - return workflow.Phase{ - Name: "check-etcd", - Run: runCheckEtcdPhase, - Hidden: true, - } -} - -func runCheckEtcdPhase(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("check-etcd phase invoked with an invalid data struct") - } - - // Skip if this is not a control plane - if data.Cfg().ControlPlane == nil { - return nil - } - - cfg, err := data.InitCfg() - if err != nil { - return err - } - - if cfg.Etcd.External != nil { - fmt.Println("[check-etcd] Skipping etcd check in external mode") - return nil - } - - fmt.Println("[check-etcd] Checking that the etcd cluster is healthy") - - // Checks that the etcd cluster is healthy - // NB. this check cannot be implemented before because it requires the admin.conf and all the certificates - // for connecting to etcd already in place - client, err := data.ClientSet() - if err != nil { - return err - } - - return etcdphase.CheckLocalEtcdClusterStatus(client, &cfg.ClusterConfiguration) -} diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go b/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go deleted file mode 100644 index c5b75eede63e6..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane" - uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" -) - -var controlPlaneJoinExample = cmdutil.Examples(` - # Joins a machine as a control plane instance - kubeadm join phase control-plane-join all -`) - -func getControlPlaneJoinPhaseFlags(name string) []string { - flags := []string{ - options.CfgPath, - options.ControlPlane, - options.NodeName, - } - if name == "etcd" { - flags = append(flags, options.Patches) - } - if name != "mark-control-plane" { - flags = append(flags, options.APIServerAdvertiseAddress) - } - return flags -} - -// NewControlPlaneJoinPhase creates a kubeadm workflow phase that implements joining a machine as a control plane instance -func NewControlPlaneJoinPhase() workflow.Phase { - return workflow.Phase{ - Name: "control-plane-join", - Short: "Join a machine as a control plane instance", - Example: controlPlaneJoinExample, - Phases: []workflow.Phase{ - { - Name: "all", - Short: "Join a machine as a control plane instance", - InheritFlags: getControlPlaneJoinPhaseFlags("all"), - RunAllSiblings: true, - ArgsValidator: cobra.NoArgs, - }, - newEtcdLocalSubphase(), - newUpdateStatusSubphase(), - newMarkControlPlaneSubphase(), - }, - } -} - -func newEtcdLocalSubphase() workflow.Phase { - return workflow.Phase{ - Name: "etcd", - Short: "Add a new local etcd member", - Run: runEtcdPhase, - InheritFlags: getControlPlaneJoinPhaseFlags("etcd"), - ArgsValidator: cobra.NoArgs, - } -} - -func newUpdateStatusSubphase() workflow.Phase { - return workflow.Phase{ - Name: "update-status", - Short: fmt.Sprintf( - "Register the new control-plane node into the %s maintained in the %s ConfigMap", - kubeadmconstants.ClusterStatusConfigMapKey, - kubeadmconstants.KubeadmConfigConfigMap, - ), - Run: runUpdateStatusPhase, - InheritFlags: getControlPlaneJoinPhaseFlags("update-status"), - ArgsValidator: cobra.NoArgs, - } -} - -func newMarkControlPlaneSubphase() workflow.Phase { - return workflow.Phase{ - Name: "mark-control-plane", - Short: "Mark a node as a control-plane", - Run: runMarkControlPlanePhase, - InheritFlags: getControlPlaneJoinPhaseFlags("mark-control-plane"), - ArgsValidator: cobra.NoArgs, - } -} - -func runEtcdPhase(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("control-plane-join phase invoked with an invalid data struct") - } - - if data.Cfg().ControlPlane == nil { - return nil - } - - // gets access to the cluster using the identity defined in admin.conf - client, err := data.ClientSet() - if err != nil { - return errors.Wrap(err, "couldn't create Kubernetes client") - } - cfg, err := data.InitCfg() - if err != nil { - return err - } - // in case of local etcd - if cfg.Etcd.External != nil { - fmt.Println("[control-plane-join] using external etcd - no local stacked instance added") - return nil - } - - // Create the etcd data directory - if err := etcdutil.CreateDataDirectory(cfg.Etcd.Local.DataDir); err != nil { - return err - } - - // Adds a new etcd instance; in order to do this the new etcd instance should be "announced" to - // the existing etcd members before being created. - // This operation must be executed after kubelet is already started in order to minimize the time - // between the new etcd member is announced and the start of the static pod running the new etcd member, because during - // this time frame etcd gets temporary not available (only when moving from 1 to 2 members in the etcd cluster). - // From https://coreos.com/etcd/docs/latest/v2/runtime-configuration.html - // "If you add a new member to a 1-node cluster, the cluster cannot make progress before the new member starts - // because it needs two members as majority to agree on the consensus. You will only see this behavior between the time - // etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the - // existing one." - if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { - return errors.Wrap(err, "error creating local etcd static pod manifest file") - } - - return nil -} - -func runUpdateStatusPhase(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("control-plane-join phase invoked with an invalid data struct") - } - - if data.Cfg().ControlPlane == nil { - return nil - } - - // gets access to the cluster using the identity defined in admin.conf - client, err := data.ClientSet() - if err != nil { - return errors.Wrap(err, "couldn't create Kubernetes client") - } - cfg, err := data.InitCfg() - if err != nil { - return err - } - - if err := uploadconfigphase.UploadConfiguration(cfg, client); err != nil { - return errors.Wrap(err, "error uploading configuration") - } - - return nil -} - -func runMarkControlPlanePhase(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("control-plane-join phase invoked with an invalid data struct") - } - - if data.Cfg().ControlPlane == nil { - return nil - } - - // gets access to the cluster using the identity defined in admin.conf - client, err := data.ClientSet() - if err != nil { - return errors.Wrap(err, "couldn't create Kubernetes client") - } - cfg, err := data.InitCfg() - if err != nil { - return err - } - - if err := markcontrolplanephase.MarkControlPlane(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.Taints); err != nil { - return errors.Wrap(err, "error applying control-plane label and taints") - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go b/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go deleted file mode 100644 index bc6de6f5778bd..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go +++ /dev/null @@ -1,292 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var controlPlanePrepareExample = cmdutil.Examples(` - # Prepares the machine for serving a control plane - kubeadm join phase control-plane-prepare all -`) - -// NewControlPlanePreparePhase creates a kubeadm workflow phase that implements the preparation of the node to serve a control plane -func NewControlPlanePreparePhase() workflow.Phase { - return workflow.Phase{ - Name: "control-plane-prepare", - Short: "Prepare the machine for serving a control plane", - Example: controlPlanePrepareExample, - Phases: []workflow.Phase{ - { - Name: "all [api-server-endpoint]", - Short: "Prepare the machine for serving a control plane", - InheritFlags: getControlPlanePreparePhaseFlags("all"), - RunAllSiblings: true, - }, - newControlPlanePrepareDownloadCertsSubphase(), - newControlPlanePrepareCertsSubphase(), - newControlPlanePrepareKubeconfigSubphase(), - newControlPlanePrepareControlPlaneSubphase(), - }, - } -} - -func getControlPlanePreparePhaseFlags(name string) []string { - var flags []string - switch name { - case "all": - flags = []string{ - options.APIServerAdvertiseAddress, - options.APIServerBindPort, - options.CfgPath, - options.ControlPlane, - options.NodeName, - options.FileDiscovery, - options.TokenDiscovery, - options.TokenDiscoveryCAHash, - options.TokenDiscoverySkipCAHash, - options.TLSBootstrapToken, - options.TokenStr, - options.CertificateKey, - options.Patches, - } - case "download-certs": - flags = []string{ - options.CfgPath, - options.ControlPlane, - options.FileDiscovery, - options.TokenDiscovery, - options.TokenDiscoveryCAHash, - options.TokenDiscoverySkipCAHash, - options.TLSBootstrapToken, - options.TokenStr, - options.CertificateKey, - } - case "certs": - flags = []string{ - options.APIServerAdvertiseAddress, - options.CfgPath, - options.ControlPlane, - options.NodeName, - options.FileDiscovery, - options.TokenDiscovery, - options.TokenDiscoveryCAHash, - options.TokenDiscoverySkipCAHash, - options.TLSBootstrapToken, - options.TokenStr, - } - case "kubeconfig": - flags = []string{ - options.CfgPath, - options.ControlPlane, - options.FileDiscovery, - options.TokenDiscovery, - options.TokenDiscoveryCAHash, - options.TokenDiscoverySkipCAHash, - options.TLSBootstrapToken, - options.TokenStr, - options.CertificateKey, - } - case "control-plane": - flags = []string{ - options.APIServerAdvertiseAddress, - options.APIServerBindPort, - options.CfgPath, - options.ControlPlane, - options.Patches, - } - default: - flags = []string{} - } - return flags -} - -func newControlPlanePrepareDownloadCertsSubphase() workflow.Phase { - return workflow.Phase{ - Name: "download-certs [api-server-endpoint]", - Short: fmt.Sprintf("[EXPERIMENTAL] Download certificates shared among control-plane nodes from the %s Secret", kubeadmconstants.KubeadmCertsSecret), - Run: runControlPlanePrepareDownloadCertsPhaseLocal, - InheritFlags: getControlPlanePreparePhaseFlags("download-certs"), - } -} - -func newControlPlanePrepareCertsSubphase() workflow.Phase { - return workflow.Phase{ - Name: "certs [api-server-endpoint]", - Short: "Generate the certificates for the new control plane components", - Run: runControlPlanePrepareCertsPhaseLocal, //NB. eventually in future we would like to break down this in sub phases for each cert or add the --csr option - InheritFlags: getControlPlanePreparePhaseFlags("certs"), - } -} - -func newControlPlanePrepareKubeconfigSubphase() workflow.Phase { - return workflow.Phase{ - Name: "kubeconfig [api-server-endpoint]", - Short: "Generate the kubeconfig for the new control plane components", - Run: runControlPlanePrepareKubeconfigPhaseLocal, //NB. eventually in future we would like to break down this in sub phases for each kubeconfig - InheritFlags: getControlPlanePreparePhaseFlags("kubeconfig"), - } -} - -func newControlPlanePrepareControlPlaneSubphase() workflow.Phase { - return workflow.Phase{ - Name: "control-plane", - Short: "Generate the manifests for the new control plane components", - Run: runControlPlanePrepareControlPlaneSubphase, //NB. eventually in future we would like to break down this in sub phases for each component - InheritFlags: getControlPlanePreparePhaseFlags("control-plane"), - ArgsValidator: cobra.NoArgs, - } -} - -func runControlPlanePrepareControlPlaneSubphase(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("control-plane-prepare phase invoked with an invalid data struct") - } - - // Skip if this is not a control plane - if data.Cfg().ControlPlane == nil { - return nil - } - - cfg, err := data.InitCfg() - if err != nil { - return err - } - - fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory()) - for _, component := range kubeadmconstants.ControlPlaneComponents { - fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component) - err := controlplane.CreateStaticPodFiles( - kubeadmconstants.GetStaticPodDirectory(), - data.PatchesDir(), - &cfg.ClusterConfiguration, - &cfg.LocalAPIEndpoint, - component, - ) - if err != nil { - return err - } - } - return nil -} - -func runControlPlanePrepareDownloadCertsPhaseLocal(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("download-certs phase invoked with an invalid data struct") - } - - if data.Cfg().ControlPlane == nil || len(data.CertificateKey()) == 0 { - klog.V(1).Infoln("[download-certs] Skipping certs download") - return nil - } - - cfg, err := data.InitCfg() - if err != nil { - return err - } - - client, err := bootstrapClient(data) - if err != nil { - return err - } - - if err := copycerts.DownloadCerts(client, cfg, data.CertificateKey()); err != nil { - return errors.Wrap(err, "error downloading certs") - } - return nil -} - -func runControlPlanePrepareCertsPhaseLocal(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("control-plane-prepare phase invoked with an invalid data struct") - } - - // Skip if this is not a control plane - if data.Cfg().ControlPlane == nil { - return nil - } - - cfg, err := data.InitCfg() - if err != nil { - return err - } - - fmt.Printf("[certs] Using certificateDir folder %q\n", cfg.CertificatesDir) - - // Generate missing certificates (if any) - return certsphase.CreatePKIAssets(cfg) -} - -func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error { - data, ok := c.(JoinData) - if !ok { - return errors.New("control-plane-prepare phase invoked with an invalid data struct") - } - - // Skip if this is not a control plane - if data.Cfg().ControlPlane == nil { - return nil - } - - cfg, err := data.InitCfg() - if err != nil { - return err - } - - fmt.Println("[kubeconfig] Generating kubeconfig files") - fmt.Printf("[kubeconfig] Using kubeconfig folder %q\n", kubeadmconstants.KubernetesDir) - - // Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself - // NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in - // following steps of the join --control-plane workflow - if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, cfg); err != nil { - return errors.Wrap(err, "error generating kubeconfig files") - } - - return nil -} - -func bootstrapClient(data JoinData) (clientset.Interface, error) { - tlsBootstrapCfg, err := data.TLSBootstrapCfg() - if err != nil { - return nil, errors.Wrap(err, "unable to access the cluster") - } - client, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg) - if err != nil { - return nil, errors.Wrap(err, "unable to access the cluster") - } - return client, nil -} diff --git a/cmd/kubeadm/app/cmd/phases/join/data.go b/cmd/kubeadm/app/cmd/phases/join/data.go deleted file mode 100644 index f772471828636..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/data.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io" - - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -// JoinData is the interface to use for join phases. -// The "joinData" type from "cmd/join.go" must satisfy this interface. -type JoinData interface { - CertificateKey() string - Cfg() *kubeadmapi.JoinConfiguration - TLSBootstrapCfg() (*clientcmdapi.Config, error) - InitCfg() (*kubeadmapi.InitConfiguration, error) - ClientSet() (*clientset.Clientset, error) - IgnorePreflightErrors() sets.String - OutputWriter() io.Writer - PatchesDir() string -} diff --git a/cmd/kubeadm/app/cmd/phases/join/data_test.go b/cmd/kubeadm/app/cmd/phases/join/data_test.go deleted file mode 100644 index cfc956ee7f61b..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/data_test.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io" - - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -// a package local type for testing purposes. -type testJoinData struct{} - -// testJoinData must satisfy JoinData. -var _ JoinData = &testJoinData{} - -func (j *testJoinData) CertificateKey() string { return "" } -func (j *testJoinData) Cfg() *kubeadmapi.JoinConfiguration { return nil } -func (j *testJoinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) { return nil, nil } -func (j *testJoinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { return nil, nil } -func (j *testJoinData) ClientSet() (*clientset.Clientset, error) { return nil, nil } -func (j *testJoinData) IgnorePreflightErrors() sets.String { return nil } -func (j *testJoinData) OutputWriter() io.Writer { return nil } -func (j *testJoinData) PatchesDir() string { return "" } diff --git a/cmd/kubeadm/app/cmd/phases/join/kubelet.go b/cmd/kubeadm/app/cmd/phases/join/kubelet.go deleted file mode 100644 index 4abb2d6ba4e39..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/kubelet.go +++ /dev/null @@ -1,208 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "context" - "fmt" - "os" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -var ( - kubeadmJoinFailMsg = dedent.Dedent(` - Unfortunately, an error has occurred: - %v - - This error is likely caused by: - - The kubelet is not running - - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled) - - If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands: - - 'systemctl status kubelet' - - 'journalctl -xeu kubelet' - `) -) - -// NewKubeletStartPhase creates a kubeadm workflow phase that start kubelet on a node. -func NewKubeletStartPhase() workflow.Phase { - return workflow.Phase{ - Name: "kubelet-start [api-server-endpoint]", - Short: "Write kubelet settings, certificates and (re)start the kubelet", - Long: "Write a file with KubeletConfiguration and an environment file with node specific kubelet settings, and then (re)start kubelet.", - Run: runKubeletStartJoinPhase, - InheritFlags: []string{ - options.CfgPath, - options.NodeCRISocket, - options.NodeName, - options.FileDiscovery, - options.TokenDiscovery, - options.TokenDiscoveryCAHash, - options.TokenDiscoverySkipCAHash, - options.TLSBootstrapToken, - options.TokenStr, - }, - } -} - -func getKubeletStartJoinData(c workflow.RunData) (*kubeadmapi.JoinConfiguration, *kubeadmapi.InitConfiguration, *clientcmdapi.Config, error) { - data, ok := c.(JoinData) - if !ok { - return nil, nil, nil, errors.New("kubelet-start phase invoked with an invalid data struct") - } - cfg := data.Cfg() - initCfg, err := data.InitCfg() - if err != nil { - return nil, nil, nil, err - } - tlsBootstrapCfg, err := data.TLSBootstrapCfg() - if err != nil { - return nil, nil, nil, err - } - return cfg, initCfg, tlsBootstrapCfg, nil -} - -// runKubeletStartJoinPhase executes the kubelet TLS bootstrap process. -// This process is executed by the kubelet and completes with the node joining the cluster -// with a dedicates set of credentials as required by the node authorizer -func runKubeletStartJoinPhase(c workflow.RunData) (returnErr error) { - cfg, initCfg, tlsBootstrapCfg, err := getKubeletStartJoinData(c) - if err != nil { - return err - } - bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath() - - // Deletes the bootstrapKubeConfigFile, so the credential used for TLS bootstrap is removed from disk - defer os.Remove(bootstrapKubeConfigFile) - - // Write the bootstrap kubelet config file or the TLS-Bootstrapped kubelet config file down to disk - klog.V(1).Infof("[kubelet-start] writing bootstrap kubelet config file at %s", bootstrapKubeConfigFile) - if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, tlsBootstrapCfg); err != nil { - return errors.Wrap(err, "couldn't save bootstrap-kubelet.conf to disk") - } - - // Write the ca certificate to disk so kubelet can use it for authentication - cluster := tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster - if _, err := os.Stat(cfg.CACertPath); os.IsNotExist(err) { - klog.V(1).Infof("[kubelet-start] writing CA certificate at %s", cfg.CACertPath) - if err := certutil.WriteCert(cfg.CACertPath, tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil { - return errors.Wrap(err, "couldn't save the CA certificate to disk") - } - } - - bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile) - if err != nil { - return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile) - } - - // Obtain the name of this Node. - nodeName, _, err := kubeletphase.GetNodeNameAndHostname(&cfg.NodeRegistration) - if err != nil { - klog.Warning(err) - } - - // Make sure to exit before TLS bootstrap if a Node with the same name exist in the cluster - // and it has the "Ready" status. - // A new Node with the same name as an existing control-plane Node can cause undefined - // behavior and ultimately control-plane failure. - klog.V(1).Infof("[kubelet-start] Checking for an existing Node in the cluster with name %q and status %q", nodeName, v1.NodeReady) - node, err := bootstrapClient.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return errors.Wrapf(err, "cannot get Node %q", nodeName) - } - for _, cond := range node.Status.Conditions { - if cond.Type == v1.NodeReady && cond.Status == v1.ConditionTrue { - return errors.Errorf("a Node with name %q and status %q already exists in the cluster. "+ - "You must delete the existing Node or change the name of this new joining Node", nodeName, v1.NodeReady) - } - } - - // Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet - // Try to stop the kubelet service so no race conditions occur when configuring it - klog.V(1).Infoln("[kubelet-start] Stopping the kubelet") - kubeletphase.TryStopKubelet() - - // Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start - if err := kubeletphase.WriteConfigToDisk(&initCfg.ClusterConfiguration, kubeadmconstants.KubeletRunDirectory); err != nil { - return err - } - - // Write env file with flags for the kubelet to use. We only want to - // register the joining node with the specified taints if the node - // is not a control-plane. The mark-control-plane phase will register the taints otherwise. - registerTaintsUsingFlags := cfg.ControlPlane == nil - if err := kubeletphase.WriteKubeletDynamicEnvFile(&initCfg.ClusterConfiguration, &initCfg.NodeRegistration, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil { - return err - } - - // Try to start the kubelet service in case it's inactive - fmt.Println("[kubelet-start] Starting the kubelet") - kubeletphase.TryStartKubelet() - - // Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf - // Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process - // times out, display a somewhat user-friendly message. - waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout) - if err := waiter.WaitForKubeletAndFunc(waitForTLSBootstrappedClient); err != nil { - fmt.Printf(kubeadmJoinFailMsg, err) - return err - } - - // When we know the /etc/kubernetes/kubelet.conf file is available, get the client - client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) - if err != nil { - return err - } - - klog.V(1).Infoln("[kubelet-start] preserving the crisocket information for the node") - if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { - return errors.Wrap(err, "error uploading crisocket") - } - - return nil -} - -// waitForTLSBootstrappedClient waits for the /etc/kubernetes/kubelet.conf file to be available -func waitForTLSBootstrappedClient() error { - fmt.Println("[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...") - - // Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned. - return wait.PollImmediate(kubeadmconstants.TLSBootstrapRetryInterval, kubeadmconstants.TLSBootstrapTimeout, func() (bool, error) { - // Check that we can create a client set out of the kubelet kubeconfig. This ensures not - // only that the kubeconfig file exists, but that other files required by it also exist (like - // client certificate and key) - _, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath()) - return (err == nil), nil - }) -} diff --git a/cmd/kubeadm/app/cmd/phases/join/preflight.go b/cmd/kubeadm/app/cmd/phases/join/preflight.go deleted file mode 100644 index 1da9fdb0ae20f..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/join/preflight.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - utilsexec "k8s.io/utils/exec" -) - -var ( - preflightExample = cmdutil.Examples(` - # Run join pre-flight checks using a config file. - kubeadm join phase preflight --config kubeadm-config.yml - `) - - notReadyToJoinControlPlaneTemp = template.Must(template.New("join").Parse(dedent.Dedent(` - One or more conditions for hosting a new control plane instance is not satisfied. - - {{.Error}} - - Please ensure that: - * The cluster has a stable controlPlaneEndpoint address. - * The certificates that must be shared among control plane instances are provided. - - `))) -) - -// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join -func NewPreflightPhase() workflow.Phase { - return workflow.Phase{ - Name: "preflight [api-server-endpoint]", - Short: "Run join pre-flight checks", - Long: "Run pre-flight checks for kubeadm join.", - Example: preflightExample, - Run: runPreflight, - InheritFlags: []string{ - options.CfgPath, - options.IgnorePreflightErrors, - options.TLSBootstrapToken, - options.TokenStr, - options.ControlPlane, - options.APIServerAdvertiseAddress, - options.APIServerBindPort, - options.NodeCRISocket, - options.NodeName, - options.FileDiscovery, - options.TokenDiscovery, - options.TokenDiscoveryCAHash, - options.TokenDiscoverySkipCAHash, - options.CertificateKey, - }, - } -} - -// runPreflight executes preflight checks logic. -func runPreflight(c workflow.RunData) error { - j, ok := c.(JoinData) - if !ok { - return errors.New("preflight phase invoked with an invalid data struct") - } - fmt.Println("[preflight] Running pre-flight checks") - - // Start with general checks - klog.V(1).Infoln("[preflight] Running general checks") - if err := preflight.RunJoinNodeChecks(utilsexec.New(), j.Cfg(), j.IgnorePreflightErrors()); err != nil { - return err - } - - initCfg, err := j.InitCfg() - if err != nil { - return err - } - - // Continue with more specific checks based on the init configuration - klog.V(1).Infoln("[preflight] Running configuration dependant checks") - if j.Cfg().ControlPlane != nil { - // Checks if the cluster configuration supports - // joining a new control plane instance and if all the necessary certificates are provided - hasCertificateKey := len(j.CertificateKey()) > 0 - if err := checkIfReadyForAdditionalControlPlane(&initCfg.ClusterConfiguration, hasCertificateKey); err != nil { - // outputs the not ready for hosting a new control plane instance message - ctx := map[string]string{ - "Error": err.Error(), - } - - var msg bytes.Buffer - notReadyToJoinControlPlaneTemp.Execute(&msg, ctx) - return errors.New(msg.String()) - } - - // run kubeadm init preflight checks for checking all the prerequisites - fmt.Println("[preflight] Running pre-flight checks before initializing the new control plane instance") - - if err := preflight.RunInitNodeChecks(utilsexec.New(), initCfg, j.IgnorePreflightErrors(), true, hasCertificateKey); err != nil { - return err - } - - fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster") - fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection") - fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'") - if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, j.IgnorePreflightErrors()); err != nil { - return err - } - } - return nil -} - -// checkIfReadyForAdditionalControlPlane ensures that the cluster is in a state that supports -// joining an additional control plane instance and if the node is ready to preflight -func checkIfReadyForAdditionalControlPlane(initConfiguration *kubeadmapi.ClusterConfiguration, hasCertificateKey bool) error { - // blocks if the cluster was created without a stable control plane endpoint - if initConfiguration.ControlPlaneEndpoint == "" { - return errors.New("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address") - } - - if !hasCertificateKey { - // checks if the certificates that must be equal across controlplane instances are provided - if ret, err := certs.SharedCertificateExists(initConfiguration); !ret { - return err - } - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/BUILD b/cmd/kubeadm/app/cmd/phases/reset/BUILD deleted file mode 100644 index ba809ad35bf5e..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/BUILD +++ /dev/null @@ -1,65 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "cleanupnode.go", - "data.go", - "preflight.go", - "removeetcdmember.go", - "unmount.go", - "unmount_linux.go", - "updateclusterstatus.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/reset", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/etcd:go_default_library", - "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/app/util/initsystem:go_default_library", - "//cmd/kubeadm/app/util/runtime:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = [ - "cleanupnode_test.go", - "removeetcdmember_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go deleted file mode 100644 index a10f4c9b5ce9c..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode.go +++ /dev/null @@ -1,173 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "k8s.io/klog/v2" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/initsystem" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" - utilsexec "k8s.io/utils/exec" -) - -// NewCleanupNodePhase creates a kubeadm workflow phase that cleanup the node -func NewCleanupNodePhase() workflow.Phase { - return workflow.Phase{ - Name: "cleanup-node", - Aliases: []string{"cleanupnode"}, - Short: "Run cleanup node.", - Run: runCleanupNode, - InheritFlags: []string{ - options.CertificatesDir, - options.NodeCRISocket, - }, - } -} - -func runCleanupNode(c workflow.RunData) error { - r, ok := c.(resetData) - if !ok { - return errors.New("cleanup-node phase invoked with an invalid data struct") - } - certsDir := r.CertificatesDir() - - // Try to stop the kubelet service - klog.V(1).Infoln("[reset] Getting init system") - initSystem, err := initsystem.GetInitSystem() - if err != nil { - klog.Warningln("[reset] The kubelet service could not be stopped by kubeadm. Unable to detect a supported init system!") - klog.Warningln("[reset] Please ensure kubelet is stopped manually") - } else { - fmt.Println("[reset] Stopping the kubelet service") - if err := initSystem.ServiceStop("kubelet"); err != nil { - klog.Warningf("[reset] The kubelet service could not be stopped by kubeadm: [%v]\n", err) - klog.Warningln("[reset] Please ensure kubelet is stopped manually") - } - } - - // Try to unmount mounted directories under kubeadmconstants.KubeletRunDirectory in order to be able to remove the kubeadmconstants.KubeletRunDirectory directory later - fmt.Printf("[reset] Unmounting mounted directories in %q\n", kubeadmconstants.KubeletRunDirectory) - // In case KubeletRunDirectory holds a symbolic link, evaluate it - kubeletRunDir, err := absoluteKubeletRunDirectory() - if err == nil { - // Only clean absoluteKubeletRunDirectory if umountDirsCmd passed without error - r.AddDirsToClean(kubeletRunDir) - } - - klog.V(1).Info("[reset] Removing Kubernetes-managed containers") - if err := removeContainers(utilsexec.New(), r.CRISocketPath()); err != nil { - klog.Warningf("[reset] Failed to remove containers: %v\n", err) - } - - r.AddDirsToClean("/var/lib/dockershim", "/var/run/kubernetes", "/var/lib/cni") - - // Remove contents from the config and pki directories - klog.V(1).Infoln("[reset] Removing contents from the config and pki directories") - if certsDir != kubeadmapiv1beta2.DefaultCertificatesDir { - klog.Warningf("[reset] WARNING: Cleaning a non-default certificates directory: %q\n", certsDir) - } - resetConfigDir(kubeadmconstants.KubernetesDir, certsDir) - - return nil -} - -func absoluteKubeletRunDirectory() (string, error) { - absoluteKubeletRunDirectory, err := filepath.EvalSymlinks(kubeadmconstants.KubeletRunDirectory) - if err != nil { - klog.Warningf("[reset] Failed to evaluate the %q directory. Skipping its unmount and cleanup: %v\n", kubeadmconstants.KubeletRunDirectory, err) - return "", err - } - err = unmountKubeletDirectory(absoluteKubeletRunDirectory) - if err != nil { - klog.Warningf("[reset] Failed to unmount mounted directories in %s \n", kubeadmconstants.KubeletRunDirectory) - return "", err - } - return absoluteKubeletRunDirectory, nil -} - -func removeContainers(execer utilsexec.Interface, criSocketPath string) error { - containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath) - if err != nil { - return err - } - containers, err := containerRuntime.ListKubeContainers() - if err != nil { - return err - } - return containerRuntime.RemoveContainers(containers) -} - -// resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/. -func resetConfigDir(configPathDir, pkiPathDir string) { - dirsToClean := []string{ - filepath.Join(configPathDir, kubeadmconstants.ManifestsSubDirName), - pkiPathDir, - } - fmt.Printf("[reset] Deleting contents of config directories: %v\n", dirsToClean) - for _, dir := range dirsToClean { - if err := CleanDir(dir); err != nil { - klog.Warningf("[reset] Failed to delete contents of %q directory: %v", dir, err) - } - } - - filesToClean := []string{ - filepath.Join(configPathDir, kubeadmconstants.AdminKubeConfigFileName), - filepath.Join(configPathDir, kubeadmconstants.KubeletKubeConfigFileName), - filepath.Join(configPathDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName), - filepath.Join(configPathDir, kubeadmconstants.ControllerManagerKubeConfigFileName), - filepath.Join(configPathDir, kubeadmconstants.SchedulerKubeConfigFileName), - } - fmt.Printf("[reset] Deleting files: %v\n", filesToClean) - for _, path := range filesToClean { - if err := os.RemoveAll(path); err != nil { - klog.Warningf("[reset] Failed to remove file: %q [%v]\n", path, err) - } - } -} - -// CleanDir removes everything in a directory, but not the directory itself -func CleanDir(filePath string) error { - // If the directory doesn't even exist there's nothing to do, and we do - // not consider this an error - if _, err := os.Stat(filePath); os.IsNotExist(err) { - return nil - } - - d, err := os.Open(filePath) - if err != nil { - return err - } - defer d.Close() - names, err := d.Readdirnames(-1) - if err != nil { - return err - } - for _, name := range names { - if err = os.RemoveAll(filepath.Join(filePath, name)); err != nil { - return err - } - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go b/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go deleted file mode 100644 index d472c8e8d3e59..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/cleanupnode_test.go +++ /dev/null @@ -1,220 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -func assertExists(t *testing.T, path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Errorf("file/directory does not exist; error: %s", err) - t.Errorf("file/directory does not exist: %s", path) - } -} - -func assertNotExists(t *testing.T, path string) { - if _, err := os.Stat(path); err == nil { - t.Errorf("file/dir exists: %s", path) - } -} - -// assertDirEmpty verifies a directory either does not exist, or is empty. -func assertDirEmpty(t *testing.T, path string) { - dac := preflight.DirAvailableCheck{Path: path} - _, errors := dac.Check() - if len(errors) != 0 { - t.Errorf("directory not empty: [%v]", errors) - } -} - -func TestConfigDirCleaner(t *testing.T) { - tests := map[string]struct { - resetDir string - setupDirs []string - setupFiles []string - verifyExists []string - verifyNotExists []string - }{ - "simple reset": { - setupDirs: []string{ - "manifests", - "pki", - }, - setupFiles: []string{ - "manifests/etcd.yaml", - "manifests/kube-apiserver.yaml", - "pki/ca.pem", - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.KubeletKubeConfigFileName, - }, - verifyExists: []string{ - "manifests", - "pki", - }, - }, - "partial reset": { - setupDirs: []string{ - "pki", - }, - setupFiles: []string{ - "pki/ca.pem", - kubeadmconstants.KubeletKubeConfigFileName, - }, - verifyExists: []string{ - "pki", - }, - verifyNotExists: []string{ - "manifests", - }, - }, - "preserve unrelated file foo": { - setupDirs: []string{ - "manifests", - "pki", - }, - setupFiles: []string{ - "manifests/etcd.yaml", - "manifests/kube-apiserver.yaml", - "pki/ca.pem", - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.KubeletKubeConfigFileName, - "foo", - }, - verifyExists: []string{ - "manifests", - "pki", - "foo", - }, - }, - "preserve hidden files and directories": { - setupDirs: []string{ - "manifests", - "pki", - ".mydir", - }, - setupFiles: []string{ - "manifests/etcd.yaml", - "manifests/kube-apiserver.yaml", - "pki/ca.pem", - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.KubeletKubeConfigFileName, - ".mydir/.myfile", - }, - verifyExists: []string{ - "manifests", - "pki", - ".mydir", - ".mydir/.myfile", - }, - }, - "no-op reset": { - verifyNotExists: []string{ - "pki", - "manifests", - }, - }, - "not a directory": { - resetDir: "test-path", - setupFiles: []string{ - "test-path", - }, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - t.Logf("Running test: %s", name) - - // Create a temporary directory for our fake config dir: - tmpDir, err := ioutil.TempDir("", "kubeadm-reset-test") - if err != nil { - t.Errorf("Unable to create temporary directory: %s", err) - } - - for _, createDir := range test.setupDirs { - err := os.Mkdir(filepath.Join(tmpDir, createDir), 0700) - if err != nil { - t.Errorf("Unable to setup test config directory: %s", err) - } - } - - for _, createFile := range test.setupFiles { - fullPath := filepath.Join(tmpDir, createFile) - f, err := os.Create(fullPath) - if err != nil { - t.Errorf("Unable to create test file: %s", err) - } - f.Close() - } - - if test.resetDir == "" { - test.resetDir = "pki" - } - resetConfigDir(tmpDir, filepath.Join(tmpDir, test.resetDir)) - - // Verify the files we cleanup implicitly in every test: - assertExists(t, tmpDir) - assertNotExists(t, filepath.Join(tmpDir, kubeadmconstants.AdminKubeConfigFileName)) - assertNotExists(t, filepath.Join(tmpDir, kubeadmconstants.KubeletKubeConfigFileName)) - assertDirEmpty(t, filepath.Join(tmpDir, "manifests")) - assertDirEmpty(t, filepath.Join(tmpDir, "pki")) - - // Verify the files as requested by the test: - for _, path := range test.verifyExists { - assertExists(t, filepath.Join(tmpDir, path)) - } - for _, path := range test.verifyNotExists { - assertNotExists(t, filepath.Join(tmpDir, path)) - } - - os.RemoveAll(tmpDir) - }) - } -} - -func TestRemoveContainers(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("id1\nid2"), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - func() ([]byte, []byte, error) { return []byte(""), nil, nil }, - }, - } - fexec := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - removeContainers(&fexec, "unix:///var/run/crio/crio.sock") -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/data.go b/cmd/kubeadm/app/cmd/phases/reset/data.go deleted file mode 100644 index 0bbfa00ab3ee6..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/data.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io" - - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -// resetData is the interface to use for reset phases. -// The "resetData" type from "cmd/reset.go" must satisfy this interface. -type resetData interface { - ForceReset() bool - InputReader() io.Reader - IgnorePreflightErrors() sets.String - Cfg() *kubeadmapi.InitConfiguration - Client() clientset.Interface - AddDirsToClean(dirs ...string) - CertificatesDir() string - CRISocketPath() string -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/preflight.go b/cmd/kubeadm/app/cmd/phases/reset/preflight.go deleted file mode 100644 index e68985ea76b2a..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/preflight.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "bufio" - "errors" - "fmt" - "strings" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" -) - -// NewPreflightPhase creates a kubeadm workflow phase implements preflight checks for reset -func NewPreflightPhase() workflow.Phase { - return workflow.Phase{ - Name: "preflight", - Aliases: []string{"pre-flight"}, - Short: "Run reset pre-flight checks", - Long: "Run pre-flight checks for kubeadm reset.", - Run: runPreflight, - InheritFlags: []string{ - options.IgnorePreflightErrors, - options.ForceReset, - }, - } -} - -// runPreflight executes preflight checks logic. -func runPreflight(c workflow.RunData) error { - r, ok := c.(resetData) - if !ok { - return errors.New("preflight phase invoked with an invalid data struct") - } - - if !r.ForceReset() { - fmt.Println("[reset] WARNING: Changes made to this host by 'kubeadm init' or 'kubeadm join' will be reverted.") - fmt.Print("[reset] Are you sure you want to proceed? [y/N]: ") - s := bufio.NewScanner(r.InputReader()) - s.Scan() - if err := s.Err(); err != nil { - return err - } - if strings.ToLower(s.Text()) != "y" { - return errors.New("aborted reset operation") - } - } - - fmt.Println("[preflight] Running pre-flight checks") - return preflight.RunRootCheckOnly(r.IgnorePreflightErrors()) -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/removeetcdmember.go b/cmd/kubeadm/app/cmd/phases/reset/removeetcdmember.go deleted file mode 100644 index 5910364c5a824..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/removeetcdmember.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "errors" - "fmt" - "path/filepath" - - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - utilstaticpod "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" -) - -// NewRemoveETCDMemberPhase creates a kubeadm workflow phase for remove-etcd-member -func NewRemoveETCDMemberPhase() workflow.Phase { - return workflow.Phase{ - Name: "remove-etcd-member", - Short: "Remove a local etcd member.", - Long: "Remove a local etcd member for a control plane node.", - Run: runRemoveETCDMemberPhase, - InheritFlags: []string{ - options.KubeconfigPath, - }, - } -} - -func runRemoveETCDMemberPhase(c workflow.RunData) error { - r, ok := c.(resetData) - if !ok { - return errors.New("remove-etcd-member-phase phase invoked with an invalid data struct") - } - cfg := r.Cfg() - - // Only clear etcd data when using local etcd. - klog.V(1).Infoln("[reset] Checking for etcd config") - etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml") - etcdDataDir, err := getEtcdDataDir(etcdManifestPath, cfg) - if err == nil { - r.AddDirsToClean(etcdDataDir) - if cfg != nil { - if err := etcdphase.RemoveStackedEtcdMemberFromCluster(r.Client(), cfg); err != nil { - klog.Warningf("[reset] failed to remove etcd member: %v, please manually remove this etcd member using etcdctl", err) - } - } - } else { - fmt.Println("[reset] No etcd config found. Assuming external etcd") - fmt.Println("[reset] Please, manually reset etcd to prevent further issues") - } - - return nil -} - -func getEtcdDataDir(manifestPath string, cfg *kubeadmapi.InitConfiguration) (string, error) { - const etcdVolumeName = "etcd-data" - var dataDir string - - if cfg != nil && cfg.Etcd.Local != nil { - return cfg.Etcd.Local.DataDir, nil - } - klog.Warningln("[reset] No kubeadm config, using etcd pod spec to get data directory") - - etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath) - if err != nil { - return "", err - } - - for _, volumeMount := range etcdPod.Spec.Volumes { - if volumeMount.Name == etcdVolumeName { - dataDir = volumeMount.HostPath.Path - break - } - } - if dataDir == "" { - return dataDir, errors.New("invalid etcd pod manifest") - } - return dataDir, nil -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/removeetcdmember_test.go b/cmd/kubeadm/app/cmd/phases/reset/removeetcdmember_test.go deleted file mode 100644 index 007881d83f58c..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/removeetcdmember_test.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/lithammer/dedent" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -const ( - etcdPod = `apiVersion: v1 -kind: Pod -metadata: -spec: - volumes: - - hostPath: - path: /path/to/etcd - type: DirectoryOrCreate - name: etcd-data - - hostPath: - path: /etc/kubernetes/pki/etcd - type: DirectoryOrCreate - name: etcd-certs` - - etcdPodWithoutDataVolume = `apiVersion: v1 -kind: Pod -metadata: -spec: - volumes: - - hostPath: - path: /etc/kubernetes/pki/etcd - type: DirectoryOrCreate - name: etcd-certs` - - etcdPodInvalid = `invalid pod` -) - -func TestGetEtcdDataDir(t *testing.T) { - tests := map[string]struct { - dataDir string - podYaml string - expectErr bool - writeManifest bool - validConfig bool - }{ - "non-existent file returns error": { - expectErr: true, - writeManifest: false, - validConfig: true, - }, - "return etcd data dir": { - dataDir: "/path/to/etcd", - podYaml: etcdPod, - expectErr: false, - writeManifest: true, - validConfig: true, - }, - "invalid etcd pod": { - podYaml: etcdPodInvalid, - expectErr: true, - writeManifest: true, - validConfig: true, - }, - "etcd pod spec without data volume": { - podYaml: etcdPodWithoutDataVolume, - expectErr: true, - writeManifest: true, - validConfig: true, - }, - "kubeconfig file doesn't exist": { - dataDir: "/path/to/etcd", - podYaml: etcdPod, - expectErr: false, - writeManifest: true, - validConfig: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - manifestPath := filepath.Join(tmpdir, "etcd.yaml") - if test.writeManifest { - err := ioutil.WriteFile(manifestPath, []byte(test.podYaml), 0644) - if err != nil { - t.Fatalf(dedent.Dedent("failed to write pod manifest\n%s\n\tfatal error: %v"), name, err) - } - } - - var dataDir string - var err error - if test.validConfig { - cfg := &kubeadmapi.InitConfiguration{} - dataDir, err = getEtcdDataDir(manifestPath, cfg) - } else { - dataDir, err = getEtcdDataDir(manifestPath, nil) - } - - if (err != nil) != test.expectErr { - t.Fatalf(dedent.Dedent( - "getEtcdDataDir failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), - name, - test.expectErr, - (err != nil), - err, - ) - } - - if dataDir != test.dataDir { - t.Fatalf(dedent.Dedent("getEtcdDataDir failed\n%s\n\texpected: %s\ngot: %s"), name, test.dataDir, dataDir) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/unmount.go b/cmd/kubeadm/app/cmd/phases/reset/unmount.go deleted file mode 100644 index ab0f81bb1189c..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/unmount.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build !linux - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "k8s.io/klog/v2" -) - -// unmountKubeletDirectory is a NOOP on all but linux. -func unmountKubeletDirectory(absoluteKubeletRunDirectory string) error { - klog.Warning("Cannot unmount filesystems on current OS, all mounted file systems will need to be manually unmounted") - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/unmount_linux.go b/cmd/kubeadm/app/cmd/phases/reset/unmount_linux.go deleted file mode 100644 index 7d92d445547fc..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/unmount_linux.go +++ /dev/null @@ -1,52 +0,0 @@ -// +build linux - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "io/ioutil" - "strings" - "syscall" - - "k8s.io/klog/v2" -) - -// unmountKubeletDirectory unmounts all paths that contain KubeletRunDirectory -func unmountKubeletDirectory(absoluteKubeletRunDirectory string) error { - raw, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return err - } - - if !strings.HasSuffix(absoluteKubeletRunDirectory, "/") { - // trailing "/" is needed to ensure that possibly mounted /var/lib/kubelet is skipped - absoluteKubeletRunDirectory += "/" - } - - mounts := strings.Split(string(raw), "\n") - for _, mount := range mounts { - m := strings.Split(mount, " ") - if len(m) < 2 || !strings.HasPrefix(m[1], absoluteKubeletRunDirectory) { - continue - } - if err := syscall.Unmount(m[1], 0); err != nil { - klog.Warningf("[reset] Failed to unmount mounted directory in %s: %s", absoluteKubeletRunDirectory, m[1]) - } - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/reset/updateclusterstatus.go b/cmd/kubeadm/app/cmd/phases/reset/updateclusterstatus.go deleted file mode 100644 index 422147c2e60f7..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/reset/updateclusterstatus.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "errors" - "os" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" -) - -// NewUpdateClusterStatus creates a kubeadm workflow phase for update-cluster-status -func NewUpdateClusterStatus() workflow.Phase { - return workflow.Phase{ - Name: "update-cluster-status", - Short: "Remove this node from the ClusterStatus object.", - Long: "Remove this node from the ClusterStatus object if the node is a control plane node.", - Run: runUpdateClusterStatus, - } -} - -func runUpdateClusterStatus(c workflow.RunData) error { - r, ok := c.(resetData) - if !ok { - return errors.New("update-cluster-status phase invoked with an invalid data struct") - } - - // Reset the ClusterStatus for a given control-plane node. - cfg := r.Cfg() - if isControlPlane() && cfg != nil { - if err := uploadconfig.ResetClusterStatusForNode(cfg.NodeRegistration.Name, r.Client()); err != nil { - return err - } - } - - return nil -} - -// isControlPlane checks if a node is a control-plane node by looking up -// the kube-apiserver manifest file -func isControlPlane() bool { - filepath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, kubeadmconstants.GetStaticPodDirectory()) - _, err := os.Stat(filepath) - return !os.IsNotExist(err) -} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD b/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD deleted file mode 100644 index 2e1080798cc3a..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = [ - "controlplane.go", - "data.go", - "kubeletconfig.go", - "preflight.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/cmd/util:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/kubelet:go_default_library", - "//cmd/kubeadm/app/phases/upgrade:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/dryrun:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go deleted file mode 100644 index 1f83a155ed912..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - "os" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// NewControlPlane creates a kubeadm workflow phase that implements handling of control-plane upgrade. -func NewControlPlane() workflow.Phase { - phase := workflow.Phase{ - Name: "control-plane", - Short: "Upgrade the control plane instance deployed on this node, if any", - Run: runControlPlane(), - InheritFlags: []string{ - options.DryRun, - options.KubeconfigPath, - options.CertificateRenewal, - options.EtcdUpgrade, - options.Patches, - }, - } - return phase -} - -func runControlPlane() func(c workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(Data) - if !ok { - return errors.New("control-plane phase invoked with an invalid data struct") - } - - // if this is not a control-plane node, this phase should not be executed - if !data.IsControlPlaneNode() { - fmt.Println("[upgrade] Skipping phase. Not a control plane node.") - return nil - } - - // otherwise, retrieve all the info required for control plane upgrade - cfg := data.Cfg() - client := data.Client() - dryRun := data.DryRun() - etcdUpgrade := data.EtcdUpgrade() - renewCerts := data.RenewCerts() - patchesDir := data.PatchesDir() - - // Upgrade the control plane and etcd if installed on this node - fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion) - if dryRun { - return upgrade.DryRunStaticPodUpgrade(patchesDir, cfg) - } - - waiter := apiclient.NewKubeWaiter(data.Client(), upgrade.UpgradeManifestTimeout, os.Stdout) - - if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts, patchesDir); err != nil { - return errors.Wrap(err, "couldn't complete the static pod upgrade") - } - - fmt.Println("[upgrade] The control plane instance for this node was successfully updated!") - - return nil - } -} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go deleted file mode 100644 index ea56ea18fa7ef..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/data.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -// Data is the interface to use for kubeadm upgrade node phases. -// The "nodeData" type from "cmd/upgrade/node.go" must satisfy this interface. -type Data interface { - EtcdUpgrade() bool - RenewCerts() bool - DryRun() bool - Cfg() *kubeadmapi.InitConfiguration - IsControlPlaneNode() bool - Client() clientset.Interface - IgnorePreflightErrors() sets.String - PatchesDir() string -} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go deleted file mode 100644 index d4602fea94dc2..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" -) - -var ( - kubeletConfigLongDesc = cmdutil.LongDesc(` - Download the kubelet configuration from a ConfigMap of the form "kubelet-config-1.X" in the cluster, - where X is the minor version of the kubelet. kubeadm uses the KuberneteVersion field in the kubeadm-config - ConfigMap to determine what the _desired_ kubelet version is. - `) -) - -// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade. -func NewKubeletConfigPhase() workflow.Phase { - phase := workflow.Phase{ - Name: "kubelet-config", - Short: "Upgrade the kubelet configuration for this node", - Long: kubeletConfigLongDesc, - Run: runKubeletConfigPhase(), - InheritFlags: []string{ - options.DryRun, - options.KubeconfigPath, - }, - } - return phase -} - -func runKubeletConfigPhase() func(c workflow.RunData) error { - return func(c workflow.RunData) error { - data, ok := c.(Data) - if !ok { - return errors.New("kubelet-config phase invoked with an invalid data struct") - } - - // otherwise, retrieve all the info required for kubelet config upgrade - cfg := data.Cfg() - dryRun := data.DryRun() - - // Set up the kubelet directory to use. If dry-running, this will return a fake directory - kubeletDir, err := upgrade.GetKubeletDir(dryRun) - if err != nil { - return err - } - - // TODO: Checkpoint the current configuration first so that if something goes wrong it can be recovered - - // Store the kubelet component configuration. - if err = kubeletphase.WriteConfigToDisk(&cfg.ClusterConfiguration, kubeletDir); err != nil { - return err - } - - // If we're dry-running, print the generated manifests - if dryRun { - if err := printFilesIfDryRunning(dryRun, kubeletDir); err != nil { - return errors.Wrap(err, "error printing files on dryrun") - } - return nil - } - - fmt.Println("[upgrade] The configuration for this node was successfully updated!") - fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") - return nil - } -} - -// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup -func printFilesIfDryRunning(dryRun bool, kubeletDir string) error { - if !dryRun { - return nil - } - - // Print the contents of the upgraded file and pretend like they were in kubeadmconstants.KubeletRunDirectory - fileToPrint := dryrunutil.FileToPrint{ - RealPath: filepath.Join(kubeletDir, constants.KubeletConfigurationFileName), - PrintPath: filepath.Join(constants.KubeletRunDirectory, constants.KubeletConfigurationFileName), - } - return dryrunutil.PrintDryRunFiles([]dryrunutil.FileToPrint{fileToPrint}, os.Stdout) -} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go deleted file mode 100644 index 4bcfc5aa58a77..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - utilsexec "k8s.io/utils/exec" -) - -// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join -func NewPreflightPhase() workflow.Phase { - return workflow.Phase{ - Name: "preflight", - Short: "Run upgrade node pre-flight checks", - Long: "Run pre-flight checks for kubeadm upgrade node.", - Run: runPreflight, - InheritFlags: []string{ - options.IgnorePreflightErrors, - }, - } -} - -// runPreflight executes preflight checks logic. -func runPreflight(c workflow.RunData) error { - data, ok := c.(Data) - if !ok { - return errors.New("preflight phase invoked with an invalid data struct") - } - fmt.Println("[preflight] Running pre-flight checks") - - // First, check if we're root separately from the other preflight checks and fail fast - if err := preflight.RunRootCheckOnly(data.IgnorePreflightErrors()); err != nil { - return err - } - - // if this is a control-plane node, pull the basic images - if data.IsControlPlaneNode() { - if !data.DryRun() { - fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster") - fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection") - fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'") - if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil { - return err - } - } else { - fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')") - } - } else { - fmt.Println("[preflight] Skipping prepull. Not a control plane node.") - return nil - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/phases/util.go b/cmd/kubeadm/app/cmd/phases/util.go deleted file mode 100644 index 0cd39873f2557..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/util.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "k8s.io/component-base/version" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" -) - -// SetKubernetesVersion gets the current Kubeadm version and sets it as KubeadmVersion in the config, -// unless it's already set to a value different from the default. -func SetKubernetesVersion(cfg *kubeadmapiv1beta2.ClusterConfiguration) { - - if cfg.KubernetesVersion != kubeadmapiv1beta2.DefaultKubernetesVersion && cfg.KubernetesVersion != "" { - return - } - cfg.KubernetesVersion = version.Get().String() -} diff --git a/cmd/kubeadm/app/cmd/phases/util_test.go b/cmd/kubeadm/app/cmd/phases/util_test.go deleted file mode 100644 index 6e06c717eac30..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/util_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package phases - -import ( - "testing" - - "k8s.io/component-base/version" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" -) - -func TestSetKubernetesVersion(t *testing.T) { - - ver := version.Get().String() - - tests := []struct { - name string - input string - output string - }{ - { - name: "empty version is processed", - input: "", - output: ver, - }, - { - name: "default version is processed", - input: kubeadmapiv1beta2.DefaultKubernetesVersion, - output: ver, - }, - { - name: "any other version is skipped", - input: "v1.12.0", - output: "v1.12.0", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := &kubeadmapiv1beta2.ClusterConfiguration{KubernetesVersion: test.input} - SetKubernetesVersion(cfg) - if cfg.KubernetesVersion != test.output { - t.Fatalf("expected %q, got %q", test.output, cfg.KubernetesVersion) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/phases/workflow/BUILD b/cmd/kubeadm/app/cmd/phases/workflow/BUILD deleted file mode 100644 index 01571c97a71f5..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/workflow/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "doc.go", - "phase.go", - "runner.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow", - visibility = ["//visibility:public"], - deps = [ - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "doc_test.go", - "runner_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/phases/workflow/doc.go b/cmd/kubeadm/app/cmd/phases/workflow/doc.go deleted file mode 100644 index a5bd5871b74cb..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/workflow/doc.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Package workflow implements a workflow manager to be used for -implementing composable kubeadm workflows. - -Composable kubeadm workflows are built by an ordered sequence of phases; -each phase can have it's own, nested, ordered sequence of sub phases. -For instance - - preflight Run control-plane pre-flight checks - certs Generates all PKI assets necessary to establish the control plane - /ca Generates a self-signed Kubernetes CA to provision identities for Kubernetes components - /apiserver Generates an API server serving certificate and key - ... - kubeconfig Generates all kubeconfig files necessary to establish the control plane - /admin Generates a kubeconfig file for the admin to use and for kubeadm itself - /kubelet Generates a kubeconfig file for the kubelet to use. - ... - ... - -Phases are designed to be reusable across different kubeadm workflows thus allowing -e.g. reuse of phase certs in both kubeadm init and kubeadm join --control-plane workflows. - -Each workflow can be defined and managed using a Runner, that will run all -the phases according to the given order; nested phases will be executed immediately -after their parent phase. - -The phase runner can be bound to a cobra command; this operation sets the command description -giving evidence of the list of phases, and automatically creates sub commands -for invoking phases atomically. - -Autogenerated sub commands get flags according to the following rule: - -- global flags will be always inherited by autogenerated commands (this is managed by cobra) - -- local flags defined in the parent command might be inherited by autogenerated commands, -but this requires explicit opt-in so each phase can select the subset of relevant flags - -- it is possible to define additional flags that might be inherited by autogenerated commands -via explicit opt-in, but are not applied to the parent command - -In order to keep flags definition under control, please refer to the -"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" package. -*/ -package workflow diff --git a/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go b/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go deleted file mode 100644 index 894e7cd0afd95..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/workflow/doc_test.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/spf13/cobra" -) - -var myWorkflowRunner = NewRunner() - -type myWorkflowData struct { - data string -} - -func (c *myWorkflowData) Data() string { - return c.data -} - -type myPhaseData interface { - Data() string -} - -func ExamplePhase() { - // Create a phase - var myPhase1 = Phase{ - Name: "myPhase1", - Short: "A phase of a kubeadm composable workflow...", - Run: func(data RunData) error { - // transform data into a typed data struct - d, ok := data.(myPhaseData) - if !ok { - return errors.New("invalid RunData type") - } - - // implement your phase logic... - fmt.Printf("%v\n", d.Data()) - return nil - }, - } - - // Create another phase - var myPhase2 = Phase{ - Name: "myPhase2", - Short: "Another phase of a kubeadm composable workflow...", - Run: func(data RunData) error { - // transform data into a typed data struct - d, ok := data.(myPhaseData) - if !ok { - return errors.New("invalid RunData type") - } - - // implement your phase logic... - fmt.Printf("%v\n", d.Data()) - return nil - }, - } - - // Adds the new phases to the workflow - // Phases will be executed the same order they are added to the workflow - myWorkflowRunner.AppendPhase(myPhase1) - myWorkflowRunner.AppendPhase(myPhase2) -} - -func ExampleRunner_Run() { - // Create a phase - var myPhase = Phase{ - Name: "myPhase", - Short: "A phase of a kubeadm composable workflow...", - Run: func(data RunData) error { - // transform data into a typed data struct - d, ok := data.(myPhaseData) - if !ok { - return errors.New("invalid RunData type") - } - - // implement your phase logic... - fmt.Printf("%v\n", d.Data()) - return nil - }, - } - - // Adds the new phase to the workflow - var myWorkflowRunner = NewRunner() - myWorkflowRunner.AppendPhase(myPhase) - - // Defines the method that creates the runtime data shared - // among all the phases included in the workflow - myWorkflowRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (RunData, error) { - return myWorkflowData{data: "some data"}, nil - }) - - // Runs the workflow by passing a list of arguments - myWorkflowRunner.Run([]string{}) -} diff --git a/cmd/kubeadm/app/cmd/phases/workflow/phase.go b/cmd/kubeadm/app/cmd/phases/workflow/phase.go deleted file mode 100644 index 554635ac56c59..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/workflow/phase.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -// Phase provides an implementation of a workflow phase that allows -// creation of new phases by simply instantiating a variable of this type. -type Phase struct { - // name of the phase. - // Phase name should be unique among peer phases (phases belonging to - // the same workflow or phases belonging to the same parent phase). - Name string - - // Aliases returns the aliases for the phase. - Aliases []string - - // Short description of the phase. - Short string - - // Long returns the long description of the phase. - Long string - - // Example returns the example for the phase. - Example string - - // Hidden define if the phase should be hidden in the workflow help. - // e.g. PrintFilesIfDryRunning phase in the kubeadm init workflow is candidate for being hidden to the users - Hidden bool - - // Phases defines a nested, ordered sequence of phases. - Phases []Phase - - // RunAllSiblings allows to assign to a phase the responsibility to - // run all the sibling phases - // Nb. phase marked as RunAllSiblings can not have Run functions - RunAllSiblings bool - - // Run defines a function implementing the phase action. - // It is recommended to implent type assertion, e.g. using golang type switch, - // for validating the RunData type. - Run func(data RunData) error - - // RunIf define a function that implements a condition that should be checked - // before executing the phase action. - // If this function return nil, the phase action is always executed. - RunIf func(data RunData) (bool, error) - - // InheritFlags defines the list of flags that the cobra command generated for this phase should Inherit - // from local flags defined in the parent command / or additional flags defined in the phase runner. - // If the values is not set or empty, no flags will be assigned to the command - // Nb. global flags are automatically inherited by nested cobra command - InheritFlags []string - - // LocalFlags defines the list of flags that should be assigned to the cobra command generated - // for this phase. - // Nb. if two or phases have the same local flags, please consider using local flags in the parent command - // or additional flags defined in the phase runner. - LocalFlags *pflag.FlagSet - - // ArgsValidator defines the positional arg function to be used for validating args for this phase - // If not set a phase will adopt the args of the top level command. - ArgsValidator cobra.PositionalArgs -} - -// AppendPhase adds the given phase to the nested, ordered sequence of phases. -func (t *Phase) AppendPhase(phase Phase) { - t.Phases = append(t.Phases, phase) -} diff --git a/cmd/kubeadm/app/cmd/phases/workflow/runner.go b/cmd/kubeadm/app/cmd/phases/workflow/runner.go deleted file mode 100644 index 2a27b89971e0c..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/workflow/runner.go +++ /dev/null @@ -1,484 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -// phaseSeparator defines the separator to be used when concatenating nested -// phase names -const phaseSeparator = "/" - -// RunnerOptions defines the options supported during the execution of a -// kubeadm composable workflows -type RunnerOptions struct { - // FilterPhases defines the list of phases to be executed (if empty, all). - FilterPhases []string - - // SkipPhases defines the list of phases to be excluded by execution (if empty, none). - SkipPhases []string -} - -// RunData defines the data shared among all the phases included in the workflow, that is any type. -type RunData = interface{} - -// Runner implements management of composable kubeadm workflows. -type Runner struct { - // Options that regulate the runner behavior. - Options RunnerOptions - - // Phases composing the workflow to be managed by the runner. - Phases []Phase - - // runDataInitializer defines a function that creates the runtime data shared - // among all the phases included in the workflow - runDataInitializer func(*cobra.Command, []string) (RunData, error) - - // runData is part of the internal state of the runner and it is used for implementing - // a singleton in the InitData methods (thus avoiding to initialize data - // more than one time) - runData RunData - - // runCmd is part of the internal state of the runner and it is used to track the - // command that will trigger the runner (only if the runner is BindToCommand). - runCmd *cobra.Command - - // cmdAdditionalFlags holds additional, shared flags that could be added to the subcommands generated - // for phases. Flags could be inherited from the parent command too or added directly to each phase - cmdAdditionalFlags *pflag.FlagSet - - // phaseRunners is part of the internal state of the runner and provides - // a list of wrappers to phases composing the workflow with contextual - // information supporting phase execution. - phaseRunners []*phaseRunner -} - -// phaseRunner provides a wrapper to a Phase with the addition of a set -// of contextual information derived by the workflow managed by the Runner. -// TODO: If we ever decide to get more sophisticated we can swap this type with a well defined dag or tree library. -type phaseRunner struct { - // Phase provide access to the phase implementation - Phase - - // provide access to the parent phase in the workflow managed by the Runner. - parent *phaseRunner - - // level define the level of nesting of this phase into the workflow managed by - // the Runner. - level int - - // selfPath contains all the elements of the path that identify the phase into - // the workflow managed by the Runner. - selfPath []string - - // generatedName is the full name of the phase, that corresponds to the absolute - // path of the phase in the workflow managed by the Runner. - generatedName string - - // use is the phase usage string that will be printed in the workflow help. - // It corresponds to the relative path of the phase in the workflow managed by the Runner. - use string -} - -// NewRunner return a new runner for composable kubeadm workflows. -func NewRunner() *Runner { - return &Runner{ - Phases: []Phase{}, - } -} - -// AppendPhase adds the given phase to the ordered sequence of phases managed by the runner. -func (e *Runner) AppendPhase(t Phase) { - e.Phases = append(e.Phases, t) -} - -// computePhaseRunFlags return a map defining which phase should be run and which not. -// PhaseRunFlags are computed according to RunnerOptions. -func (e *Runner) computePhaseRunFlags() (map[string]bool, error) { - // Initialize support data structure - phaseRunFlags := map[string]bool{} - phaseHierarchy := map[string][]string{} - e.visitAll(func(p *phaseRunner) error { - // Initialize phaseRunFlags assuming that all the phases should be run. - phaseRunFlags[p.generatedName] = true - - // Initialize phaseHierarchy for the current phase (the list of phases - // depending on the current phase - phaseHierarchy[p.generatedName] = []string{} - - // Register current phase as part of its own parent hierarchy - parent := p.parent - for parent != nil { - phaseHierarchy[parent.generatedName] = append(phaseHierarchy[parent.generatedName], p.generatedName) - parent = parent.parent - } - return nil - }) - - // If a filter option is specified, set all phaseRunFlags to false except for - // the phases included in the filter and their hierarchy of nested phases. - if len(e.Options.FilterPhases) > 0 { - for i := range phaseRunFlags { - phaseRunFlags[i] = false - } - for _, f := range e.Options.FilterPhases { - if _, ok := phaseRunFlags[f]; !ok { - return phaseRunFlags, errors.Errorf("invalid phase name: %s", f) - } - phaseRunFlags[f] = true - for _, c := range phaseHierarchy[f] { - phaseRunFlags[c] = true - } - } - } - - // If a phase skip option is specified, set the corresponding phaseRunFlags - // to false and apply the same change to the underlying hierarchy - for _, f := range e.Options.SkipPhases { - if _, ok := phaseRunFlags[f]; !ok { - return phaseRunFlags, errors.Errorf("invalid phase name: %s", f) - } - phaseRunFlags[f] = false - for _, c := range phaseHierarchy[f] { - phaseRunFlags[c] = false - } - } - - return phaseRunFlags, nil -} - -// SetDataInitializer allows to setup a function that initialize the runtime data shared -// among all the phases included in the workflow. -// The method will receive in input the cmd that triggers the Runner (only if the runner is BindToCommand) -func (e *Runner) SetDataInitializer(builder func(cmd *cobra.Command, args []string) (RunData, error)) { - e.runDataInitializer = builder -} - -// InitData triggers the creation of runtime data shared among all the phases included in the workflow. -// This action can be executed explicitly out, when it is necessary to get the RunData -// before actually executing Run, or implicitly when invoking Run. -func (e *Runner) InitData(args []string) (RunData, error) { - if e.runData == nil && e.runDataInitializer != nil { - var err error - if e.runData, err = e.runDataInitializer(e.runCmd, args); err != nil { - return nil, err - } - } - - return e.runData, nil -} - -// Run the kubeadm composable kubeadm workflows. -func (e *Runner) Run(args []string) error { - e.prepareForExecution() - - // determine which phase should be run according to RunnerOptions - phaseRunFlags, err := e.computePhaseRunFlags() - if err != nil { - return err - } - - // builds the runner data - var data RunData - if data, err = e.InitData(args); err != nil { - return err - } - - err = e.visitAll(func(p *phaseRunner) error { - // if the phase should not be run, skip the phase. - if run, ok := phaseRunFlags[p.generatedName]; !run || !ok { - return nil - } - - // Errors if phases that are meant to create special subcommands only - // are wrongly assigned Run Methods - if p.RunAllSiblings && (p.RunIf != nil || p.Run != nil) { - return errors.Wrapf(err, "phase marked as RunAllSiblings can not have Run functions %s", p.generatedName) - } - - // If the phase defines a condition to be checked before executing the phase action. - if p.RunIf != nil { - // Check the condition and returns if the condition isn't satisfied (or fails) - ok, err := p.RunIf(data) - if err != nil { - return errors.Wrapf(err, "error execution run condition for phase %s", p.generatedName) - } - - if !ok { - return nil - } - } - - // Runs the phase action (if defined) - if p.Run != nil { - if err := p.Run(data); err != nil { - return errors.Wrapf(err, "error execution phase %s", p.generatedName) - } - } - - return nil - }) - - return err -} - -// Help returns text with the list of phases included in the workflow. -func (e *Runner) Help(cmdUse string) string { - e.prepareForExecution() - - // computes the max length of for each phase use line - maxLength := 0 - e.visitAll(func(p *phaseRunner) error { - if !p.Hidden && !p.RunAllSiblings { - length := len(p.use) - if maxLength < length { - maxLength = length - } - } - return nil - }) - - // prints the list of phases indented by level and formatted using the maxlength - // the list is enclosed in a mardown code block for ensuring better readability in the public web site - line := fmt.Sprintf("The %q command executes the following phases:\n", cmdUse) - line += "```\n" - offset := 2 - e.visitAll(func(p *phaseRunner) error { - if !p.Hidden && !p.RunAllSiblings { - padding := maxLength - len(p.use) + offset - line += strings.Repeat(" ", offset*p.level) // indentation - line += p.use // name + aliases - line += strings.Repeat(" ", padding) // padding right up to max length (+ offset for spacing) - line += p.Short // phase short description - line += "\n" - } - - return nil - }) - line += "```" - return line -} - -// SetAdditionalFlags allows to define flags to be added -// to the subcommands generated for each phase (but not existing in the parent command). -// Please note that this command needs to be done before BindToCommand. -// Nb. if a flag is used only by one phase, please consider using phase LocalFlags. -func (e *Runner) SetAdditionalFlags(fn func(*pflag.FlagSet)) { - // creates a new NewFlagSet - e.cmdAdditionalFlags = pflag.NewFlagSet("phaseAdditionalFlags", pflag.ContinueOnError) - // invokes the function that sets additional flags - fn(e.cmdAdditionalFlags) -} - -// BindToCommand bind the Runner to a cobra command by altering -// command help, adding phase related flags and by adding phases subcommands -// Please note that this command needs to be done once all the phases are added to the Runner. -func (e *Runner) BindToCommand(cmd *cobra.Command) { - // keep track of the command triggering the runner - e.runCmd = cmd - - // return early if no phases were added - if len(e.Phases) == 0 { - return - } - - e.prepareForExecution() - - // adds the phases subcommand - phaseCommand := &cobra.Command{ - Use: "phase", - Short: fmt.Sprintf("Use this command to invoke single phase of the %s workflow", cmd.Name()), - } - - cmd.AddCommand(phaseCommand) - - // generate all the nested subcommands for invoking single phases - subcommands := map[string]*cobra.Command{} - e.visitAll(func(p *phaseRunner) error { - // skip hidden phases - if p.Hidden { - return nil - } - - // initialize phase selector - phaseSelector := p.generatedName - - // if requested, set the phase to run all the sibling phases - if p.RunAllSiblings { - phaseSelector = p.parent.generatedName - } - - // creates phase subcommand - phaseCmd := &cobra.Command{ - Use: strings.ToLower(p.Name), - Short: p.Short, - Long: p.Long, - Example: p.Example, - Aliases: p.Aliases, - RunE: func(cmd *cobra.Command, args []string) error { - // if the phase has subphases, print the help and exits - if len(p.Phases) > 0 { - return cmd.Help() - } - - // overrides the command triggering the Runner using the phaseCmd - e.runCmd = cmd - e.Options.FilterPhases = []string{phaseSelector} - return e.Run(args) - }, - } - - // makes the new command inherits local flags from the parent command - // Nb. global flags will be inherited automatically - inheritsFlags(cmd.Flags(), phaseCmd.Flags(), p.InheritFlags) - - // makes the new command inherits additional flags for phases - if e.cmdAdditionalFlags != nil { - inheritsFlags(e.cmdAdditionalFlags, phaseCmd.Flags(), p.InheritFlags) - } - - // If defined, added phase local flags - if p.LocalFlags != nil { - p.LocalFlags.VisitAll(func(f *pflag.Flag) { - phaseCmd.Flags().AddFlag(f) - }) - } - - // if this phase has children (not a leaf) it doesn't accept any args - if len(p.Phases) > 0 { - phaseCmd.Args = cobra.NoArgs - } else { - if p.ArgsValidator == nil { - phaseCmd.Args = cmd.Args - } else { - phaseCmd.Args = p.ArgsValidator - } - } - - // adds the command to parent - if p.level == 0 { - phaseCommand.AddCommand(phaseCmd) - } else { - subcommands[p.parent.generatedName].AddCommand(phaseCmd) - } - - subcommands[p.generatedName] = phaseCmd - return nil - }) - - // alters the command description to show available phases - if cmd.Long != "" { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Long, e.Help(cmd.Use)) - } else { - cmd.Long = fmt.Sprintf("%s\n\n%s\n", cmd.Short, e.Help(cmd.Use)) - } - - // adds phase related flags to the main command - cmd.Flags().StringSliceVar(&e.Options.SkipPhases, "skip-phases", nil, "List of phases to be skipped") -} - -func inheritsFlags(sourceFlags, targetFlags *pflag.FlagSet, cmdFlags []string) { - // If the list of flag to be inherited from the parent command is not defined, no flag is added - if cmdFlags == nil { - return - } - - // add all the flags to be inherited to the target flagSet - sourceFlags.VisitAll(func(f *pflag.Flag) { - for _, c := range cmdFlags { - if f.Name == c { - targetFlags.AddFlag(f) - } - } - }) -} - -// visitAll provides a utility method for visiting all the phases in the workflow -// in the execution order and executing a func on each phase. -// Nested phase are visited immediately after their parent phase. -func (e *Runner) visitAll(fn func(*phaseRunner) error) error { - for _, currentRunner := range e.phaseRunners { - if err := fn(currentRunner); err != nil { - return err - } - } - return nil -} - -// prepareForExecution initialize the internal state of the Runner (the list of phaseRunner). -func (e *Runner) prepareForExecution() { - e.phaseRunners = []*phaseRunner{} - var parentRunner *phaseRunner - for _, phase := range e.Phases { - // skips phases that are meant to create special subcommands only - if phase.RunAllSiblings { - continue - } - - // add phases to the execution list - addPhaseRunner(e, parentRunner, phase) - } -} - -// addPhaseRunner adds the phaseRunner for a given phase to the phaseRunners list -func addPhaseRunner(e *Runner, parentRunner *phaseRunner, phase Phase) { - // computes contextual information derived by the workflow managed by the Runner. - use := cleanName(phase.Name) - generatedName := use - selfPath := []string{generatedName} - - if parentRunner != nil { - generatedName = strings.Join([]string{parentRunner.generatedName, generatedName}, phaseSeparator) - use = fmt.Sprintf("%s%s", phaseSeparator, use) - selfPath = append(parentRunner.selfPath, selfPath...) - } - - // creates the phaseRunner - currentRunner := &phaseRunner{ - Phase: phase, - parent: parentRunner, - level: len(selfPath) - 1, - selfPath: selfPath, - generatedName: generatedName, - use: use, - } - - // adds to the phaseRunners list - e.phaseRunners = append(e.phaseRunners, currentRunner) - - // iterate for the nested, ordered list of phases, thus storing - // phases in the expected executing order (child phase are stored immediately after their parent phase). - for _, childPhase := range phase.Phases { - addPhaseRunner(e, currentRunner, childPhase) - } -} - -// cleanName makes phase name suitable for the runner help, by lowercasing the name -// and removing args descriptors, if any -func cleanName(name string) string { - ret := strings.ToLower(name) - if pos := strings.Index(ret, " "); pos != -1 { - ret = ret[:pos] - } - return ret -} diff --git a/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go b/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go deleted file mode 100644 index ef546fdd36762..0000000000000 --- a/cmd/kubeadm/app/cmd/phases/workflow/runner_test.go +++ /dev/null @@ -1,625 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package workflow - -import ( - "fmt" - "reflect" - "strings" - "testing" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" -) - -func phaseBuilder(name string, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - } -} - -func TestComputePhaseRunFlags(t *testing.T) { - - var usecases = []struct { - name string - options RunnerOptions - expected map[string]bool - expectedError bool - }{ - { - name: "no options > all phases", - options: RunnerOptions{}, - expected: map[string]bool{"foo": true, "foo/bar": true, "foo/baz": true, "qux": true}, - }, - { - name: "options can filter phases", - options: RunnerOptions{FilterPhases: []string{"foo/baz", "qux"}}, - expected: map[string]bool{"foo": false, "foo/bar": false, "foo/baz": true, "qux": true}, - }, - { - name: "options can filter phases - hierarchy is considered", - options: RunnerOptions{FilterPhases: []string{"foo"}}, - expected: map[string]bool{"foo": true, "foo/bar": true, "foo/baz": true, "qux": false}, - }, - { - name: "options can skip phases", - options: RunnerOptions{SkipPhases: []string{"foo/bar", "qux"}}, - expected: map[string]bool{"foo": true, "foo/bar": false, "foo/baz": true, "qux": false}, - }, - { - name: "options can skip phases - hierarchy is considered", - options: RunnerOptions{SkipPhases: []string{"foo"}}, - expected: map[string]bool{"foo": false, "foo/bar": false, "foo/baz": false, "qux": true}, - }, - { - name: "skip options have higher precedence than filter options", - options: RunnerOptions{ - FilterPhases: []string{"foo"}, // "foo", "foo/bar", "foo/baz" true - SkipPhases: []string{"foo/bar"}, // "foo/bar" false - }, - expected: map[string]bool{"foo": true, "foo/bar": false, "foo/baz": true, "qux": false}, - }, - { - name: "invalid filter option", - options: RunnerOptions{FilterPhases: []string{"invalid"}}, - expectedError: true, - }, - { - name: "invalid skip option", - options: RunnerOptions{SkipPhases: []string{"invalid"}}, - expectedError: true, - }, - } - for _, u := range usecases { - t.Run(u.name, func(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder("foo", - phaseBuilder("bar"), - phaseBuilder("baz"), - ), - phaseBuilder("qux"), - }, - } - - w.prepareForExecution() - w.Options = u.options - actual, err := w.computePhaseRunFlags() - if (err != nil) != u.expectedError { - t.Errorf("Unexpected error: %v", err) - } - if err != nil { - return - } - if !reflect.DeepEqual(actual, u.expected) { - t.Errorf("\nactual:\n\t%v\nexpected:\n\t%v\n", actual, u.expected) - } - }) - } -} - -func phaseBuilder1(name string, runIf func(data RunData) (bool, error), phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - Run: runBuilder(name), - RunIf: runIf, - } -} - -var callstack []string - -func runBuilder(name string) func(data RunData) error { - return func(data RunData) error { - callstack = append(callstack, name) - return nil - } -} - -func runConditionTrue(data RunData) (bool, error) { - return true, nil -} - -func runConditionFalse(data RunData) (bool, error) { - return false, nil -} - -func TestRunOrderAndConditions(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder1("foo", nil, - phaseBuilder1("bar", runConditionTrue), - phaseBuilder1("baz", runConditionFalse), - ), - phaseBuilder1("qux", runConditionTrue), - }, - } - - var usecases = []struct { - name string - options RunnerOptions - expectedOrder []string - }{ - { - name: "Run respect runCondition", - expectedOrder: []string{"foo", "bar", "qux"}, - }, - { - name: "Run takes options into account", - options: RunnerOptions{FilterPhases: []string{"foo"}, SkipPhases: []string{"foo/baz"}}, - expectedOrder: []string{"foo", "bar"}, - }, - } - for _, u := range usecases { - t.Run(u.name, func(t *testing.T) { - callstack = []string{} - w.Options = u.options - err := w.Run([]string{}) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if !reflect.DeepEqual(callstack, u.expectedOrder) { - t.Errorf("\ncallstack:\n\t%v\nexpected:\n\t%v\n", callstack, u.expectedOrder) - } - }) - } -} - -func phaseBuilder2(name string, runIf func(data RunData) (bool, error), run func(data RunData) error, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - Run: run, - RunIf: runIf, - } -} - -func runPass(data RunData) error { - return nil -} - -func runFails(data RunData) error { - return errors.New("run fails") -} - -func runConditionPass(data RunData) (bool, error) { - return true, nil -} - -func runConditionFails(data RunData) (bool, error) { - return false, errors.New("run condition fails") -} - -func TestRunHandleErrors(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder2("foo", runConditionPass, runPass), - phaseBuilder2("bar", runConditionPass, runFails), - phaseBuilder2("baz", runConditionFails, runPass), - }, - } - - var usecases = []struct { - name string - options RunnerOptions - expectedError bool - }{ - { - name: "no errors", - options: RunnerOptions{FilterPhases: []string{"foo"}}, - }, - { - name: "run fails", - options: RunnerOptions{FilterPhases: []string{"bar"}}, - expectedError: true, - }, - { - name: "run condition fails", - options: RunnerOptions{FilterPhases: []string{"baz"}}, - expectedError: true, - }, - } - for _, u := range usecases { - t.Run(u.name, func(t *testing.T) { - w.Options = u.options - err := w.Run([]string{}) - if (err != nil) != u.expectedError { - t.Errorf("Unexpected error: %v", err) - } - }) - } -} - -func phaseBuilder3(name string, hidden bool, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - Hidden: hidden, - } -} - -func TestHelp(t *testing.T) { - var w = Runner{ - Phases: []Phase{ - phaseBuilder3("foo", false, - phaseBuilder3("bar [arg]", false), - phaseBuilder3("baz", true), - ), - phaseBuilder3("qux", false), - }, - } - - expected := "The \"myCommand\" command executes the following phases:\n" + - "```\n" + - "foo long description for foo ...\n" + - " /bar long description for bar [arg] ...\n" + - "qux long description for qux ...\n" + - "```" - - actual := w.Help("myCommand") - if !reflect.DeepEqual(actual, expected) { - t.Errorf("\nactual:\n\t%v\nexpected:\n\t%v\n", actual, expected) - } -} - -func phaseBuilder4(name string, cmdFlags []string, phases ...Phase) Phase { - return Phase{ - Name: name, - Phases: phases, - InheritFlags: cmdFlags, - } -} - -func phaseBuilder5(name string, flags *pflag.FlagSet) Phase { - return Phase{ - Name: name, - LocalFlags: flags, - } -} - -type argTest struct { - args cobra.PositionalArgs - pass []string - fail []string -} - -func phaseBuilder6(name string, args cobra.PositionalArgs, phases ...Phase) Phase { - return Phase{ - Name: name, - Short: fmt.Sprintf("long description for %s ...", name), - Phases: phases, - ArgsValidator: args, - } -} - -// customArgs is a custom cobra.PositionArgs function -func customArgs(cmd *cobra.Command, args []string) error { - for _, a := range args { - if a != "qux" { - return fmt.Errorf("arg %s does not equal qux", a) - } - } - return nil -} - -func TestBindToCommandArgRequirements(t *testing.T) { - - // because cobra.ExactArgs(1) == cobra.ExactArgs(3), it is needed - // to run test argument sets that both pass and fail to ensure the correct function was set. - var usecases = []struct { - name string - runner Runner - testCases map[string]argTest - cmd *cobra.Command - }{ - { - name: "leaf command, no defined args, follow parent", - runner: Runner{ - Phases: []Phase{phaseBuilder("foo")}, - }, - testCases: map[string]argTest{ - "phase foo": { - pass: []string{"one", "two", "three"}, - fail: []string{"one", "two"}, - args: cobra.ExactArgs(3), - }, - }, - cmd: &cobra.Command{ - Use: "init", - Args: cobra.ExactArgs(3), - }, - }, - { - name: "container cmd expect none, custom arg check for leaf", - runner: Runner{ - Phases: []Phase{phaseBuilder6("foo", cobra.NoArgs, - phaseBuilder6("bar", cobra.ExactArgs(1)), - phaseBuilder6("baz", customArgs), - )}, - }, - testCases: map[string]argTest{ - "phase foo": { - pass: []string{}, - fail: []string{"one"}, - args: cobra.NoArgs, - }, - "phase foo bar": { - pass: []string{"one"}, - fail: []string{"one", "two"}, - args: cobra.ExactArgs(1), - }, - "phase foo baz": { - pass: []string{"qux"}, - fail: []string{"one"}, - args: customArgs, - }, - }, - cmd: &cobra.Command{ - Use: "init", - Args: cobra.NoArgs, - }, - }, - } - - for _, rt := range usecases { - t.Run(rt.name, func(t *testing.T) { - - rt.runner.BindToCommand(rt.cmd) - - // Checks that cmd gets a new phase subcommand - phaseCmd := getCmd(rt.cmd, "phase") - if phaseCmd == nil { - t.Error("cmd didn't have phase subcommand\n") - return - } - - for c, args := range rt.testCases { - - cCmd := getCmd(rt.cmd, c) - if cCmd == nil { - t.Errorf("cmd didn't have %s subcommand\n", c) - continue - } - - // Ensure it is the expected function - if reflect.ValueOf(cCmd.Args).Pointer() != reflect.ValueOf(args.args).Pointer() { - t.Error("The function poiners where not equal.") - } - - // Test passing argument set - err := cCmd.Args(cCmd, args.pass) - - if err != nil { - t.Errorf("command %s should validate the args: %v\n %v", cCmd.Name(), args.pass, err) - } - - // Test failing argument set - err = cCmd.Args(cCmd, args.fail) - - if err == nil { - t.Errorf("command %s should fail to validate the args: %v\n %v", cCmd.Name(), args.pass, err) - } - } - - }) - } -} - -func TestBindToCommand(t *testing.T) { - - var dummy string - localFlags := pflag.NewFlagSet("dummy", pflag.ContinueOnError) - localFlags.StringVarP(&dummy, "flag4", "d", "d", "d") - - var usecases = []struct { - name string - runner Runner - expectedCmdAndFlags map[string][]string - setAdditionalFlags func(*pflag.FlagSet) - }{ - { - name: "when there are no phases, cmd should be left untouched", - runner: Runner{}, - }, - { - name: "phases should not inherits any parent flags by default", - runner: Runner{ - Phases: []Phase{phaseBuilder4("foo", nil)}, - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {}, - }, - }, - { - name: "phases should be allowed to select parent flags to inherits", - runner: Runner{ - Phases: []Phase{phaseBuilder4("foo", []string{"flag1"})}, - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag1"}, //not "flag2" - }, - }, - { - name: "it should be possible to apply additional flags to all phases", - runner: Runner{ - Phases: []Phase{ - phaseBuilder4("foo", []string{"flag3"}), - phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), - phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too - }, - }, - setAdditionalFlags: func(flags *pflag.FlagSet) { - var dummy3 string - flags.StringVarP(&dummy3, "flag3", "c", "c", "c") - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag3"}, - "phase bar": {"flag1", "flag2", "flag3"}, - "phase baz": {"flag1"}, - }, - }, - { - name: "it should be possible to apply custom flags to single phases", - runner: Runner{ - Phases: []Phase{phaseBuilder5("foo", localFlags)}, - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag4"}, - }, - }, - { - name: "all the above applies to nested phases too", - runner: Runner{ - Phases: []Phase{ - phaseBuilder4("foo", []string{"flag3"}, - phaseBuilder4("bar", []string{"flag1", "flag2", "flag3"}), - phaseBuilder4("baz", []string{"flag1"}), //test if additional flags are filtered too - phaseBuilder5("qux", localFlags), - ), - }, - }, - setAdditionalFlags: func(flags *pflag.FlagSet) { - var dummy3 string - flags.StringVarP(&dummy3, "flag3", "c", "c", "c") - }, - expectedCmdAndFlags: map[string][]string{ - "phase foo": {"flag3"}, - "phase foo bar": {"flag1", "flag2", "flag3"}, - "phase foo baz": {"flag1"}, - "phase foo qux": {"flag4"}, - }, - }, - } - for _, rt := range usecases { - t.Run(rt.name, func(t *testing.T) { - - var dummy1, dummy2 string - cmd := &cobra.Command{ - Use: "init", - } - - cmd.Flags().StringVarP(&dummy1, "flag1", "a", "a", "a") - cmd.Flags().StringVarP(&dummy2, "flag2", "b", "b", "b") - - if rt.setAdditionalFlags != nil { - rt.runner.SetAdditionalFlags(rt.setAdditionalFlags) - } - - rt.runner.BindToCommand(cmd) - - // in case of no phases, checks that cmd is untouched - if len(rt.runner.Phases) == 0 { - if cmd.Long != "" { - t.Error("cmd.Long is set while it should be leaved untouched\n") - } - - if cmd.Flags().Lookup("skip-phases") != nil { - t.Error("cmd has skip-phases flag while it should not\n") - } - - if getCmd(cmd, "phase") != nil { - t.Error("cmd has phase subcommand while it should not\n") - } - - return - } - - // Otherwise, if there are phases - - // Checks that cmd get the description set and the skip-phases flags - if cmd.Long == "" { - t.Error("cmd.Long not set\n") - } - - if cmd.Flags().Lookup("skip-phases") == nil { - t.Error("cmd didn't have skip-phases flag\n") - } - - // Checks that cmd gets a new phase subcommand (without local flags) - phaseCmd := getCmd(cmd, "phase") - if phaseCmd == nil { - t.Error("cmd didn't have phase subcommand\n") - return - } - if err := cmdHasFlags(phaseCmd); err != nil { - t.Errorf("command phase didn't have expected flags: %v\n", err) - } - - // Checks that cmd subcommand gets subcommand for phases (without flags properly sets) - for c, flags := range rt.expectedCmdAndFlags { - - cCmd := getCmd(cmd, c) - if cCmd == nil { - t.Errorf("cmd didn't have %s subcommand\n", c) - continue - } - - if err := cmdHasFlags(cCmd, flags...); err != nil { - t.Errorf("command %s didn't have expected flags: %v\n", c, err) - } - } - - }) - } -} - -func getCmd(parent *cobra.Command, nestedName string) *cobra.Command { - names := strings.Split(nestedName, " ") - for i, n := range names { - for _, c := range parent.Commands() { - if c.Name() == n { - if i == len(names)-1 { - return c - } - parent = c - } - } - } - - return nil -} - -func cmdHasFlags(cmd *cobra.Command, expectedFlags ...string) error { - flags := []string{} - cmd.Flags().VisitAll(func(f *pflag.Flag) { - flags = append(flags, f.Name) - }) - - for _, e := range expectedFlags { - found := false - for _, f := range flags { - if f == e { - found = true - } - } - if !found { - return errors.Errorf("flag %q does not exists in %s", e, flags) - } - } - - if len(flags) != len(expectedFlags) { - return errors.Errorf("expected flags %s, got %s", expectedFlags, flags) - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go deleted file mode 100644 index bd95d16a00132..0000000000000 --- a/cmd/kubeadm/app/cmd/reset.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "fmt" - "io" - - "github.com/lithammer/dedent" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/reset" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" -) - -var ( - iptablesCleanupInstructions = dedent.Dedent(` - The reset process does not reset or clean up iptables rules or IPVS tables. - If you wish to reset iptables, you must do so manually by using the "iptables" command. - - If your cluster was setup to utilize IPVS, run ipvsadm --clear (or similar) - to reset your system's IPVS tables. - - The reset process does not clean your kubeconfig files and you must remove them manually. - Please, check the contents of the $HOME/.kube/config file. - `) - - cniCleanupInstructions = dedent.Dedent(` - The reset process does not clean CNI configuration. To do so, you must remove /etc/cni/net.d - `) -) - -// resetOptions defines all the options exposed via flags by kubeadm reset. -type resetOptions struct { - certificatesDir string - criSocketPath string - forceReset bool - ignorePreflightErrors []string - kubeconfigPath string -} - -// resetData defines all the runtime information used when running the kubeadm reset workflow; -// this data is shared across all the phases that are included in the workflow. -type resetData struct { - certificatesDir string - client clientset.Interface - criSocketPath string - forceReset bool - ignorePreflightErrors sets.String - inputReader io.Reader - outputWriter io.Writer - cfg *kubeadmapi.InitConfiguration - dirsToClean []string -} - -// newResetOptions returns a struct ready for being used for creating cmd join flags. -func newResetOptions() *resetOptions { - return &resetOptions{ - certificatesDir: kubeadmapiv1beta2.DefaultCertificatesDir, - forceReset: false, - kubeconfigPath: kubeadmconstants.GetAdminKubeConfigPath(), - } -} - -// newResetData returns a new resetData struct to be used for the execution of the kubeadm reset workflow. -func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out io.Writer) (*resetData, error) { - var cfg *kubeadmapi.InitConfiguration - - client, err := getClientset(options.kubeconfigPath, false) - if err == nil { - klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath) - cfg, err = configutil.FetchInitConfigurationFromCluster(client, out, "reset", false, false) - if err != nil { - klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err) - } - } else { - klog.V(1).Infof("[reset] Could not obtain a client set from the kubeconfig file: %s", options.kubeconfigPath) - } - - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, ignorePreflightErrors(cfg)) - if err != nil { - return nil, err - } - if cfg != nil { - // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: - cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() - } - - var criSocketPath string - if options.criSocketPath == "" { - criSocketPath, err = resetDetectCRISocket(cfg) - if err != nil { - return nil, err - } - klog.V(1).Infof("[reset] Detected and using CRI socket: %s", criSocketPath) - } else { - criSocketPath = options.criSocketPath - klog.V(1).Infof("[reset] Using specified CRI socket: %s", criSocketPath) - } - - return &resetData{ - certificatesDir: options.certificatesDir, - client: client, - criSocketPath: criSocketPath, - forceReset: options.forceReset, - ignorePreflightErrors: ignorePreflightErrorsSet, - inputReader: in, - outputWriter: out, - cfg: cfg, - }, nil -} - -func ignorePreflightErrors(cfg *kubeadmapi.InitConfiguration) []string { - if cfg == nil { - return []string{} - } - return cfg.NodeRegistration.IgnorePreflightErrors -} - -// AddResetFlags adds reset flags -func AddResetFlags(flagSet *flag.FlagSet, resetOptions *resetOptions) { - flagSet.StringVar( - &resetOptions.certificatesDir, options.CertificatesDir, resetOptions.certificatesDir, - `The path to the directory where the certificates are stored. If specified, clean this directory.`, - ) - flagSet.BoolVarP( - &resetOptions.forceReset, options.ForceReset, "f", false, - "Reset the node without prompting for confirmation.", - ) - - options.AddKubeConfigFlag(flagSet, &resetOptions.kubeconfigPath) - options.AddIgnorePreflightErrorsFlag(flagSet, &resetOptions.ignorePreflightErrors) - cmdutil.AddCRISocketFlag(flagSet, &resetOptions.criSocketPath) -} - -// newCmdReset returns the "kubeadm reset" command -func newCmdReset(in io.Reader, out io.Writer, resetOptions *resetOptions) *cobra.Command { - if resetOptions == nil { - resetOptions = newResetOptions() - } - resetRunner := workflow.NewRunner() - - cmd := &cobra.Command{ - Use: "reset", - Short: "Performs a best effort revert of changes made to this host by 'kubeadm init' or 'kubeadm join'", - RunE: func(cmd *cobra.Command, args []string) error { - c, err := resetRunner.InitData(args) - if err != nil { - return err - } - - err = resetRunner.Run(args) - if err != nil { - return err - } - - // Then clean contents from the stateful kubelet, etcd and cni directories - data := c.(*resetData) - cleanDirs(data) - - // output help text instructing user how to remove cni folders - fmt.Print(cniCleanupInstructions) - // Output help text instructing user how to remove iptables rules - fmt.Print(iptablesCleanupInstructions) - return nil - }, - } - - AddResetFlags(cmd.Flags(), resetOptions) - - // initialize the workflow runner with the list of phases - resetRunner.AppendPhase(phases.NewPreflightPhase()) - resetRunner.AppendPhase(phases.NewUpdateClusterStatus()) - resetRunner.AppendPhase(phases.NewRemoveETCDMemberPhase()) - resetRunner.AppendPhase(phases.NewCleanupNodePhase()) - - // sets the data builder function, that will be used by the runner - // both when running the entire workflow or single phases - resetRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { - return newResetData(cmd, resetOptions, in, out) - }) - - // binds the Runner to kubeadm init command by altering - // command help, adding --skip-phases flag and by adding phases subcommands - resetRunner.BindToCommand(cmd) - - return cmd -} - -func cleanDirs(data *resetData) { - fmt.Printf("[reset] Deleting contents of stateful directories: %v\n", data.dirsToClean) - for _, dir := range data.dirsToClean { - klog.V(1).Infof("[reset] Deleting contents of %s", dir) - if err := phases.CleanDir(dir); err != nil { - klog.Warningf("[reset] Failed to delete contents of %q directory: %v", dir, err) - } - } -} - -// Cfg returns the InitConfiguration. -func (r *resetData) Cfg() *kubeadmapi.InitConfiguration { - return r.cfg -} - -// CertificatesDir returns the CertificatesDir. -func (r *resetData) CertificatesDir() string { - return r.certificatesDir -} - -// Client returns the Client for accessing the cluster. -func (r *resetData) Client() clientset.Interface { - return r.client -} - -// ForceReset returns the forceReset flag. -func (r *resetData) ForceReset() bool { - return r.forceReset -} - -// InputReader returns the io.reader used to read messages. -func (r *resetData) InputReader() io.Reader { - return r.inputReader -} - -// IgnorePreflightErrors returns the list of preflight errors to ignore. -func (r *resetData) IgnorePreflightErrors() sets.String { - return r.ignorePreflightErrors -} - -// AddDirsToClean add a list of dirs to the list of dirs that will be removed. -func (r *resetData) AddDirsToClean(dirs ...string) { - r.dirsToClean = append(r.dirsToClean, dirs...) -} - -// CRISocketPath returns the criSocketPath. -func (r *resetData) CRISocketPath() string { - return r.criSocketPath -} - -func resetDetectCRISocket(cfg *kubeadmapi.InitConfiguration) (string, error) { - if cfg != nil { - // first try to get the CRI socket from the cluster configuration - return cfg.NodeRegistration.CRISocket, nil - } - - // if this fails, try to detect it - return utilruntime.DetectCRISocket() -} diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go deleted file mode 100644 index 264b3f28b653b..0000000000000 --- a/cmd/kubeadm/app/cmd/token.go +++ /dev/null @@ -1,453 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "context" - "fmt" - "io" - "os" - "strings" - "text/tabwriter" - "time" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - "k8s.io/klog/v2" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/duration" - "k8s.io/cli-runtime/pkg/genericclioptions" - clientset "k8s.io/client-go/kubernetes" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" - outputapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha1" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" -) - -// newCmdToken returns cobra.Command for token management -func newCmdToken(out io.Writer, errW io.Writer) *cobra.Command { - var kubeConfigFile string - var dryRun bool - tokenCmd := &cobra.Command{ - Use: "token", - Short: "Manage bootstrap tokens", - Long: dedent.Dedent(` - This command manages bootstrap tokens. It is optional and needed only for advanced use cases. - - In short, bootstrap tokens are used for establishing bidirectional trust between a client and a server. - A bootstrap token can be used when a client (for example a node that is about to join the cluster) needs - to trust the server it is talking to. Then a bootstrap token with the "signing" usage can be used. - bootstrap tokens can also function as a way to allow short-lived authentication to the API Server - (the token serves as a way for the API Server to trust the client), for example for doing the TLS Bootstrap. - - What is a bootstrap token more exactly? - - It is a Secret in the kube-system namespace of type "bootstrap.kubernetes.io/token". - - A bootstrap token must be of the form "[a-z0-9]{6}.[a-z0-9]{16}". The former part is the public token ID, - while the latter is the Token Secret and it must be kept private at all circumstances! - - The name of the Secret must be named "bootstrap-token-(token-id)". - - You can read more about bootstrap tokens here: - https://kubernetes.io/docs/admin/bootstrap-tokens/ - `), - - // Without this callback, if a user runs just the "token" - // command without a subcommand, or with an invalid subcommand, - // cobra will print usage information, but still exit cleanly. - // We want to return an error code in these cases so that the - // user knows that their command was invalid. - RunE: cmdutil.SubCmdRunE("token"), - } - - options.AddKubeConfigFlag(tokenCmd.PersistentFlags(), &kubeConfigFile) - tokenCmd.PersistentFlags().BoolVar(&dryRun, - options.DryRun, dryRun, "Whether to enable dry-run mode or not") - - cfg := &kubeadmapiv1beta2.InitConfiguration{} - - // Default values for the cobra help text - kubeadmscheme.Scheme.Default(cfg) - - var cfgPath string - var printJoinCommand bool - var certificateKey string - bto := options.NewBootstrapTokenOptions() - - createCmd := &cobra.Command{ - Use: "create [token]", - DisableFlagsInUseLine: true, - Short: "Create bootstrap tokens on the server", - Long: dedent.Dedent(` - This command will create a bootstrap token for you. - You can specify the usages for this token, the "time to live" and an optional human friendly description. - - The [token] is the actual token to write. - This should be a securely generated random token of the form "[a-z0-9]{6}.[a-z0-9]{16}". - If no [token] is given, kubeadm will generate a random token instead. - `), - RunE: func(tokenCmd *cobra.Command, args []string) error { - if len(args) > 0 { - bto.TokenStr = args[0] - } - klog.V(1).Infoln("[token] validating mixed arguments") - if err := validation.ValidateMixedArguments(tokenCmd.Flags()); err != nil { - return err - } - - if err := bto.ApplyTo(cfg); err != nil { - return err - } - - klog.V(1).Infoln("[token] getting Clientsets from kubeconfig file") - kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile) - client, err := getClientset(kubeConfigFile, dryRun) - if err != nil { - return err - } - - return RunCreateToken(out, client, cfgPath, cfg, printJoinCommand, certificateKey, kubeConfigFile) - }, - } - - options.AddConfigFlag(createCmd.Flags(), &cfgPath) - createCmd.Flags().BoolVar(&printJoinCommand, - "print-join-command", false, "Instead of printing only the token, print the full 'kubeadm join' flag needed to join the cluster using the token.") - createCmd.Flags().StringVar(&certificateKey, - options.CertificateKey, "", "When used together with '--print-join-command', print the full 'kubeadm join' flag needed to join the cluster as a control-plane. To create a new certificate key you must use 'kubeadm init phase upload-certs --upload-certs'.") - bto.AddTTLFlagWithName(createCmd.Flags(), "ttl") - bto.AddUsagesFlag(createCmd.Flags()) - bto.AddGroupsFlag(createCmd.Flags()) - bto.AddDescriptionFlag(createCmd.Flags()) - - tokenCmd.AddCommand(createCmd) - tokenCmd.AddCommand(newCmdTokenGenerate(out)) - - outputFlags := output.NewOutputFlags(&tokenTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(output.TextOutput) - - listCmd := &cobra.Command{ - Use: "list", - Short: "List bootstrap tokens on the server", - Long: dedent.Dedent(` - This command will list all bootstrap tokens for you. - `), - RunE: func(tokenCmd *cobra.Command, args []string) error { - kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile) - client, err := getClientset(kubeConfigFile, dryRun) - if err != nil { - return err - } - - printer, err := outputFlags.ToPrinter() - if err != nil { - return err - } - - return RunListTokens(out, errW, client, printer) - }, - Args: cobra.NoArgs, - } - - outputFlags.AddFlags(listCmd) - - tokenCmd.AddCommand(listCmd) - - deleteCmd := &cobra.Command{ - Use: "delete [token-value] ...", - DisableFlagsInUseLine: true, - Short: "Delete bootstrap tokens on the server", - Long: dedent.Dedent(` - This command will delete a list of bootstrap tokens for you. - - The [token-value] is the full Token of the form "[a-z0-9]{6}.[a-z0-9]{16}" or the - Token ID of the form "[a-z0-9]{6}" to delete. - `), - RunE: func(tokenCmd *cobra.Command, args []string) error { - if len(args) < 1 { - return errors.Errorf("missing subcommand; 'token delete' is missing token of form %q", bootstrapapi.BootstrapTokenIDPattern) - } - kubeConfigFile = cmdutil.GetKubeConfigPath(kubeConfigFile) - client, err := getClientset(kubeConfigFile, dryRun) - if err != nil { - return err - } - - return RunDeleteTokens(out, client, args) - }, - } - tokenCmd.AddCommand(deleteCmd) - - return tokenCmd -} - -// newCmdTokenGenerate returns cobra.Command to generate new token -func newCmdTokenGenerate(out io.Writer) *cobra.Command { - return &cobra.Command{ - Use: "generate", - Short: "Generate and print a bootstrap token, but do not create it on the server", - Long: dedent.Dedent(` - This command will print out a randomly-generated bootstrap token that can be used with - the "init" and "join" commands. - - You don't have to use this command in order to generate a token. You can do so - yourself as long as it is in the format "[a-z0-9]{6}.[a-z0-9]{16}". This - command is provided for convenience to generate tokens in the given format. - - You can also use "kubeadm init" without specifying a token and it will - generate and print one for you. - `), - RunE: func(cmd *cobra.Command, args []string) error { - return RunGenerateToken(out) - }, - Args: cobra.NoArgs, - } -} - -// RunCreateToken generates a new bootstrap token and stores it as a secret on the server. -func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, initCfg *kubeadmapiv1beta2.InitConfiguration, printJoinCommand bool, certificateKey string, kubeConfigFile string) error { - // ClusterConfiguration is needed just for the call to LoadOrDefaultInitConfiguration - clusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{ - // KubernetesVersion is not used, but we set this explicitly to avoid - // the lookup of the version from the internet when executing LoadOrDefaultInitConfiguration - KubernetesVersion: kubeadmconstants.CurrentKubernetesVersion.String(), - } - kubeadmscheme.Scheme.Default(clusterCfg) - - // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags - klog.V(1).Infoln("[token] loading configurations") - - // In fact, we don't do any CRI ops at all. - // This is just to force skipping the CRI detection. - // Ref: https://github.com/kubernetes/kubeadm/issues/1559 - initCfg.NodeRegistration.CRISocket = kubeadmconstants.DefaultDockerCRISocket - - internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, initCfg, clusterCfg) - if err != nil { - return err - } - - klog.V(1).Infoln("[token] creating token") - if err := tokenphase.CreateNewTokens(client, internalcfg.BootstrapTokens); err != nil { - return err - } - - // if --print-join-command was specified, print a machine-readable full `kubeadm join` command - // otherwise, just print the token - if printJoinCommand { - skipTokenPrint := false - if certificateKey != "" { - skipCertificateKeyPrint := false - joinCommand, err := cmdutil.GetJoinControlPlaneCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), certificateKey, skipTokenPrint, skipCertificateKeyPrint) - if err != nil { - return errors.Wrap(err, "failed to get join command") - } - joinCommand = strings.ReplaceAll(joinCommand, "\\\n", "") - joinCommand = strings.ReplaceAll(joinCommand, "\t", "") - fmt.Fprintln(out, joinCommand) - } else { - joinCommand, err := cmdutil.GetJoinWorkerCommand(kubeConfigFile, internalcfg.BootstrapTokens[0].Token.String(), skipTokenPrint) - if err != nil { - return errors.Wrap(err, "failed to get join command") - } - joinCommand = strings.ReplaceAll(joinCommand, "\\\n", "") - joinCommand = strings.ReplaceAll(joinCommand, "\t", "") - fmt.Fprintln(out, joinCommand) - } - } else { - if certificateKey != "" { - return errors.Wrap(err, "cannot use --certificate-key without --print-join-command") - } - fmt.Fprintln(out, internalcfg.BootstrapTokens[0].Token.String()) - } - - return nil -} - -// RunGenerateToken just generates a random token for the user -func RunGenerateToken(out io.Writer) error { - klog.V(1).Infoln("[token] generating random token") - token, err := bootstraputil.GenerateBootstrapToken() - if err != nil { - return err - } - - fmt.Fprintln(out, token) - return nil -} - -func formatBootstrapToken(obj *outputapiv1alpha1.BootstrapToken) string { - ttl := "" - expires := "" - if obj.Expires != nil { - ttl = duration.ShortHumanDuration(time.Until(obj.Expires.Time)) - expires = obj.Expires.Format(time.RFC3339) - } - ttl = fmt.Sprintf("%-9s", ttl) - - usages := strings.Join(obj.Usages, ",") - if len(usages) == 0 { - usages = "" - } - usages = fmt.Sprintf("%-22s", usages) - - description := obj.Description - if len(description) == 0 { - description = "" - } - description = fmt.Sprintf("%-56s", description) - - groups := strings.Join(obj.Groups, ",") - if len(groups) == 0 { - groups = "" - } - - return fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%s\n", obj.Token, ttl, expires, usages, description, groups) -} - -// tokenTextPrinter prints bootstrap token in a text form -type tokenTextPrinter struct { - output.TextPrinter - columns []string - headerIsPrinted bool -} - -// PrintObj is an implementation of ResourcePrinter.PrintObj for plain text output -func (ttp *tokenTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { - tabw := tabwriter.NewWriter(writer, 10, 4, 3, ' ', 0) - - // Print header - if !ttp.headerIsPrinted { - fmt.Fprintln(tabw, strings.Join(ttp.columns, "\t")) - ttp.headerIsPrinted = true - } - - // Print token - fmt.Fprint(tabw, formatBootstrapToken(obj.(*outputapiv1alpha1.BootstrapToken))) - - return tabw.Flush() -} - -// tokenTextPrintFlags provides flags necessary for printing bootstrap token in a text form. -type tokenTextPrintFlags struct{} - -// ToPrinter returns kubeadm printer for the text output format -func (tpf *tokenTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { - if outputFormat == output.TextOutput { - return &tokenTextPrinter{columns: []string{"TOKEN", "TTL", "EXPIRES", "USAGES", "DESCRIPTION", "EXTRA GROUPS"}}, nil - } - return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.TextOutput}} -} - -// RunListTokens lists details on all existing bootstrap tokens on the server. -func RunListTokens(out io.Writer, errW io.Writer, client clientset.Interface, printer output.Printer) error { - // First, build our selector for bootstrap tokens only - klog.V(1).Infoln("[token] preparing selector for bootstrap token") - tokenSelector := fields.SelectorFromSet( - map[string]string{ - // TODO: We hard-code "type" here until `field_constants.go` that is - // currently in `pkg/apis/core/` exists in the external API, i.e. - // k8s.io/api/v1. Should be v1.SecretTypeField - "type": string(bootstrapapi.SecretTypeBootstrapToken), - }, - ) - listOptions := metav1.ListOptions{ - FieldSelector: tokenSelector.String(), - } - - klog.V(1).Info("[token] retrieving list of bootstrap tokens") - secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(context.TODO(), listOptions) - if err != nil { - return errors.Wrap(err, "failed to list bootstrap tokens") - } - - for _, secret := range secrets.Items { - // Get the BootstrapToken struct representation from the Secret object - token, err := kubeadmapi.BootstrapTokenFromSecret(&secret) - if err != nil { - fmt.Fprintf(errW, "%v", err) - continue - } - - // Convert token into versioned output structure - outputToken := outputapiv1alpha1.BootstrapToken{ - BootstrapToken: kubeadmapiv1beta2.BootstrapToken{ - Token: &kubeadmapiv1beta2.BootstrapTokenString{ID: token.Token.ID, Secret: token.Token.Secret}, - Description: token.Description, - TTL: token.TTL, - Expires: token.Expires, - Usages: token.Usages, - Groups: token.Groups, - }, - } - - if err := printer.PrintObj(&outputToken, out); err != nil { - return errors.Wrapf(err, "unable to print token %s", token.Token) - } - } - return nil -} - -// RunDeleteTokens removes a bootstrap tokens from the server. -func RunDeleteTokens(out io.Writer, client clientset.Interface, tokenIDsOrTokens []string) error { - for _, tokenIDOrToken := range tokenIDsOrTokens { - // Assume this is a token id and try to parse it - tokenID := tokenIDOrToken - klog.V(1).Info("[token] parsing token") - if !bootstraputil.IsValidBootstrapTokenID(tokenIDOrToken) { - // Okay, the full token with both id and secret was probably passed. Parse it and extract the ID only - bts, err := kubeadmapiv1beta2.NewBootstrapTokenString(tokenIDOrToken) - if err != nil { - return errors.Errorf("given token didn't match pattern %q or %q", - bootstrapapi.BootstrapTokenIDPattern, bootstrapapi.BootstrapTokenIDPattern) - } - tokenID = bts.ID - } - - tokenSecretName := bootstraputil.BootstrapTokenSecretName(tokenID) - klog.V(1).Infof("[token] deleting token %q", tokenID) - if err := client.CoreV1().Secrets(metav1.NamespaceSystem).Delete(context.TODO(), tokenSecretName, metav1.DeleteOptions{}); err != nil { - return errors.Wrapf(err, "failed to delete bootstrap token %q", tokenID) - } - fmt.Fprintf(out, "bootstrap token %q deleted\n", tokenID) - } - return nil -} - -func getClientset(file string, dryRun bool) (clientset.Interface, error) { - if dryRun { - dryRunGetter, err := apiclient.NewClientBackedDryRunGetterFromKubeconfig(file) - if err != nil { - return nil, err - } - return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil - } - return kubeconfigutil.ClientSetFromFile(file) -} diff --git a/cmd/kubeadm/app/cmd/token_test.go b/cmd/kubeadm/app/cmd/token_test.go deleted file mode 100644 index 1da30a8673ce8..0000000000000 --- a/cmd/kubeadm/app/cmd/token_test.go +++ /dev/null @@ -1,454 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "testing" - - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" - "k8s.io/client-go/tools/clientcmd" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" - outputapiv1alpha1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha1" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" -) - -const ( - tokenExpectedRegex = "^\\S{6}\\.\\S{16}\n$" - testConfigToken = `apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: - server: localhost:8000 - name: prod -contexts: -- context: - cluster: prod - namespace: default - user: default-service-account - name: default -current-context: default -kind: Config -preferences: {} -users: -- name: kubernetes-admin - user: - client-certificate-data: - client-key-data: -` -) - -func TestRunGenerateToken(t *testing.T) { - var buf bytes.Buffer - - err := RunGenerateToken(&buf) - if err != nil { - t.Errorf("RunGenerateToken returned an error: %v", err) - } - - output := buf.String() - - matched, err := regexp.MatchString(tokenExpectedRegex, output) - if err != nil { - t.Fatalf("Encountered an error while trying to match RunGenerateToken's output: %v", err) - } - if !matched { - t.Errorf("RunGenerateToken's output did not match expected regex; wanted: [%s], got: [%s]", tokenExpectedRegex, output) - } -} - -func TestRunCreateToken(t *testing.T) { - var buf bytes.Buffer - fakeClient := &fake.Clientset{} - fakeClient.AddReactor("get", "secrets", func(action core.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apierrors.NewNotFound(v1.Resource("secrets"), "foo") - }) - - testCases := []struct { - name string - token string - usages []string - extraGroups []string - printJoin bool - expectedError bool - }{ - { - name: "valid: empty token", - token: "", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:foo"}, - expectedError: false, - }, - { - name: "valid: non-empty token", - token: "abcdef.1234567890123456", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:foo"}, - expectedError: false, - }, - { - name: "valid: no extraGroups", - token: "abcdef.1234567890123456", - usages: []string{"signing", "authentication"}, - extraGroups: []string{}, - expectedError: false, - }, - { - name: "invalid: incorrect extraGroups", - token: "abcdef.1234567890123456", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"foo"}, - expectedError: true, - }, - { - name: "invalid: specifying --groups when --usages doesn't include authentication", - token: "abcdef.1234567890123456", - usages: []string{"signing"}, - extraGroups: []string{"foo"}, - expectedError: true, - }, - { - name: "invalid: partially incorrect usages", - token: "abcdef.1234567890123456", - usages: []string{"foo", "authentication"}, - extraGroups: []string{"system:bootstrappers:foo"}, - expectedError: true, - }, - { - name: "invalid: all incorrect usages", - token: "abcdef.1234567890123456", - usages: []string{"foo", "bar"}, - extraGroups: []string{"system:bootstrappers:foo"}, - expectedError: true, - }, - { - name: "invalid: print join command", - token: "", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:foo"}, - printJoin: true, - expectedError: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - bts, err := kubeadmapiv1beta2.NewBootstrapTokenString(tc.token) - if err != nil && len(tc.token) != 0 { // if tc.token is "" it's okay as it will be generated later at runtime - t.Fatalf("token couldn't be parsed for testing: %v", err) - } - - cfg := &kubeadmapiv1beta2.InitConfiguration{ - BootstrapTokens: []kubeadmapiv1beta2.BootstrapToken{ - { - Token: bts, - TTL: &metav1.Duration{Duration: 0}, - Usages: tc.usages, - Groups: tc.extraGroups, - }, - }, - } - - err = RunCreateToken(&buf, fakeClient, "", cfg, tc.printJoin, "", "") - if tc.expectedError && err == nil { - t.Error("unexpected success") - } else if !tc.expectedError && err != nil { - t.Errorf("unexpected error: %v", err) - } - }) - } -} - -func TestNewCmdTokenGenerate(t *testing.T) { - var buf bytes.Buffer - args := []string{} - - cmd := newCmdTokenGenerate(&buf) - cmd.SetArgs(args) - - if err := cmd.Execute(); err != nil { - t.Errorf("Cannot execute token command: %v", err) - } -} - -func TestNewCmdToken(t *testing.T) { - var buf, bufErr bytes.Buffer - testConfigTokenFile := "test-config-file" - - tmpDir, err := ioutil.TempDir("", "kubeadm-token-test") - if err != nil { - t.Errorf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - fullPath := filepath.Join(tmpDir, testConfigTokenFile) - - f, err := os.Create(fullPath) - if err != nil { - t.Errorf("Unable to create test file %q: %v", fullPath, err) - } - defer f.Close() - - testCases := []struct { - name string - args []string - configToWrite string - kubeConfigEnv string - expectedError bool - }{ - { - name: "valid: generate", - args: []string{"generate"}, - configToWrite: "", - expectedError: false, - }, - { - name: "valid: delete from --kubeconfig", - args: []string{"delete", "abcdef.1234567890123456", "--dry-run", "--kubeconfig=" + fullPath}, - configToWrite: testConfigToken, - expectedError: false, - }, - { - name: "valid: delete from " + clientcmd.RecommendedConfigPathEnvVar, - args: []string{"delete", "abcdef.1234567890123456", "--dry-run"}, - configToWrite: testConfigToken, - kubeConfigEnv: fullPath, - expectedError: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // the command is created for each test so that the kubeConfigFile - // variable in newCmdToken() is reset. - cmd := newCmdToken(&buf, &bufErr) - if _, err = f.WriteString(tc.configToWrite); err != nil { - t.Errorf("Unable to write test file %q: %v", fullPath, err) - } - // store the current value of the environment variable. - storedEnv := os.Getenv(clientcmd.RecommendedConfigPathEnvVar) - if tc.kubeConfigEnv != "" { - os.Setenv(clientcmd.RecommendedConfigPathEnvVar, tc.kubeConfigEnv) - } - cmd.SetArgs(tc.args) - err := cmd.Execute() - if (err != nil) != tc.expectedError { - t.Errorf("Test case %q: newCmdToken expected error: %v, saw: %v", tc.name, tc.expectedError, (err != nil)) - } - // restore the environment variable. - os.Setenv(clientcmd.RecommendedConfigPathEnvVar, storedEnv) - }) - } -} - -func TestGetClientset(t *testing.T) { - testConfigTokenFile := "test-config-file" - - tmpDir, err := ioutil.TempDir("", "kubeadm-token-test") - if err != nil { - t.Errorf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - fullPath := filepath.Join(tmpDir, testConfigTokenFile) - - // test dryRun = false on a non-exisiting file - if _, err = getClientset(fullPath, false); err == nil { - t.Errorf("getClientset(); dry-run: false; did no fail for test file %q: %v", fullPath, err) - } - - // test dryRun = true on a non-exisiting file - if _, err = getClientset(fullPath, true); err == nil { - t.Errorf("getClientset(); dry-run: true; did no fail for test file %q: %v", fullPath, err) - } - - f, err := os.Create(fullPath) - if err != nil { - t.Errorf("Unable to create test file %q: %v", fullPath, err) - } - defer f.Close() - - if _, err = f.WriteString(testConfigToken); err != nil { - t.Errorf("Unable to write test file %q: %v", fullPath, err) - } - - // test dryRun = true on an exisiting file - if _, err = getClientset(fullPath, true); err != nil { - t.Errorf("getClientset(); dry-run: true; failed for test file %q: %v", fullPath, err) - } -} - -func TestRunDeleteTokens(t *testing.T) { - var buf bytes.Buffer - - tmpDir, err := ioutil.TempDir("", "kubeadm-token-test") - if err != nil { - t.Errorf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(tmpDir) - fullPath := filepath.Join(tmpDir, "test-config-file") - - f, err := os.Create(fullPath) - if err != nil { - t.Errorf("Unable to create test file %q: %v", fullPath, err) - } - defer f.Close() - - if _, err = f.WriteString(testConfigToken); err != nil { - t.Errorf("Unable to write test file %q: %v", fullPath, err) - } - - client, err := getClientset(fullPath, true) - if err != nil { - t.Errorf("Unable to run getClientset() for test file %q: %v", fullPath, err) - } - - // test valid; should not fail - // for some reason Secrets().Delete() does not fail even for this dummy config - if err = RunDeleteTokens(&buf, client, []string{"abcdef.1234567890123456", "abcdef.2345678901234567"}); err != nil { - t.Errorf("RunDeleteToken() failed for a valid token: %v", err) - } - - // test invalid token; should fail - if err = RunDeleteTokens(&buf, client, []string{"invalid-token"}); err == nil { - t.Errorf("RunDeleteToken() succeeded for an invalid token: %v", err) - } -} - -func TestTokenOutput(t *testing.T) { - testCases := []struct { - name string - id string - secret string - description string - usages []string - extraGroups []string - outputFormat string - expected string - }{ - { - name: "JSON output", - id: "abcdef", - secret: "1234567890123456", - description: "valid bootstrap tooken", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:kubeadm:default-node-token"}, - outputFormat: "json", - expected: `{ - "kind": "BootstrapToken", - "apiVersion": "output.kubeadm.k8s.io/v1alpha1", - "token": "abcdef.1234567890123456", - "description": "valid bootstrap tooken", - "usages": [ - "signing", - "authentication" - ], - "groups": [ - "system:bootstrappers:kubeadm:default-node-token" - ] -} -`, - }, - { - name: "YAML output", - id: "abcdef", - secret: "1234567890123456", - description: "valid bootstrap tooken", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:kubeadm:default-node-token"}, - outputFormat: "yaml", - expected: `apiVersion: output.kubeadm.k8s.io/v1alpha1 -description: valid bootstrap tooken -groups: -- system:bootstrappers:kubeadm:default-node-token -kind: BootstrapToken -token: abcdef.1234567890123456 -usages: -- signing -- authentication -`, - }, - { - name: "Go template output", - id: "abcdef", - secret: "1234567890123456", - description: "valid bootstrap tooken", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:kubeadm:default-node-token"}, - outputFormat: "go-template={{println .token .description .usages .groups}}", - expected: `abcdef.1234567890123456 valid bootstrap tooken [signing authentication] [system:bootstrappers:kubeadm:default-node-token] -`, - }, - { - name: "text output", - id: "abcdef", - secret: "1234567890123456", - description: "valid bootstrap tooken", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:kubeadm:default-node-token"}, - outputFormat: "text", - expected: `TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS -abcdef.1234567890123456 signing,authentication valid bootstrap tooken system:bootstrappers:kubeadm:default-node-token -`, - }, - { - name: "jsonpath output", - id: "abcdef", - secret: "1234567890123456", - description: "valid bootstrap tooken", - usages: []string{"signing", "authentication"}, - extraGroups: []string{"system:bootstrappers:kubeadm:default-node-token"}, - outputFormat: "jsonpath={.token} {.groups}", - expected: "abcdef.1234567890123456 [\"system:bootstrappers:kubeadm:default-node-token\"]", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - token := outputapiv1alpha1.BootstrapToken{ - BootstrapToken: kubeadmapiv1beta2.BootstrapToken{ - Token: &kubeadmapiv1beta2.BootstrapTokenString{ID: tc.id, Secret: tc.secret}, - Description: tc.description, - Usages: tc.usages, - Groups: tc.extraGroups, - }, - } - buf := bytes.Buffer{} - outputFlags := output.NewOutputFlags(&tokenTextPrintFlags{}).WithTypeSetter(outputapischeme.Scheme).WithDefaultOutput(tc.outputFormat) - printer, err := outputFlags.ToPrinter() - if err != nil { - t.Errorf("can't create printer for output format %s: %+v", tc.outputFormat, err) - } - - if err := printer.PrintObj(&token, &buf); err != nil { - t.Errorf("unable to print token %s: %+v", token.Token, err) - } - - actual := buf.String() - if actual != tc.expected { - t.Errorf( - "failed TestTokenOutput:\n\nexpected:\n%s\n\nactual:\n%s", tc.expected, actual) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/upgrade/BUILD b/cmd/kubeadm/app/cmd/upgrade/BUILD deleted file mode 100644 index 4a8d2d1d14a89..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/BUILD +++ /dev/null @@ -1,81 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "apply.go", - "common.go", - "diff.go", - "node.go", - "plan.go", - "upgrade.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/upgrade", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", - "//cmd/kubeadm/app/apis/output:go_default_library", - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/cmd/phases/upgrade/node:go_default_library", - "//cmd/kubeadm/app/cmd/phases/workflow:go_default_library", - "//cmd/kubeadm/app/cmd/util:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/phases/controlplane:go_default_library", - "//cmd/kubeadm/app/phases/upgrade:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//cmd/kubeadm/app/util/dryrun:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/client-go/discovery/fake:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "apply_test.go", - "common_test.go", - "diff_test.go", - "plan_test.go", - ], - data = glob(["testdata/**"]), - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/upgrade:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go deleted file mode 100644 index 89767d8a43861..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ /dev/null @@ -1,230 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "time" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - utilsexec "k8s.io/utils/exec" -) - -const ( - defaultImagePullTimeout = 15 * time.Minute -) - -// applyFlags holds the information about the flags that can be passed to apply -type applyFlags struct { - *applyPlanFlags - - nonInteractiveMode bool - force bool - dryRun bool - etcdUpgrade bool - renewCerts bool - imagePullTimeout time.Duration - patchesDir string -} - -// sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run) -func (f *applyFlags) sessionIsInteractive() bool { - return !(f.nonInteractiveMode || f.dryRun || f.force) -} - -// newCmdApply returns the cobra command for `kubeadm upgrade apply` -func newCmdApply(apf *applyPlanFlags) *cobra.Command { - flags := &applyFlags{ - applyPlanFlags: apf, - imagePullTimeout: defaultImagePullTimeout, - etcdUpgrade: true, - renewCerts: true, - } - - cmd := &cobra.Command{ - Use: "apply [version]", - DisableFlagsInUseLine: true, - Short: "Upgrade your Kubernetes cluster to the specified version", - RunE: func(cmd *cobra.Command, args []string) error { - return runApply(flags, args) - }, - } - - // Register the common flags for apply and plan - addApplyPlanFlags(cmd.Flags(), flags.applyPlanFlags) - // Specify the valid flags specific for apply - cmd.Flags().BoolVarP(&flags.nonInteractiveMode, "yes", "y", flags.nonInteractiveMode, "Perform the upgrade and do not prompt for confirmation (non-interactive mode).") - cmd.Flags().BoolVarP(&flags.force, "force", "f", flags.force, "Force upgrading although some requirements might not be met. This also implies non-interactive mode.") - cmd.Flags().BoolVar(&flags.dryRun, options.DryRun, flags.dryRun, "Do not change any state, just output what actions would be performed.") - cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.") - cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") - cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.") - // TODO: The flag was deprecated in 1.19; remove the flag following a GA deprecation policy of 12 months or 2 releases (whichever is longer) - cmd.Flags().MarkDeprecated("image-pull-timeout", "This flag is deprecated and will be removed in a future version.") - options.AddPatchesFlag(cmd.Flags(), &flags.patchesDir) - - return cmd -} - -// runApply takes care of the actual upgrade functionality -// It does the following things: -// - Checks if the cluster is healthy -// - Gets the configuration from the kubeadm-config ConfigMap in the cluster -// - Enforces all version skew policies -// - Asks the user if they really want to upgrade -// - Makes sure the control plane images are available locally on the control-plane(s) -// - Upgrades the control plane components -// - Applies the other resources that'd be created with kubeadm init as well, like -// - Creating the RBAC rules for the bootstrap tokens and the cluster-info ConfigMap -// - Applying new kube-dns and kube-proxy manifests -// - Uploads the newly used configuration to the cluster ConfigMap -func runApply(flags *applyFlags, args []string) error { - - // Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap) - klog.V(1).Infoln("[upgrade/apply] verifying health of cluster") - klog.V(1).Infoln("[upgrade/apply] retrieving configuration from cluster") - client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, flags.dryRun, true) - if err != nil { - return err - } - - // Validate requested and validate actual version - klog.V(1).Infoln("[upgrade/apply] validating requested and actual version") - if err := configutil.NormalizeKubernetesVersion(&cfg.ClusterConfiguration); err != nil { - return err - } - - // Use normalized version string in all following code. - newK8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) - if err != nil { - return errors.Errorf("unable to parse normalized version %q as a semantic version", cfg.KubernetesVersion) - } - - if err := features.ValidateVersion(features.InitFeatureGates, cfg.FeatureGates, cfg.KubernetesVersion); err != nil { - return err - } - - // Enforce the version skew policies - klog.V(1).Infoln("[upgrade/version] enforcing version skew policies") - if err := EnforceVersionPolicies(cfg.KubernetesVersion, newK8sVersion, flags, versionGetter); err != nil { - return errors.Wrap(err, "[upgrade/version] FATAL") - } - - // If the current session is interactive, ask the user whether they really want to upgrade. - if flags.sessionIsInteractive() { - if err := InteractivelyConfirmUpgrade("Are you sure you want to proceed with the upgrade?"); err != nil { - return err - } - } - - if !flags.dryRun { - fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster") - fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection") - fmt.Println("[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'") - if err := preflight.RunPullImagesCheck(utilsexec.New(), cfg, sets.NewString(cfg.NodeRegistration.IgnorePreflightErrors...)); err != nil { - return err - } - } else { - fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')") - } - - waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout) - - // Now; perform the upgrade procedure - klog.V(1).Infoln("[upgrade/apply] performing upgrade") - if err := PerformControlPlaneUpgrade(flags, client, waiter, cfg); err != nil { - return errors.Wrap(err, "[upgrade/apply] FATAL") - } - - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - fmt.Printf("[upgrade/postupgrade] Applying label %s='' to Nodes with label %s='' (deprecated)\n", - kubeadmconstants.LabelNodeRoleControlPlane, kubeadmconstants.LabelNodeRoleOldControlPlane) - if err := upgrade.LabelOldControlPlaneNodes(client); err != nil { - return err - } - - // Upgrade RBAC rules and addons. - klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons") - if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.dryRun); err != nil { - return errors.Wrap(err, "[upgrade/postupgrade] FATAL post-upgrade error") - } - - if flags.dryRun { - fmt.Println("[dryrun] Finished dryrunning successfully!") - return nil - } - - fmt.Println("") - fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", cfg.KubernetesVersion) - fmt.Println("") - fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.") - - return nil -} - -// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to -// There are both fatal and skippable (with --force) errors -func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, flags *applyFlags, versionGetter upgrade.VersionGetter) error { - fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr) - - versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, flags.allowExperimentalUpgrades, flags.allowRCUpgrades) - if versionSkewErrs != nil { - - if len(versionSkewErrs.Mandatory) > 0 { - return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again", - kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory)) - } - - if len(versionSkewErrs.Skippable) > 0 { - // Return the error if the user hasn't specified the --force flag - if !flags.force { - return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag", - kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable)) - } - // Soft errors found, but --force was specified - fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable)) - } - } - return nil -} - -// PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted) -func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error { - - // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster - fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", internalcfg.KubernetesVersion) - - if flags.dryRun { - return upgrade.DryRunStaticPodUpgrade(flags.patchesDir, internalcfg) - } - - return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts, flags.patchesDir) -} diff --git a/cmd/kubeadm/app/cmd/upgrade/apply_test.go b/cmd/kubeadm/app/cmd/upgrade/apply_test.go deleted file mode 100644 index 5a19eaf8585f0..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/apply_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "testing" -) - -func TestSessionIsInteractive(t *testing.T) { - var tcases = []struct { - name string - flags *applyFlags - expected bool - }{ - { - name: "Explicitly non-interactive", - flags: &applyFlags{ - nonInteractiveMode: true, - }, - expected: false, - }, - { - name: "Implicitly non-interactive since --dryRun is used", - flags: &applyFlags{ - dryRun: true, - }, - expected: false, - }, - { - name: "Implicitly non-interactive since --force is used", - flags: &applyFlags{ - force: true, - }, - expected: false, - }, - { - name: "Interactive session", - flags: &applyFlags{}, - expected: true, - }, - } - for _, tt := range tcases { - t.Run(tt.name, func(t *testing.T) { - if tt.flags.sessionIsInteractive() != tt.expected { - t.Error("unexpected result") - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go deleted file mode 100644 index 71b2fab6bccf2..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ /dev/null @@ -1,315 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - "time" - - "github.com/pkg/errors" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - fakediscovery "k8s.io/client-go/discovery/fake" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -// isKubeadmConfigPresent checks if a kubeadm config type is found in the provided document map -func isKubeadmConfigPresent(docmap kubeadmapi.DocumentMap) bool { - for gvk := range docmap { - if gvk.Group == kubeadmapi.GroupName { - return true - } - } - return false -} - -// loadConfig loads configuration from a file and/or the cluster. InitConfiguration, ClusterConfiguration and (optionally) component configs -// are loaded. This function allows the component configs to be loaded from a file that contains only them. If the file contains any kubeadm types -// in it (API group "kubeadm.kubernetes.io" present), then the supplied file is treaded as a legacy reconfiguration style "--config" use and the -// returned bool value is set to true (the only case to be done so). -func loadConfig(cfgPath string, client clientset.Interface, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, bool, error) { - // Used for info logs here - const logPrefix = "upgrade/config" - - // The usual case here is to not have a config file, but rather load the config from the cluster. - // This is probably 90% of the time. So we handle it first. - if cfgPath == "" { - cfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, logPrefix, false, skipComponentConfigs) - return cfg, false, err - } - - // Otherwise, we have a config file. Let's load it. - configBytes, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, false, errors.Wrapf(err, "unable to load config from file %q", cfgPath) - } - - // Split the YAML documents in the file into a DocumentMap - docmap, err := kubeadmutil.SplitYAMLDocuments(configBytes) - if err != nil { - return nil, false, err - } - - // If there are kubeadm types (API group kubeadm.kubernetes.io) present, we need to keep the existing behavior - // here. Basically, we have to load all of the configs from the file and none from the cluster. Configs that are - // missing from the file will be automatically regenerated by kubeadm even if they are present in the cluster. - // The resulting configs overwrite the existing cluster ones at the end of a successful upgrade apply operation. - if isKubeadmConfigPresent(docmap) { - klog.Warning("WARNING: Usage of the --config flag with kubeadm config types for reconfiguring the cluster during upgrade is not recommended!") - cfg, err := configutil.BytesToInitConfiguration(configBytes) - return cfg, true, err - } - - // If no kubeadm config types are present, we assume that there are manually upgraded component configs in the file. - // Hence, we load the kubeadm types from the cluster. - initCfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, logPrefix, false, true) - if err != nil { - return nil, false, err - } - - // Stop here if the caller does not want us to load the component configs - if !skipComponentConfigs { - // Load the component configs with upgrades - if err := componentconfigs.FetchFromClusterWithLocalOverwrites(&initCfg.ClusterConfiguration, client, docmap); err != nil { - return nil, false, err - } - - // Now default and validate the configs - componentconfigs.Default(&initCfg.ClusterConfiguration, &initCfg.LocalAPIEndpoint, &initCfg.NodeRegistration) - if errs := componentconfigs.Validate(&initCfg.ClusterConfiguration); len(errs) != 0 { - return nil, false, errs.ToAggregate() - } - } - - return initCfg, false, nil -} - -// enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure -func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgradeApply bool) (clientset.Interface, upgrade.VersionGetter, *kubeadmapi.InitConfiguration, error) { - client, err := getClient(flags.kubeConfigPath, dryRun) - if err != nil { - return nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) - } - - // Check if the cluster is self-hosted - if upgrade.IsControlPlaneSelfHosted(client) { - return nil, nil, nil, errors.New("cannot upgrade a self-hosted control plane") - } - - // Fetch the configuration from a file or ConfigMap and validate it - fmt.Println("[upgrade/config] Making sure the configuration is correct:") - - var newK8sVersion string - cfg, legacyReconfigure, err := loadConfig(flags.cfgPath, client, !upgradeApply) - if err != nil { - if apierrors.IsNotFound(err) { - fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.") - fmt.Println("") - fmt.Println("[upgrade/config] Next steps:") - fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your control-plane.\n") - fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your control-plane.\n") - fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n") - fmt.Println("") - err = errors.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - } - return nil, nil, nil, errors.Wrap(err, "[upgrade/config] FATAL") - } else if legacyReconfigure { - // Set the newK8sVersion to the value in the ClusterConfiguration. This is done, so that users who use the --config option - // to supply a new ClusterConfiguration don't have to specify the Kubernetes version twice, - // if they don't want to upgrade but just change a setting. - newK8sVersion = cfg.KubernetesVersion - } - - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(flags.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) - if err != nil { - return nil, nil, nil, err - } - // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: - cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() - - // Ensure the user is root - klog.V(1).Info("running preflight checks") - if err := runPreflightChecks(client, ignorePreflightErrorsSet, &cfg.ClusterConfiguration); err != nil { - return nil, nil, nil, err - } - - // Run healthchecks against the cluster - if err := upgrade.CheckClusterHealth(client, &cfg.ClusterConfiguration, ignorePreflightErrorsSet); err != nil { - return nil, nil, nil, errors.Wrap(err, "[upgrade/health] FATAL") - } - - // The version arg is mandatory, during upgrade apply, unless it's specified in the config file - if upgradeApply && newK8sVersion == "" { - if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil { - return nil, nil, nil, err - } - } - - // If option was specified in both args and config file, args will overwrite the config file. - if len(args) == 1 { - newK8sVersion = args[0] - if upgradeApply { - // The `upgrade apply` version always overwrites the KubernetesVersion in the returned cfg with the target - // version. While this is not the same for `upgrade plan` where the KubernetesVersion should be the old - // one (because the call to getComponentConfigVersionStates requires the currently installed version). - // This also makes the KubernetesVersion value returned for `upgrade plan` consistent as that command - // allows to not specify a target version in which case KubernetesVersion will always hold the currently - // installed one. - cfg.KubernetesVersion = newK8sVersion - } - } - - // If features gates are passed to the command line, use it (otherwise use featureGates from configuration) - if flags.featureGatesString != "" { - cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, flags.featureGatesString) - if err != nil { - return nil, nil, nil, errors.Wrap(err, "[upgrade/config] FATAL") - } - } - - // Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm - if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, cfg.FeatureGates); len(msg) > 0 { - for _, m := range msg { - fmt.Printf("[upgrade/config] %s\n", m) - } - return nil, nil, nil, errors.New("[upgrade/config] FATAL. Unable to upgrade a cluster using deprecated feature-gate flags. Please see the release notes") - } - - // If the user told us to print this information out; do it! - if flags.printConfig { - printConfiguration(&cfg.ClusterConfiguration, os.Stdout) - } - - // Use a real version getter interface that queries the API server, the kubeadm client and the Kubernetes CI system for latest versions - return client, upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(client), newK8sVersion), cfg, nil -} - -// printConfiguration prints the external version of the API to yaml -func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer) { - // Short-circuit if cfg is nil, so we can safely get the value of the pointer below - if clustercfg == nil { - return - } - - cfgYaml, err := configutil.MarshalKubeadmConfigObject(clustercfg) - if err == nil { - fmt.Fprintln(w, "[upgrade/config] Configuration used:") - - scanner := bufio.NewScanner(bytes.NewReader(cfgYaml)) - for scanner.Scan() { - fmt.Fprintf(w, "\t%s\n", scanner.Text()) - } - } -} - -// runPreflightChecks runs the root preflight check -func runPreflightChecks(client clientset.Interface, ignorePreflightErrors sets.String, cfg *kubeadmapi.ClusterConfiguration) error { - fmt.Println("[preflight] Running pre-flight checks.") - err := preflight.RunRootCheckOnly(ignorePreflightErrors) - if err != nil { - return err - } - err = upgrade.RunCoreDNSMigrationCheck(client, ignorePreflightErrors, cfg.DNS.Type) - if err != nil { - return err - } - return nil -} - -// getClient gets a real or fake client depending on whether the user is dry-running or not -func getClient(file string, dryRun bool) (clientset.Interface, error) { - if dryRun { - dryRunGetter, err := apiclient.NewClientBackedDryRunGetterFromKubeconfig(file) - if err != nil { - return nil, err - } - - // In order for fakeclient.Discovery().ServerVersion() to return the backing API Server's - // real version; we have to do some clever API machinery tricks. First, we get the real - // API Server's version - realServerVersion, err := dryRunGetter.Client().Discovery().ServerVersion() - if err != nil { - return nil, errors.Wrap(err, "failed to get server version") - } - - // Get the fake clientset - dryRunOpts := apiclient.GetDefaultDryRunClientOptions(dryRunGetter, os.Stdout) - // Print GET and LIST requests - dryRunOpts.PrintGETAndLIST = true - fakeclient := apiclient.NewDryRunClientWithOpts(dryRunOpts) - // As we know the return of Discovery() of the fake clientset is of type *fakediscovery.FakeDiscovery - // we can convert it to that struct. - fakeclientDiscovery, ok := fakeclient.Discovery().(*fakediscovery.FakeDiscovery) - if !ok { - return nil, errors.New("couldn't set fake discovery's server version") - } - // Lastly, set the right server version to be used - fakeclientDiscovery.FakedServerVersion = realServerVersion - // return the fake clientset used for dry-running - return fakeclient, nil - } - return kubeconfigutil.ClientSetFromFile(file) -} - -// getWaiter gets the right waiter implementation -func getWaiter(dryRun bool, client clientset.Interface, timeout time.Duration) apiclient.Waiter { - if dryRun { - return dryrunutil.NewWaiter() - } - return apiclient.NewKubeWaiter(client, timeout, os.Stdout) -} - -// InteractivelyConfirmUpgrade asks the user whether they _really_ want to upgrade. -func InteractivelyConfirmUpgrade(question string) error { - - fmt.Printf("[upgrade/confirm] %s [y/N]: ", question) - - scanner := bufio.NewScanner(os.Stdin) - scanner.Scan() - if err := scanner.Err(); err != nil { - return errors.Wrap(err, "couldn't read from standard input") - } - answer := scanner.Text() - if strings.ToLower(answer) == "y" || strings.ToLower(answer) == "yes" { - return nil - } - - return errors.New("won't proceed; the user didn't answer (Y|y) in order to continue") -} diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go deleted file mode 100644 index 76fbfedcff07e..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ /dev/null @@ -1,158 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "bytes" - "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestEnforceRequirements(t *testing.T) { - tcases := []struct { - name string - newK8sVersion string - dryRun bool - flags applyPlanFlags - expectedErr bool - }{ - { - name: "Fail pre-flight check", - expectedErr: true, - }, - { - name: "Bogus preflight check disabled when also 'all' is specified", - flags: applyPlanFlags{ - ignorePreflightErrors: []string{"bogusvalue", "all"}, - }, - expectedErr: true, - }, - { - name: "Fail to create client", - flags: applyPlanFlags{ - ignorePreflightErrors: []string{"all"}, - }, - expectedErr: true, - }, - } - for _, tt := range tcases { - t.Run(tt.name, func(t *testing.T) { - _, _, _, err := enforceRequirements(&tt.flags, nil, tt.dryRun, false) - - if err == nil && tt.expectedErr { - t.Error("Expected error, but got success") - } - if err != nil && !tt.expectedErr { - t.Errorf("Unexpected error: %+v", err) - } - }) - } -} - -func TestPrintConfiguration(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - buf *bytes.Buffer - expectedBytes []byte - }{ - { - name: "config is nil", - cfg: nil, - expectedBytes: []byte(""), - }, - { - name: "cluster config with local Etcd", - cfg: &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.7.1", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - DataDir: "/some/path", - }, - }, - DNS: kubeadmapi.DNS{ - Type: kubeadmapi.CoreDNS, - }, - }, - expectedBytes: []byte(`[upgrade/config] Configuration used: - apiServer: {} - apiVersion: kubeadm.k8s.io/v1beta2 - controllerManager: {} - dns: - type: CoreDNS - etcd: - local: - dataDir: /some/path - kind: ClusterConfiguration - kubernetesVersion: v1.7.1 - networking: {} - scheduler: {} -`), - }, - { - name: "cluster config with ServiceSubnet and external Etcd", - cfg: &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.7.1", - Networking: kubeadmapi.Networking{ - ServiceSubnet: "10.96.0.1/12", - }, - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"https://one-etcd-instance:2379"}, - }, - }, - DNS: kubeadmapi.DNS{ - Type: kubeadmapi.CoreDNS, - }, - }, - expectedBytes: []byte(`[upgrade/config] Configuration used: - apiServer: {} - apiVersion: kubeadm.k8s.io/v1beta2 - controllerManager: {} - dns: - type: CoreDNS - etcd: - external: - caFile: "" - certFile: "" - endpoints: - - https://one-etcd-instance:2379 - keyFile: "" - kind: ClusterConfiguration - kubernetesVersion: v1.7.1 - networking: - serviceSubnet: 10.96.0.1/12 - scheduler: {} -`), - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - rt.buf = bytes.NewBufferString("") - printConfiguration(rt.cfg, rt.buf) - actualBytes := rt.buf.Bytes() - if !bytes.Equal(actualBytes, rt.expectedBytes) { - t.Errorf( - "failed PrintConfiguration:\n\texpected: %q\n\t actual: %q", - string(rt.expectedBytes), - string(actualBytes), - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/upgrade/diff.go b/cmd/kubeadm/app/cmd/upgrade/diff.go deleted file mode 100644 index 500d176f5a9db..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/diff.go +++ /dev/null @@ -1,163 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "io" - "io/ioutil" - - "github.com/pkg/errors" - "github.com/pmezard/go-difflib/difflib" - "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/version" - client "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -type diffFlags struct { - apiServerManifestPath string - controllerManagerManifestPath string - schedulerManifestPath string - newK8sVersionStr string - contextLines int - kubeConfigPath string - cfgPath string - out io.Writer -} - -var ( - defaultAPIServerManifestPath = constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory()) - defaultControllerManagerManifestPath = constants.GetStaticPodFilepath(constants.KubeControllerManager, constants.GetStaticPodDirectory()) - defaultSchedulerManifestPath = constants.GetStaticPodFilepath(constants.KubeScheduler, constants.GetStaticPodDirectory()) -) - -// newCmdDiff returns the cobra command for `kubeadm upgrade diff` -func newCmdDiff(out io.Writer) *cobra.Command { - flags := &diffFlags{ - kubeConfigPath: constants.GetAdminKubeConfigPath(), - out: out, - } - - cmd := &cobra.Command{ - Use: "diff [version]", - Short: "Show what differences would be applied to existing static pod manifests. See also: kubeadm upgrade apply --dry-run", - RunE: func(cmd *cobra.Command, args []string) error { - // TODO: Run preflight checks for diff to check that the manifests already exist. - return runDiff(flags, args) - }, - } - - options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath) - options.AddConfigFlag(cmd.Flags(), &flags.cfgPath) - cmd.Flags().StringVar(&flags.apiServerManifestPath, "api-server-manifest", defaultAPIServerManifestPath, "path to API server manifest") - cmd.Flags().StringVar(&flags.controllerManagerManifestPath, "controller-manager-manifest", defaultControllerManagerManifestPath, "path to controller manifest") - cmd.Flags().StringVar(&flags.schedulerManifestPath, "scheduler-manifest", defaultSchedulerManifestPath, "path to scheduler manifest") - cmd.Flags().IntVarP(&flags.contextLines, "context-lines", "c", 3, "How many lines of context in the diff") - - return cmd -} - -func runDiff(flags *diffFlags, args []string) error { - var err error - var cfg *kubeadmapi.InitConfiguration - if flags.cfgPath != "" { - cfg, err = configutil.LoadInitConfigurationFromFile(flags.cfgPath) - } else { - var client *client.Clientset - client, err = kubeconfigutil.ClientSetFromFile(flags.kubeConfigPath) - if err != nil { - return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) - } - cfg, err = configutil.FetchInitConfigurationFromCluster(client, flags.out, "upgrade/diff", false, false) - } - if err != nil { - return err - } - - // If the version is specified in config file, pick up that value. - if cfg.KubernetesVersion != "" { - flags.newK8sVersionStr = cfg.KubernetesVersion - } - - // If the new version is already specified in config file, version arg is optional. - if flags.newK8sVersionStr == "" { - if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil { - return err - } - } - - // If option was specified in both args and config file, args will overwrite the config file. - if len(args) == 1 { - flags.newK8sVersionStr = args[0] - } - - _, err = version.ParseSemantic(flags.newK8sVersionStr) - if err != nil { - return err - } - - cfg.ClusterConfiguration.KubernetesVersion = flags.newK8sVersionStr - - specs := controlplane.GetStaticPodSpecs(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint) - for spec, pod := range specs { - var path string - switch spec { - case constants.KubeAPIServer: - path = flags.apiServerManifestPath - case constants.KubeControllerManager: - path = flags.controllerManagerManifestPath - case constants.KubeScheduler: - path = flags.schedulerManifestPath - default: - klog.Errorf("[diff] unknown spec %v", spec) - continue - } - - newManifest, err := kubeadmutil.MarshalToYaml(&pod, corev1.SchemeGroupVersion) - if err != nil { - return err - } - if path == "" { - return errors.New("empty manifest path") - } - existingManifest, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - // Populated and write out the diff - diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(string(existingManifest)), - B: difflib.SplitLines(string(newManifest)), - FromFile: path, - ToFile: "new manifest", - Context: flags.contextLines, - } - - difflib.WriteUnifiedDiff(flags.out, diff) - } - return nil -} diff --git a/cmd/kubeadm/app/cmd/upgrade/diff_test.go b/cmd/kubeadm/app/cmd/upgrade/diff_test.go deleted file mode 100644 index 897d6b589f32f..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/diff_test.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func createTestRunDiffFile(contents []byte) (string, error) { - file, err := ioutil.TempFile("", "kubeadm-upgrade-diff-config-*.yaml") - if err != nil { - return "", errors.Wrap(err, "failed to create temporary test file") - } - if _, err := file.Write([]byte(contents)); err != nil { - return "", errors.Wrap(err, "failed to write to temporary test file") - } - if err := file.Close(); err != nil { - return "", errors.Wrap(err, "failed to close temporary test file") - } - return file.Name(), nil -} - -func TestRunDiff(t *testing.T) { - currentVersion := "v" + constants.CurrentKubernetesVersion.String() - - // create a temporary file with valid ClusterConfiguration - testUpgradeDiffConfigContents := []byte("apiVersion: kubeadm.k8s.io/v1beta2\n" + - "kind: ClusterConfiguration\n" + - "kubernetesVersion: " + currentVersion) - testUpgradeDiffConfig, err := createTestRunDiffFile(testUpgradeDiffConfigContents) - if err != nil { - t.Fatal(err) - } - defer os.Remove(testUpgradeDiffConfig) - - // create a temporary manifest file with dummy contents - testUpgradeDiffManifestContents := []byte("some-contents") - testUpgradeDiffManifest, err := createTestRunDiffFile(testUpgradeDiffManifestContents) - if err != nil { - t.Fatal(err) - } - defer os.Remove(testUpgradeDiffManifest) - - flags := &diffFlags{ - cfgPath: "", - out: ioutil.Discard, - } - - // TODO: Add test cases for empty cfgPath, it should automatically fetch cfg from cluster - testCases := []struct { - name string - args []string - setManifestPath bool - manifestPath string - cfgPath string - expectedError bool - }{ - { - name: "valid: run diff on valid manifest path", - cfgPath: testUpgradeDiffConfig, - setManifestPath: true, - manifestPath: testUpgradeDiffManifest, - expectedError: false, - }, - { - name: "invalid: missing config file", - cfgPath: "missing-path-to-a-config", - expectedError: true, - }, - { - name: "invalid: valid config but empty manifest path", - cfgPath: testUpgradeDiffConfig, - setManifestPath: true, - manifestPath: "", - expectedError: true, - }, - { - name: "invalid: valid config but bad manifest path", - cfgPath: testUpgradeDiffConfig, - setManifestPath: true, - manifestPath: "bad-path", - expectedError: true, - }, - { - name: "invalid: badly formatted version as argument", - cfgPath: testUpgradeDiffConfig, - args: []string{"bad-version"}, - expectedError: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - flags.cfgPath = tc.cfgPath - if tc.setManifestPath { - flags.apiServerManifestPath = tc.manifestPath - flags.controllerManagerManifestPath = tc.manifestPath - flags.schedulerManifestPath = tc.manifestPath - } - if err := runDiff(flags, tc.args); (err != nil) != tc.expectedError { - t.Fatalf("expected error: %v, saw: %v, error: %v", tc.expectedError, (err != nil), err) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go deleted file mode 100644 index af48a373c0b8a..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ /dev/null @@ -1,202 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "os" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -// nodeOptions defines all the options exposed via flags by kubeadm upgrade node. -// Please note that this structure includes the public kubeadm config API, but only a subset of the options -// supported by this api will be exposed as a flag. -type nodeOptions struct { - kubeConfigPath string - etcdUpgrade bool - renewCerts bool - dryRun bool - patchesDir string - ignorePreflightErrors []string -} - -// compile-time assert that the local data object satisfies the phases data interface. -var _ phases.Data = &nodeData{} - -// nodeData defines all the runtime information used when running the kubeadm upgrade node worklow; -// this data is shared across all the phases that are included in the workflow. -type nodeData struct { - etcdUpgrade bool - renewCerts bool - dryRun bool - cfg *kubeadmapi.InitConfiguration - isControlPlaneNode bool - client clientset.Interface - patchesDir string - ignorePreflightErrors sets.String -} - -// newCmdNode returns the cobra command for `kubeadm upgrade node` -func newCmdNode() *cobra.Command { - nodeOptions := newNodeOptions() - nodeRunner := workflow.NewRunner() - - cmd := &cobra.Command{ - Use: "node", - Short: "Upgrade commands for a node in the cluster", - RunE: func(cmd *cobra.Command, args []string) error { - return nodeRunner.Run(args) - }, - Args: cobra.NoArgs, - } - - // adds flags to the node command - // flags could be eventually inherited by the sub-commands automatically generated for phases - addUpgradeNodeFlags(cmd.Flags(), nodeOptions) - options.AddPatchesFlag(cmd.Flags(), &nodeOptions.patchesDir) - - // initialize the workflow runner with the list of phases - nodeRunner.AppendPhase(phases.NewPreflightPhase()) - nodeRunner.AppendPhase(phases.NewControlPlane()) - nodeRunner.AppendPhase(phases.NewKubeletConfigPhase()) - - // sets the data builder function, that will be used by the runner - // both when running the entire workflow or single phases - nodeRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { - return newNodeData(cmd, args, nodeOptions) - }) - - // binds the Runner to kubeadm upgrade node command by altering - // command help, adding --skip-phases flag and by adding phases subcommands - nodeRunner.BindToCommand(cmd) - - return cmd -} - -// newNodeOptions returns a struct ready for being used for creating cmd kubeadm upgrade node flags. -func newNodeOptions() *nodeOptions { - return &nodeOptions{ - kubeConfigPath: constants.GetKubeletKubeConfigPath(), - dryRun: false, - renewCerts: true, - etcdUpgrade: true, - } -} - -func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) { - options.AddKubeConfigFlag(flagSet, &nodeOptions.kubeConfigPath) - flagSet.BoolVar(&nodeOptions.dryRun, options.DryRun, nodeOptions.dryRun, "Do not change any state, just output the actions that would be performed.") - flagSet.BoolVar(&nodeOptions.renewCerts, options.CertificateRenewal, nodeOptions.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") - flagSet.BoolVar(&nodeOptions.etcdUpgrade, options.EtcdUpgrade, nodeOptions.etcdUpgrade, "Perform the upgrade of etcd.") - flagSet.StringSliceVar(&nodeOptions.ignorePreflightErrors, options.IgnorePreflightErrors, nodeOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.") -} - -// newNodeData returns a new nodeData struct to be used for the execution of the kubeadm upgrade node workflow. -// This func takes care of validating nodeOptions passed to the command, and then it converts -// options into the internal InitConfiguration type that is used as input all the phases in the kubeadm upgrade node workflow -func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*nodeData, error) { - client, err := getClient(options.kubeConfigPath, options.dryRun) - if err != nil { - return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", options.kubeConfigPath) - } - - // isControlPlane checks if a node is a control-plane node by looking up - // the kube-apiserver manifest file - isControlPlaneNode := true - filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory()) - if _, err := os.Stat(filepath); os.IsNotExist(err) { - isControlPlaneNode = false - } - - // Fetches the cluster configuration - // NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node - // (worker node), we are not reading local API address and the CRI socket from the node object - cfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, "upgrade", !isControlPlaneNode, false) - if err != nil { - return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") - } - - ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors) - if err != nil { - return nil, err - } - // Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration: - cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List() - - return &nodeData{ - etcdUpgrade: options.etcdUpgrade, - renewCerts: options.renewCerts, - dryRun: options.dryRun, - cfg: cfg, - client: client, - isControlPlaneNode: isControlPlaneNode, - patchesDir: options.patchesDir, - ignorePreflightErrors: ignorePreflightErrorsSet, - }, nil -} - -// DryRun returns the dryRun flag. -func (d *nodeData) DryRun() bool { - return d.dryRun -} - -// EtcdUpgrade returns the etcdUpgrade flag. -func (d *nodeData) EtcdUpgrade() bool { - return d.etcdUpgrade -} - -// RenewCerts returns the renewCerts flag. -func (d *nodeData) RenewCerts() bool { - return d.renewCerts -} - -// Cfg returns initConfiguration. -func (d *nodeData) Cfg() *kubeadmapi.InitConfiguration { - return d.cfg -} - -// IsControlPlaneNode returns the isControlPlaneNode flag. -func (d *nodeData) IsControlPlaneNode() bool { - return d.isControlPlaneNode -} - -// Client returns a Kubernetes client to be used by kubeadm. -func (d *nodeData) Client() clientset.Interface { - return d.client -} - -// PatchesDir returns the folder where patches for components are stored -func (d *nodeData) PatchesDir() string { - return d.patchesDir -} - -// IgnorePreflightErrors returns the list of preflight errors to ignore. -func (d *nodeData) IgnorePreflightErrors() sets.String { - return d.ignorePreflightErrors -} diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go deleted file mode 100644 index 5f8e824589e2c..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ /dev/null @@ -1,320 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "sort" - "strings" - "text/tabwriter" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - outputapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -type planFlags struct { - *applyPlanFlags -} - -// newCmdPlan returns the cobra command for `kubeadm upgrade plan` -func newCmdPlan(apf *applyPlanFlags) *cobra.Command { - flags := &planFlags{ - applyPlanFlags: apf, - } - - cmd := &cobra.Command{ - Use: "plan [version] [flags]", - Short: "Check which versions are available to upgrade to and validate whether your current cluster is upgradeable. To skip the internet check, pass in the optional [version] parameter", - RunE: func(_ *cobra.Command, args []string) error { - return runPlan(flags, args) - }, - } - - // Register the common flags for apply and plan - addApplyPlanFlags(cmd.Flags(), flags.applyPlanFlags) - return cmd -} - -// runPlan takes care of outputting available versions to upgrade to for the user -func runPlan(flags *planFlags, args []string) error { - // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never dry-run when planning. - klog.V(1).Infoln("[upgrade/plan] verifying health of cluster") - klog.V(1).Infoln("[upgrade/plan] retrieving configuration from cluster") - client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, false, false) - if err != nil { - return err - } - - // Currently this is the only method we have for distinguishing - // external etcd vs static pod etcd - isExternalEtcd := cfg.Etcd.External != nil - - // Compute which upgrade possibilities there are - klog.V(1).Infoln("[upgrade/plan] computing upgrade possibilities") - availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, isExternalEtcd, cfg.DNS.Type, client, constants.GetStaticPodDirectory()) - if err != nil { - return errors.Wrap(err, "[upgrade/versions] FATAL") - } - - // Fetch the current state of the component configs - klog.V(1).Infoln("[upgrade/plan] analysing component config version states") - configVersionStates, err := getComponentConfigVersionStates(&cfg.ClusterConfiguration, client, flags.cfgPath) - if err != nil { - return errors.WithMessage(err, "[upgrade/versions] FATAL") - } - - // No upgrades available - if len(availUpgrades) == 0 { - klog.V(1).Infoln("[upgrade/plan] Awesome, you're up-to-date! Enjoy!") - return nil - } - - // Generate and print upgrade plans - for _, up := range availUpgrades { - plan, unstableVersionFlag, err := genUpgradePlan(&up, isExternalEtcd) - if err != nil { - return err - } - - // Actually, this is needed for machine readable output only. - // printUpgradePlan won't output the configVersionStates as it will simply print the same table several times - // in the human readable output if it did so - plan.ConfigVersions = configVersionStates - - printUpgradePlan(&up, plan, unstableVersionFlag, isExternalEtcd, os.Stdout) - } - - // Finally, print the component config state table - printComponentConfigVersionStates(configVersionStates, os.Stdout) - - return nil -} - -// newComponentUpgradePlan helper creates outputapi.ComponentUpgradePlan object -func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapi.ComponentUpgradePlan { - return outputapi.ComponentUpgradePlan{ - Name: name, - CurrentVersion: currentVersion, - NewVersion: newVersion, - } -} - -// TODO There is currently no way to cleanly output upgrades that involve adding, removing, or changing components -// https://github.com/kubernetes/kubeadm/issues/810 was created to track addressing this. -func appendDNSComponent(components []outputapi.ComponentUpgradePlan, up *upgrade.Upgrade, DNSType kubeadmapi.DNSAddOnType, name string) []outputapi.ComponentUpgradePlan { - beforeVersion, afterVersion := "", "" - if up.Before.DNSType == DNSType { - beforeVersion = up.Before.DNSVersion - } - if up.After.DNSType == DNSType { - afterVersion = up.After.DNSVersion - } - - if beforeVersion != "" || afterVersion != "" { - components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion)) - } - return components -} - -// genUpgradePlan generates output-friendly upgrade plan out of upgrade.Upgrade structure -func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapi.UpgradePlan, string, error) { - newK8sVersion, err := version.ParseSemantic(up.After.KubeVersion) - if err != nil { - return nil, "", errors.Wrapf(err, "Unable to parse normalized version %q as a semantic version", up.After.KubeVersion) - } - - unstableVersionFlag := "" - if len(newK8sVersion.PreRelease()) != 0 { - if strings.HasPrefix(newK8sVersion.PreRelease(), "rc") { - unstableVersionFlag = " --allow-release-candidate-upgrades" - } else { - unstableVersionFlag = " --allow-experimental-upgrades" - } - } - - components := []outputapi.ComponentUpgradePlan{} - - if up.CanUpgradeKubelets() { - // The map is of the form :. Here all the keys are put into a slice and sorted - // in order to always get the right order. Then the map value is extracted separately - for _, oldVersion := range sortedSliceFromStringIntMap(up.Before.KubeletVersions) { - nodeCount := up.Before.KubeletVersions[oldVersion] - components = append(components, newComponentUpgradePlan(constants.Kubelet, fmt.Sprintf("%d x %s", nodeCount, oldVersion), up.After.KubeVersion)) - } - } - - components = append(components, newComponentUpgradePlan(constants.KubeAPIServer, up.Before.KubeVersion, up.After.KubeVersion)) - components = append(components, newComponentUpgradePlan(constants.KubeControllerManager, up.Before.KubeVersion, up.After.KubeVersion)) - components = append(components, newComponentUpgradePlan(constants.KubeScheduler, up.Before.KubeVersion, up.After.KubeVersion)) - components = append(components, newComponentUpgradePlan(constants.KubeProxy, up.Before.KubeVersion, up.After.KubeVersion)) - - components = appendDNSComponent(components, up, kubeadmapi.CoreDNS, constants.CoreDNS) - components = appendDNSComponent(components, up, kubeadmapi.KubeDNS, constants.KubeDNS) - - if !isExternalEtcd { - components = append(components, newComponentUpgradePlan(constants.Etcd, up.Before.EtcdVersion, up.After.EtcdVersion)) - } - - return &outputapi.UpgradePlan{Components: components}, unstableVersionFlag, nil -} - -func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, cfgPath string) ([]outputapi.ComponentConfigVersionState, error) { - docmap := kubeadmapi.DocumentMap{} - - if cfgPath != "" { - bytes, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, errors.Wrapf(err, "unable to read config file %q", cfgPath) - } - - docmap, err = kubeadmutil.SplitYAMLDocuments(bytes) - if err != nil { - return nil, err - } - } - - return componentconfigs.GetVersionStates(cfg, client, docmap) -} - -// printUpgradePlan prints a UX-friendly overview of what versions are available to upgrade to -func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, w io.Writer) { - // The tab writer writes to the "real" writer w - tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) - - // endOfTable helper function flashes table writer - endOfTable := func() { - tabw.Flush() - fmt.Fprintln(w, "") - } - - printHeader := true - printManualUpgradeHeader := true - for _, component := range plan.Components { - if isExternalEtcd && component.Name == constants.Etcd { - // Don't print etcd if it's external - continue - } else if component.Name == constants.Kubelet { - if printManualUpgradeHeader { - fmt.Fprintln(w, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") - fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tAVAILABLE") - fmt.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) - printManualUpgradeHeader = false - } else { - fmt.Fprintf(tabw, "%s\t%s\t%s\n", "", component.CurrentVersion, component.NewVersion) - } - } else { - if printHeader { - // End of manual upgrades table - endOfTable() - - fmt.Fprintf(w, "Upgrade to the latest %s:\n", up.Description) - fmt.Fprintln(w, "") - fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tAVAILABLE") - printHeader = false - } - fmt.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) - } - } - // End of control plane table - endOfTable() - - //fmt.Fprintln(w, "") - fmt.Fprintln(w, "You can now apply the upgrade by executing the following command:") - fmt.Fprintln(w, "") - fmt.Fprintf(w, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag) - fmt.Fprintln(w, "") - - if up.Before.KubeadmVersion != up.After.KubeadmVersion { - fmt.Fprintf(w, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion) - fmt.Fprintln(w, "") - } - - printLineSeparator(w) -} - -// sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically -func sortedSliceFromStringIntMap(strMap map[string]uint16) []string { - strSlice := []string{} - for k := range strMap { - strSlice = append(strSlice, k) - } - sort.Strings(strSlice) - return strSlice -} - -func strOrDash(s string) string { - if s != "" { - return s - } - return "-" -} - -func yesOrNo(b bool) string { - if b { - return "yes" - } - return "no" -} - -func printLineSeparator(w io.Writer) { - fmt.Fprintln(w, "_____________________________________________________________________") - fmt.Fprintln(w, "") -} - -func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfigVersionState, w io.Writer) { - if len(versionStates) == 0 { - fmt.Fprintln(w, "No information available on component configs.") - return - } - - fmt.Fprintln(w, dedent.Dedent(` - The table below shows the current state of component configs as understood by this version of kubeadm. - Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or - resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually - upgrade to is denoted in the "PREFERRED VERSION" column. - `)) - - tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) - fmt.Fprintln(tabw, "API GROUP\tCURRENT VERSION\tPREFERRED VERSION\tMANUAL UPGRADE REQUIRED") - - for _, state := range versionStates { - fmt.Fprintf(tabw, - "%s\t%s\t%s\t%s\n", - state.Group, - strOrDash(state.CurrentVersion), - strOrDash(state.PreferredVersion), - yesOrNo(state.ManualUpgradeRequired), - ) - } - - tabw.Flush() - printLineSeparator(w) -} diff --git a/cmd/kubeadm/app/cmd/upgrade/plan_test.go b/cmd/kubeadm/app/cmd/upgrade/plan_test.go deleted file mode 100644 index 8e9026c29206c..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/plan_test.go +++ /dev/null @@ -1,617 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "bytes" - "reflect" - "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" -) - -func TestSortedSliceFromStringIntMap(t *testing.T) { - var tests = []struct { - name string - strMap map[string]uint16 - expectedSlice []string - }{ - { - name: "the returned slice should be alphabetically sorted based on the string keys in the map", - strMap: map[string]uint16{"foo": 1, "bar": 2}, - expectedSlice: []string{"bar", "foo"}, - }, - { - name: "the int value should not affect this func", - strMap: map[string]uint16{"foo": 2, "bar": 1}, - expectedSlice: []string{"bar", "foo"}, - }, - { - name: "slice with 4 keys and different values", - strMap: map[string]uint16{"b": 2, "a": 1, "cb": 0, "ca": 1000}, - expectedSlice: []string{"a", "b", "ca", "cb"}, - }, - { - name: "this should work for version numbers as well; and the lowest version should come first", - strMap: map[string]uint16{"v1.7.0": 1, "v1.6.1": 1, "v1.6.2": 1, "v1.8.0": 1, "v1.8.0-alpha.1": 1}, - expectedSlice: []string{"v1.6.1", "v1.6.2", "v1.7.0", "v1.8.0", "v1.8.0-alpha.1"}, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualSlice := sortedSliceFromStringIntMap(rt.strMap) - if !reflect.DeepEqual(actualSlice, rt.expectedSlice) { - t.Errorf( - "failed SortedSliceFromStringIntMap:\n\texpected: %v\n\t actual: %v", - rt.expectedSlice, - actualSlice, - ) - } - }) - } -} - -// TODO Think about modifying this test to be less verbose checking b/c it can be brittle. -func TestPrintAvailableUpgrades(t *testing.T) { - var tests = []struct { - name string - upgrades []upgrade.Upgrade - buf *bytes.Buffer - expectedBytes []byte - externalEtcd bool - }{ - { - name: "Patch version available", - upgrades: []upgrade.Upgrade{ - { - Description: "version in the v1.8 series", - Before: upgrade.ClusterState{ - KubeVersion: "v1.8.1", - KubeletVersions: map[string]uint16{ - "v1.8.1": 1, - }, - KubeadmVersion: "v1.8.2", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.8.3", - KubeadmVersion: "v1.8.3", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.8.1 v1.8.3 - -Upgrade to the latest version in the v1.8 series: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.8.1 v1.8.3 -kube-controller-manager v1.8.1 v1.8.3 -kube-scheduler v1.8.1 v1.8.3 -kube-proxy v1.8.1 v1.8.3 -kube-dns 1.14.5 1.14.5 -etcd 3.0.17 3.0.17 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.8.3 - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.8.3. - -_____________________________________________________________________ - -`), - }, - { - name: "minor version available", - upgrades: []upgrade.Upgrade{ - { - Description: "stable version", - Before: upgrade.ClusterState{ - KubeVersion: "v1.8.3", - KubeletVersions: map[string]uint16{ - "v1.8.3": 1, - }, - KubeadmVersion: "v1.9.0", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.9.0", - KubeadmVersion: "v1.9.0", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.13", - EtcdVersion: "3.1.12", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.8.3 v1.9.0 - -Upgrade to the latest stable version: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.8.3 v1.9.0 -kube-controller-manager v1.8.3 v1.9.0 -kube-scheduler v1.8.3 v1.9.0 -kube-proxy v1.8.3 v1.9.0 -kube-dns 1.14.5 1.14.13 -etcd 3.0.17 3.1.12 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.9.0 - -_____________________________________________________________________ - -`), - }, - { - name: "patch and minor version available", - upgrades: []upgrade.Upgrade{ - { - Description: "version in the v1.8 series", - Before: upgrade.ClusterState{ - KubeVersion: "v1.8.3", - KubeletVersions: map[string]uint16{ - "v1.8.3": 1, - }, - KubeadmVersion: "v1.8.3", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.8.5", - KubeadmVersion: "v1.8.3", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - }, - { - Description: "stable version", - Before: upgrade.ClusterState{ - KubeVersion: "v1.8.3", - KubeletVersions: map[string]uint16{ - "v1.8.3": 1, - }, - KubeadmVersion: "v1.8.3", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.9.0", - KubeadmVersion: "v1.9.0", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.13", - EtcdVersion: "3.1.12", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.8.3 v1.8.5 - -Upgrade to the latest version in the v1.8 series: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.8.3 v1.8.5 -kube-controller-manager v1.8.3 v1.8.5 -kube-scheduler v1.8.3 v1.8.5 -kube-proxy v1.8.3 v1.8.5 -kube-dns 1.14.5 1.14.5 -etcd 3.0.17 3.0.17 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.8.5 - -_____________________________________________________________________ - -Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.8.3 v1.9.0 - -Upgrade to the latest stable version: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.8.3 v1.9.0 -kube-controller-manager v1.8.3 v1.9.0 -kube-scheduler v1.8.3 v1.9.0 -kube-proxy v1.8.3 v1.9.0 -kube-dns 1.14.5 1.14.13 -etcd 3.0.17 3.1.12 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.9.0 - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.9.0. - -_____________________________________________________________________ - -`), - }, - { - name: "experimental version available", - upgrades: []upgrade.Upgrade{ - { - Description: "experimental version", - Before: upgrade.ClusterState{ - KubeVersion: "v1.8.5", - KubeletVersions: map[string]uint16{ - "v1.8.5": 1, - }, - KubeadmVersion: "v1.8.5", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.9.0-beta.1", - KubeadmVersion: "v1.9.0-beta.1", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.13", - EtcdVersion: "3.1.12", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.8.5 v1.9.0-beta.1 - -Upgrade to the latest experimental version: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.8.5 v1.9.0-beta.1 -kube-controller-manager v1.8.5 v1.9.0-beta.1 -kube-scheduler v1.8.5 v1.9.0-beta.1 -kube-proxy v1.8.5 v1.9.0-beta.1 -kube-dns 1.14.5 1.14.13 -etcd 3.0.17 3.1.12 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.9.0-beta.1 --allow-experimental-upgrades - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.9.0-beta.1. - -_____________________________________________________________________ - -`), - }, - { - name: "release candidate available", - upgrades: []upgrade.Upgrade{ - { - Description: "release candidate version", - Before: upgrade.ClusterState{ - KubeVersion: "v1.8.5", - KubeletVersions: map[string]uint16{ - "v1.8.5": 1, - }, - KubeadmVersion: "v1.8.5", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.9.0-rc.1", - KubeadmVersion: "v1.9.0-rc.1", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.13", - EtcdVersion: "3.1.12", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.8.5 v1.9.0-rc.1 - -Upgrade to the latest release candidate version: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.8.5 v1.9.0-rc.1 -kube-controller-manager v1.8.5 v1.9.0-rc.1 -kube-scheduler v1.8.5 v1.9.0-rc.1 -kube-proxy v1.8.5 v1.9.0-rc.1 -kube-dns 1.14.5 1.14.13 -etcd 3.0.17 3.1.12 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.9.0-rc.1 --allow-release-candidate-upgrades - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.9.0-rc.1. - -_____________________________________________________________________ - -`), - }, - { - name: "multiple kubelet versions", - upgrades: []upgrade.Upgrade{ - { - Description: "version in the v1.9 series", - Before: upgrade.ClusterState{ - KubeVersion: "v1.9.2", - KubeletVersions: map[string]uint16{ - "v1.9.2": 1, - "v1.9.3": 2, - }, - KubeadmVersion: "v1.9.2", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.9.3", - KubeadmVersion: "v1.9.3", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.8", - EtcdVersion: "3.1.12", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.9.2 v1.9.3 - 2 x v1.9.3 v1.9.3 - -Upgrade to the latest version in the v1.9 series: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.9.2 v1.9.3 -kube-controller-manager v1.9.2 v1.9.3 -kube-scheduler v1.9.2 v1.9.3 -kube-proxy v1.9.2 v1.9.3 -kube-dns 1.14.5 1.14.8 -etcd 3.0.17 3.1.12 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.9.3 - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.9.3. - -_____________________________________________________________________ - -`), - }, - - { - name: "external etcd upgrade available", - upgrades: []upgrade.Upgrade{ - { - Description: "version in the v1.9 series", - Before: upgrade.ClusterState{ - KubeVersion: "v1.9.2", - KubeletVersions: map[string]uint16{ - "v1.9.2": 1, - }, - KubeadmVersion: "v1.9.2", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.9.3", - KubeadmVersion: "v1.9.3", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.8", - EtcdVersion: "3.1.12", - }, - }, - }, - externalEtcd: true, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.9.2 v1.9.3 - -Upgrade to the latest version in the v1.9 series: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.9.2 v1.9.3 -kube-controller-manager v1.9.2 v1.9.3 -kube-scheduler v1.9.2 v1.9.3 -kube-proxy v1.9.2 v1.9.3 -kube-dns 1.14.5 1.14.8 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.9.3 - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.9.3. - -_____________________________________________________________________ - -`), - }, - { - name: "kubedns to coredns", - upgrades: []upgrade.Upgrade{ - { - Description: "kubedns to coredns", - Before: upgrade.ClusterState{ - KubeVersion: "v1.10.2", - KubeletVersions: map[string]uint16{ - "v1.10.2": 1, - }, - KubeadmVersion: "v1.11.0", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.7", - EtcdVersion: "3.1.11", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.11.0", - KubeadmVersion: "v1.11.0", - DNSType: kubeadmapi.CoreDNS, - DNSVersion: "1.0.6", - EtcdVersion: "3.2.18", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.10.2 v1.11.0 - -Upgrade to the latest kubedns to coredns: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.10.2 v1.11.0 -kube-controller-manager v1.10.2 v1.11.0 -kube-scheduler v1.10.2 v1.11.0 -kube-proxy v1.10.2 v1.11.0 -CoreDNS 1.0.6 -kube-dns 1.14.7 -etcd 3.1.11 3.2.18 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.11.0 - -_____________________________________________________________________ - -`), - }, - { - name: "coredns", - upgrades: []upgrade.Upgrade{ - { - Description: "coredns", - Before: upgrade.ClusterState{ - KubeVersion: "v1.10.2", - KubeletVersions: map[string]uint16{ - "v1.10.2": 1, - }, - KubeadmVersion: "v1.11.0", - DNSType: kubeadmapi.CoreDNS, - DNSVersion: "1.0.5", - EtcdVersion: "3.1.11", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.11.0", - KubeadmVersion: "v1.11.0", - DNSType: kubeadmapi.CoreDNS, - DNSVersion: "1.0.6", - EtcdVersion: "3.2.18", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.10.2 v1.11.0 - -Upgrade to the latest coredns: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.10.2 v1.11.0 -kube-controller-manager v1.10.2 v1.11.0 -kube-scheduler v1.10.2 v1.11.0 -kube-proxy v1.10.2 v1.11.0 -CoreDNS 1.0.5 1.0.6 -etcd 3.1.11 3.2.18 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.11.0 - -_____________________________________________________________________ - -`), - }, - { - name: "coredns to kubedns", - upgrades: []upgrade.Upgrade{ - { - Description: "coredns to kubedns", - Before: upgrade.ClusterState{ - KubeVersion: "v1.10.2", - KubeletVersions: map[string]uint16{ - "v1.10.2": 1, - }, - KubeadmVersion: "v1.11.0", - DNSType: kubeadmapi.CoreDNS, - DNSVersion: "1.0.6", - EtcdVersion: "3.1.11", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.11.0", - KubeadmVersion: "v1.11.0", - DNSType: kubeadmapi.KubeDNS, - DNSVersion: "1.14.9", - EtcdVersion: "3.2.18", - }, - }, - }, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT AVAILABLE -kubelet 1 x v1.10.2 v1.11.0 - -Upgrade to the latest coredns to kubedns: - -COMPONENT CURRENT AVAILABLE -kube-apiserver v1.10.2 v1.11.0 -kube-controller-manager v1.10.2 v1.11.0 -kube-scheduler v1.10.2 v1.11.0 -kube-proxy v1.10.2 v1.11.0 -CoreDNS 1.0.6 -kube-dns 1.14.9 -etcd 3.1.11 3.2.18 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.11.0 - -_____________________________________________________________________ - -`), - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - rt.buf = bytes.NewBufferString("") - // Generate and print upgrade plans - for _, up := range rt.upgrades { - plan, unstableVersionFlag, err := genUpgradePlan(&up, rt.externalEtcd) - if err != nil { - t.Errorf("failed genUpgradePlan, err: %+v", err) - } - printUpgradePlan(&up, plan, unstableVersionFlag, rt.externalEtcd, rt.buf) - } - actualBytes := rt.buf.Bytes() - if !bytes.Equal(actualBytes, rt.expectedBytes) { - t.Errorf( - "failed PrintAvailableUpgrades:\n\texpected: %q\n\n\tactual : %q", - string(rt.expectedBytes), - string(actualBytes), - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/upgrade/upgrade.go b/cmd/kubeadm/app/cmd/upgrade/upgrade.go deleted file mode 100644 index f65376b6f0b42..0000000000000 --- a/cmd/kubeadm/app/cmd/upgrade/upgrade.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "io" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// applyPlanFlags holds the values for the common flags in `kubeadm upgrade apply` and `kubeadm upgrade plan` -type applyPlanFlags struct { - kubeConfigPath string - cfgPath string - featureGatesString string - allowExperimentalUpgrades bool - allowRCUpgrades bool - printConfig bool - ignorePreflightErrors []string - out io.Writer -} - -// NewCmdUpgrade returns the cobra command for `kubeadm upgrade` -func NewCmdUpgrade(out io.Writer) *cobra.Command { - flags := &applyPlanFlags{ - kubeConfigPath: kubeadmconstants.GetAdminKubeConfigPath(), - cfgPath: "", - featureGatesString: "", - allowExperimentalUpgrades: false, - allowRCUpgrades: false, - printConfig: false, - out: out, - } - - cmd := &cobra.Command{ - Use: "upgrade", - Short: "Upgrade your cluster smoothly to a newer version with this command", - RunE: cmdutil.SubCmdRunE("upgrade"), - } - - cmd.AddCommand(newCmdApply(flags)) - cmd.AddCommand(newCmdPlan(flags)) - cmd.AddCommand(newCmdDiff(out)) - cmd.AddCommand(newCmdNode()) - return cmd -} - -func addApplyPlanFlags(fs *pflag.FlagSet, flags *applyPlanFlags) { - options.AddKubeConfigFlag(fs, &flags.kubeConfigPath) - options.AddConfigFlag(fs, &flags.cfgPath) - - fs.BoolVar(&flags.allowExperimentalUpgrades, "allow-experimental-upgrades", flags.allowExperimentalUpgrades, "Show unstable versions of Kubernetes as an upgrade alternative and allow upgrading to an alpha/beta/release candidate versions of Kubernetes.") - fs.BoolVar(&flags.allowRCUpgrades, "allow-release-candidate-upgrades", flags.allowRCUpgrades, "Show release candidate versions of Kubernetes as an upgrade alternative and allow upgrading to a release candidate versions of Kubernetes.") - fs.BoolVar(&flags.printConfig, "print-config", flags.printConfig, "Specifies whether the configuration file that will be used in the upgrade should be printed or not.") - options.AddFeatureGatesStringFlag(fs, &flags.featureGatesString) - options.AddIgnorePreflightErrorsFlag(fs, &flags.ignorePreflightErrors) -} diff --git a/cmd/kubeadm/app/cmd/util/BUILD b/cmd/kubeadm/app/cmd/util/BUILD deleted file mode 100644 index 3df5ea4406491..0000000000000 --- a/cmd/kubeadm/app/cmd/util/BUILD +++ /dev/null @@ -1,47 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "cmdutil.go", - "documentation.go", - "join.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/cmd/options:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/pubkeypin:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "cmdutil_test.go", - "documentation_test.go", - ], - embed = [":go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/cmd/util/cmdutil.go b/cmd/kubeadm/app/cmd/util/cmdutil.go deleted file mode 100644 index af1a1243edc94..0000000000000 --- a/cmd/kubeadm/app/cmd/util/cmdutil.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "github.com/pkg/errors" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// SubCmdRunE returns a function that handles a case where a subcommand must be specified -// Without this callback, if a user runs just the command without a subcommand, -// or with an invalid subcommand, cobra will print usage information, but still exit cleanly. -// We want to return an error code in these cases so that the -// user knows that their command was invalid. -func SubCmdRunE(name string) func(*cobra.Command, []string) error { - return func(_ *cobra.Command, args []string) error { - if len(args) < 1 { - return errors.Errorf("missing subcommand; %q is not meant to be run on its own", name) - } - - return errors.Errorf("invalid subcommand: %q", args[0]) - } -} - -// ValidateExactArgNumber validates that the required top-level arguments are specified -func ValidateExactArgNumber(args []string, supportedArgs []string) error { - lenSupported := len(supportedArgs) - validArgs := 0 - // Disregard possible "" arguments; they are invalid - for _, arg := range args { - if len(arg) > 0 { - validArgs++ - } - // break early for too many arguments - if validArgs > lenSupported { - return errors.Errorf("too many arguments. Required arguments: %v", supportedArgs) - } - } - - if validArgs < lenSupported { - return errors.Errorf("missing one or more required arguments. Required arguments: %v", supportedArgs) - } - return nil -} - -// GetKubeConfigPath can be used to search for a kubeconfig in standard locations -// if and empty string is passed to the function. If a non-empty string is passed -// the function returns the same string. -func GetKubeConfigPath(file string) string { - // If a value is provided respect that. - if file != "" { - return file - } - // Find a config in the standard locations using DefaultClientConfigLoadingRules, - // but also consider the default config path. - rules := clientcmd.NewDefaultClientConfigLoadingRules() - rules.Precedence = append(rules.Precedence, kubeadmconstants.GetAdminKubeConfigPath()) - file = rules.GetDefaultFilename() - klog.V(1).Infof("Using kubeconfig file: %s", file) - return file -} - -// AddCRISocketFlag adds the cri-socket flag to the supplied flagSet -func AddCRISocketFlag(flagSet *pflag.FlagSet, criSocket *string) { - flagSet.StringVar( - criSocket, options.NodeCRISocket, *criSocket, - "Path to the CRI socket to connect. If empty kubeadm will try to auto-detect this value; use this option only if you have more than one CRI installed or if you have non-standard CRI socket.", - ) -} diff --git a/cmd/kubeadm/app/cmd/util/cmdutil_test.go b/cmd/kubeadm/app/cmd/util/cmdutil_test.go deleted file mode 100644 index 72ee829315c3a..0000000000000 --- a/cmd/kubeadm/app/cmd/util/cmdutil_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "testing" -) - -func TestValidateExactArgNumber(t *testing.T) { - var tests = []struct { - name string - args, supportedArgs []string - expectedErr bool - }{ - { - name: "one arg given and one arg expected", - args: []string{"my-node-1234"}, - supportedArgs: []string{"node-name"}, - expectedErr: false, - }, - { - name: "two args given and two args expected", - args: []string{"my-node-1234", "foo"}, - supportedArgs: []string{"node-name", "second-toplevel-arg"}, - expectedErr: false, - }, - { - name: "too few supplied args", - args: []string{}, - supportedArgs: []string{"node-name"}, - expectedErr: true, - }, - { - name: "too few non-empty args", - args: []string{""}, - supportedArgs: []string{"node-name"}, - expectedErr: true, - }, - { - name: "too many args", - args: []string{"my-node-1234", "foo"}, - supportedArgs: []string{"node-name"}, - expectedErr: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := ValidateExactArgNumber(rt.args, rt.supportedArgs) - if (actual != nil) != rt.expectedErr { - t.Errorf( - "failed ValidateExactArgNumber:\n\texpected error: %t\n\t actual error: %t", - rt.expectedErr, - (actual != nil), - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/util/documentation.go b/cmd/kubeadm/app/cmd/util/documentation.go deleted file mode 100644 index 29c4c4b3f6ef0..0000000000000 --- a/cmd/kubeadm/app/cmd/util/documentation.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "strings" -) - -var ( - // AlphaDisclaimer to be places at the end of description of commands in alpha release - AlphaDisclaimer = ` - Alpha Disclaimer: this command is currently alpha. - ` - - // MacroCommandLongDescription provide a standard description for "macro" commands - MacroCommandLongDescription = LongDesc(` - This command is not meant to be run on its own. See list of available subcommands. - `) -) - -// LongDesc is designed to help with producing better long command line descriptions in code. -// Its behavior is somewhat inspired by the same function of kubectl, which uses Markdown for the input message. -// This one is not Markdown compliant, but it covers the needs of kubeadm. In particular: -// - Beginning and trailing space characters (including empty lines), are stripped from the output. -// - Consecutive non-empty lines of text are joined with spaces to form paragraphs. -// - Paragraphs are blocks of text divided by one or more empty lines or lines consisting only of "space" characters. -// - Paragraphs are spaced by precisely one empty line in the output. -// - A line break can be forced by adding a couple of empty spaces at the end of a text line. -// - All indentation is removed. The resulting output is not indented. -func LongDesc(s string) string { - // Strip beginning and trailing space characters (including empty lines) and split the lines into a slice - lines := strings.Split(strings.TrimSpace(s), "\n") - - output := []string{} - paragraph := []string{} - - for _, line := range lines { - // Remove indentation and trailing spaces from the current line - trimmedLine := strings.TrimSpace(line) - if trimmedLine == "" { - if len(paragraph) > 0 { - // If the line is empty and the current paragraph is not, we have reached a paragraph end. - // (if the paragraph and the line are empty, then this is non-first empty line in between paragraphs and needs to be ignored) - // In that case we join all of the paragraph lines with a single space, - // add a trailing newline character (to ensure an empty line after the paragraph), - // append the paragraph text to the output and clear everything in the current paragraph slice. - output = append(output, strings.Join(paragraph, " ")+"\n") - paragraph = []string{} - } - } else { - // Non-empty text line, append it to the current paragraph - paragraph = append(paragraph, trimmedLine) - if strings.HasSuffix(line, " ") { - // If the original line has a suffix of couple of spaces, then we add a line break. - // This is achieved by flushing out the current paragraph and starting a new one. - // No trailing space is added to the flushed paragraph, - // so that no empty line is placed between the old and the new paragraphs (a simple line break) - output = append(output, strings.Join(paragraph, " ")) - paragraph = []string{} - } - } - } - - // The last paragraph is always unflushed, so flush it. - // We don't add a trailing newline character, so that we won't have to strip the output. - output = append(output, strings.Join(paragraph, " ")) - - // Join all the paragraphs together with new lines in between them. - return strings.Join(output, "\n") -} - -// Examples is designed to help with producing examples for command line usage. -// Its behavior is mimicking a similar kubectl function in the following ways: -// - Beginning and trailing space characters (including empty lines), are stripped from the output. -// - All lines of text are stripped of beginning and trailing spaces (thus loosing indentation) and are then double-space indented. -func Examples(s string) string { - trimmedText := strings.TrimSpace(s) - if trimmedText == "" { - return "" - } - - const indent = ` ` - inLines := strings.Split(trimmedText, "\n") - outLines := make([]string, 0, len(inLines)) - - for _, line := range inLines { - outLines = append(outLines, indent+strings.TrimSpace(line)) - } - - return strings.Join(outLines, "\n") -} diff --git a/cmd/kubeadm/app/cmd/util/documentation_test.go b/cmd/kubeadm/app/cmd/util/documentation_test.go deleted file mode 100644 index 0e03e40cc1229..0000000000000 --- a/cmd/kubeadm/app/cmd/util/documentation_test.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "testing" -) - -func TestLongDesc(t *testing.T) { - tests := []struct { - desc string - in string - out string - }{ - { - desc: "Empty input produces empty output", - in: "", - out: "", - }, - { - desc: "Single line text is preserved as is", - in: "Some text", - out: "Some text", - }, - { - desc: "Consecutive new lines are combined into a single paragraph", - in: "Line1\nLine2", - out: "Line1 Line2", - }, - { - desc: "Leading and trailing spaces are stripped (single line)", - in: "\t \nThe text line \n \t", - out: "The text line", - }, - { - desc: "Leading and trailing spaces are stripped (multi line)", - in: "\t \nLine1\nLine2 \n \t", - out: "Line1 Line2", - }, - { - desc: "Multiple paragraphs are separated by a single empty line", - in: "Paragraph1\n\nParagraph2\n\n\nParagraph3", - out: "Paragraph1\n\nParagraph2\n\nParagraph3", - }, - { - desc: "Indentation is not preserved", - in: "\tParagraph1Line1\n\tParagraph1Line2\n\n Paragraph2Line1\n Paragraph2Line2", - out: "Paragraph1Line1 Paragraph1Line2\n\nParagraph2Line1 Paragraph2Line2", - }, - { - desc: "Double spaced line breaks", - in: "Line1 \nLine2", - out: "Line1\nLine2", - }, - { - desc: "Double spaced line breaks don't preserve indentation", - in: "\tLine1 \n\tLine2", - out: "Line1\nLine2", - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - got := LongDesc(test.in) - if got != test.out { - t.Errorf("expected(%d):\n%s\n=====\ngot(%d):\n%s\n", len(test.out), test.out, len(got), got) - } - }) - } -} - -func TestExamples(t *testing.T) { - tests := []struct { - desc string - in string - out string - }{ - { - desc: "Empty input produces empty output", - in: "", - out: "", - }, - { - desc: "Text is indented with a couple of spaces", - in: "\tLine1\n\tLine2", - out: " Line1\n Line2", - }, - { - desc: "Text is stripped of leading and trailing spaces", - in: "\t\n\tLine1\t \n\tLine2\t \n\t\n\n", - out: " Line1\n Line2", - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - got := Examples(test.in) - if got != test.out { - t.Errorf("expected(%d):\n%s\n=====\ngot(%d):\n%s\n", len(test.out), test.out, len(got), got) - } - }) - } -} diff --git a/cmd/kubeadm/app/cmd/util/join.go b/cmd/kubeadm/app/cmd/util/join.go deleted file mode 100644 index d045435dad533..0000000000000 --- a/cmd/kubeadm/app/cmd/util/join.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "crypto/x509" - "html/template" - "strings" - - "github.com/pkg/errors" - "k8s.io/client-go/tools/clientcmd" - clientcertutil "k8s.io/client-go/util/cert" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" -) - -var joinCommandTemplate = template.Must(template.New("join").Parse(`` + - `kubeadm join {{.ControlPlaneHostPort}} --token {{.Token}} \ - {{range $h := .CAPubKeyPins}}--discovery-token-ca-cert-hash {{$h}} {{end}}{{if .ControlPlane}}\ - --control-plane {{if .CertificateKey}}--certificate-key {{.CertificateKey}}{{end}}{{end}}`, -)) - -// GetJoinWorkerCommand returns the kubeadm join command for a given token and -// and Kubernetes cluster (the current cluster in the kubeconfig file) -func GetJoinWorkerCommand(kubeConfigFile, token string, skipTokenPrint bool) (string, error) { - return getJoinCommand(kubeConfigFile, token, "", false, skipTokenPrint, false) -} - -// GetJoinControlPlaneCommand returns the kubeadm join command for a given token and -// and Kubernetes cluster (the current cluster in the kubeconfig file) -func GetJoinControlPlaneCommand(kubeConfigFile, token, key string, skipTokenPrint, skipCertificateKeyPrint bool) (string, error) { - return getJoinCommand(kubeConfigFile, token, key, true, skipTokenPrint, skipCertificateKeyPrint) -} - -func getJoinCommand(kubeConfigFile, token, key string, controlPlane, skipTokenPrint, skipCertificateKeyPrint bool) (string, error) { - // load the kubeconfig file to get the CA certificate and endpoint - config, err := clientcmd.LoadFromFile(kubeConfigFile) - if err != nil { - return "", errors.Wrap(err, "failed to load kubeconfig") - } - - // load the default cluster config - clusterConfig := kubeconfigutil.GetClusterFromKubeConfig(config) - if clusterConfig == nil { - return "", errors.New("failed to get default cluster config") - } - - // load CA certificates from the kubeconfig (either from PEM data or by file path) - var caCerts []*x509.Certificate - if clusterConfig.CertificateAuthorityData != nil { - caCerts, err = clientcertutil.ParseCertsPEM(clusterConfig.CertificateAuthorityData) - if err != nil { - return "", errors.Wrap(err, "failed to parse CA certificate from kubeconfig") - } - } else if clusterConfig.CertificateAuthority != "" { - caCerts, err = clientcertutil.CertsFromFile(clusterConfig.CertificateAuthority) - if err != nil { - return "", errors.Wrap(err, "failed to load CA certificate referenced by kubeconfig") - } - } else { - return "", errors.New("no CA certificates found in kubeconfig") - } - - // hash all the CA certs and include their public key pins as trusted values - publicKeyPins := make([]string, 0, len(caCerts)) - for _, caCert := range caCerts { - publicKeyPins = append(publicKeyPins, pubkeypin.Hash(caCert)) - } - - ctx := map[string]interface{}{ - "Token": token, - "CAPubKeyPins": publicKeyPins, - "ControlPlaneHostPort": strings.Replace(clusterConfig.Server, "https://", "", -1), - "CertificateKey": key, - "ControlPlane": controlPlane, - } - - if skipTokenPrint { - ctx["Token"] = template.HTML("") - } - if skipCertificateKeyPrint { - ctx["CertificateKey"] = template.HTML("") - } - - var out bytes.Buffer - err = joinCommandTemplate.Execute(&out, ctx) - if err != nil { - return "", errors.Wrap(err, "failed to render join command template") - } - return out.String(), nil -} diff --git a/cmd/kubeadm/app/cmd/version.go b/cmd/kubeadm/app/cmd/version.go deleted file mode 100644 index f26fbcd2f8db6..0000000000000 --- a/cmd/kubeadm/app/cmd/version.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - "k8s.io/klog/v2" - "sigs.k8s.io/yaml" - - apimachineryversion "k8s.io/apimachinery/pkg/version" - "k8s.io/component-base/version" -) - -// Version provides the version information of kubeadm. -type Version struct { - ClientVersion *apimachineryversion.Info `json:"clientVersion"` -} - -// newCmdVersion provides the version information of kubeadm. -func newCmdVersion(out io.Writer) *cobra.Command { - cmd := &cobra.Command{ - Use: "version", - Short: "Print the version of kubeadm", - RunE: func(cmd *cobra.Command, args []string) error { - return RunVersion(out, cmd) - }, - Args: cobra.NoArgs, - } - cmd.Flags().StringP("output", "o", "", "Output format; available options are 'yaml', 'json' and 'short'") - return cmd -} - -// RunVersion provides the version information of kubeadm in format depending on arguments -// specified in cobra.Command. -func RunVersion(out io.Writer, cmd *cobra.Command) error { - klog.V(1).Infoln("[version] retrieving version info") - clientVersion := version.Get() - v := Version{ - ClientVersion: &clientVersion, - } - - const flag = "output" - of, err := cmd.Flags().GetString(flag) - if err != nil { - return errors.Wrapf(err, "error accessing flag %s for command %s", flag, cmd.Name()) - } - - switch of { - case "": - fmt.Fprintf(out, "kubeadm version: %#v\n", v.ClientVersion) - case "short": - fmt.Fprintf(out, "%s\n", v.ClientVersion.GitVersion) - case "yaml": - y, err := yaml.Marshal(&v) - if err != nil { - return err - } - fmt.Fprintln(out, string(y)) - case "json": - y, err := json.MarshalIndent(&v, "", " ") - if err != nil { - return err - } - fmt.Fprintln(out, string(y)) - default: - return errors.Errorf("invalid output format: %s", of) - } - - return nil -} diff --git a/cmd/kubeadm/app/cmd/version_test.go b/cmd/kubeadm/app/cmd/version_test.go deleted file mode 100644 index d4469410d3abf..0000000000000 --- a/cmd/kubeadm/app/cmd/version_test.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cmd - -import ( - "bytes" - "encoding/json" - "github.com/pkg/errors" - "sigs.k8s.io/yaml" - "testing" -) - -func TestNewCmdVersion(t *testing.T) { - var buf bytes.Buffer - cmd := newCmdVersion(&buf) - if err := cmd.Execute(); err != nil { - t.Errorf("Cannot execute version command: %v", err) - } -} - -func TestRunVersion(t *testing.T) { - var buf bytes.Buffer - iface := make(map[string]interface{}) - flagNameOutput := "output" - cmd := newCmdVersion(&buf) - - testCases := []struct { - name string - flag string - expectedError bool - shouldBeValidYAML bool - shouldBeValidJSON bool - }{ - { - name: "valid: run without flags", - }, - { - name: "valid: run with flag 'short'", - flag: "short", - }, - { - name: "valid: run with flag 'yaml'", - flag: "yaml", - shouldBeValidYAML: true, - }, - { - name: "valid: run with flag 'json'", - flag: "json", - shouldBeValidJSON: true, - }, - { - name: "invalid: run with unsupported flag", - flag: "unsupported-flag", - expectedError: true, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var err error - if len(tc.flag) > 0 { - if err = cmd.Flags().Set(flagNameOutput, tc.flag); err != nil { - goto error - } - } - buf.Reset() - if err = RunVersion(&buf, cmd); err != nil { - goto error - } - if buf.String() == "" { - err = errors.New("empty output") - goto error - } - if tc.shouldBeValidYAML { - err = yaml.Unmarshal(buf.Bytes(), &iface) - } else if tc.shouldBeValidJSON { - err = json.Unmarshal(buf.Bytes(), &iface) - } - error: - if (err != nil) != tc.expectedError { - t.Errorf("Test case %q: RunVersion expected error: %v, saw: %v; %v", tc.name, tc.expectedError, err != nil, err) - } - }) - } -} diff --git a/cmd/kubeadm/app/componentconfigs/BUILD b/cmd/kubeadm/app/componentconfigs/BUILD deleted file mode 100644 index 2f456d8b9a075..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/BUILD +++ /dev/null @@ -1,87 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "checksums.go", - "configset.go", - "kubelet.go", - "kubeproxy.go", - "scheme.go", - "utils.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/output:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/initsystem:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", - "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "checksums_test.go", - "configset_test.go", - "fakeconfig_test.go", - "kubelet_test.go", - "kubeproxy_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/output:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/component-base/config/v1alpha1:go_default_library", - "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", - "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/componentconfigs/checksums.go b/cmd/kubeadm/app/componentconfigs/checksums.go deleted file mode 100644 index 94a360b55e5db..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/checksums.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "crypto/sha256" - "fmt" - "sort" - - v1 "k8s.io/api/core/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// ChecksumForConfigMap calculates a checksum for the supplied config map. The exact algorithm depends on hash and prefix parameters -func ChecksumForConfigMap(cm *v1.ConfigMap) string { - hash := sha256.New() - - // Since maps are not ordered we need to make sure we order them somehow so we'll always get the same checksums - // for the same config maps. The solution here is to extract the keys into a slice and sort them. - // Then iterate over that slice to fetch the values to be hashed. - keys := []string{} - for key := range cm.Data { - keys = append(keys, key) - } - sort.Strings(keys) - - for _, key := range keys { - hash.Write([]byte(cm.Data[key])) - } - - // Do the same as above, but for binaryData this time. - keys = []string{} - for key := range cm.BinaryData { - keys = append(keys, key) - } - sort.Strings(keys) - - for _, key := range keys { - hash.Write(cm.BinaryData[key]) - } - - return fmt.Sprintf("sha256:%x", hash.Sum(nil)) -} - -// SignConfigMap calculates the supplied config map checksum and annotates it with it -func SignConfigMap(cm *v1.ConfigMap) { - if cm.Annotations == nil { - cm.Annotations = map[string]string{} - } - cm.Annotations[constants.ComponentConfigHashAnnotationKey] = ChecksumForConfigMap(cm) -} - -// VerifyConfigMapSignature returns true if the config map has checksum annotation and it matches; false otherwise -func VerifyConfigMapSignature(cm *v1.ConfigMap) bool { - signature, ok := cm.Annotations[constants.ComponentConfigHashAnnotationKey] - if !ok { - return false - } - return signature == ChecksumForConfigMap(cm) -} diff --git a/cmd/kubeadm/app/componentconfigs/checksums_test.go b/cmd/kubeadm/app/componentconfigs/checksums_test.go deleted file mode 100644 index babd3020038d1..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/checksums_test.go +++ /dev/null @@ -1,224 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -var checksumTestCases = []struct { - desc string - configMap *v1.ConfigMap - checksum string -}{ - { - desc: "checksum is calculated using both data and binaryData", - checksum: "sha256:c8f8b724728a6d6684106e5e64e94ce811c9965d19dd44dd073cf86cf43bc238", - configMap: &v1.ConfigMap{ - Data: map[string]string{ - "foo": "bar", - }, - BinaryData: map[string][]byte{ - "bar": []byte("baz"), - }, - }, - }, - { - desc: "config keys have no effect on the checksum", - checksum: "sha256:c8f8b724728a6d6684106e5e64e94ce811c9965d19dd44dd073cf86cf43bc238", - configMap: &v1.ConfigMap{ - Data: map[string]string{ - "foo2": "bar", - }, - BinaryData: map[string][]byte{ - "bar2": []byte("baz"), - }, - }, - }, - { - desc: "metadata fields have no effect on the checksum", - checksum: "sha256:c8f8b724728a6d6684106e5e64e94ce811c9965d19dd44dd073cf86cf43bc238", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "le-config", - Namespace: "le-namespace", - Labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, - Annotations: map[string]string{ - "annotation1": "value1", - "annotation2": "value2", - }, - }, - Data: map[string]string{ - "foo": "bar", - }, - BinaryData: map[string][]byte{ - "bar": []byte("baz"), - }, - }, - }, - { - desc: "checksum can be calculated without binaryData", - checksum: "sha256:fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9", - configMap: &v1.ConfigMap{ - Data: map[string]string{ - "foo": "bar", - }, - }, - }, - { - desc: "checksum can be calculated without data", - checksum: "sha256:baa5a0964d3320fbc0c6a922140453c8513ea24ab8fd0577034804a967248096", - configMap: &v1.ConfigMap{ - BinaryData: map[string][]byte{ - "bar": []byte("baz"), - }, - }, - }, -} - -func TestChecksumForConfigMap(t *testing.T) { - for _, test := range checksumTestCases { - t.Run(test.desc, func(t *testing.T) { - got := ChecksumForConfigMap(test.configMap) - if got != test.checksum { - t.Errorf("checksum mismatch - got %q, expected %q", got, test.checksum) - } - }) - } -} - -func TestSignConfigMap(t *testing.T) { - for _, test := range checksumTestCases { - t.Run(test.desc, func(t *testing.T) { - target := test.configMap.DeepCopy() - SignConfigMap(target) - - // Verify that we have a correct annotation - signature, ok := target.Annotations[constants.ComponentConfigHashAnnotationKey] - if !ok { - t.Errorf("no %s annotation found in the config map", constants.ComponentConfigHashAnnotationKey) - } else { - if signature != test.checksum { - t.Errorf("unexpected checksum - got %q, expected %q", signature, test.checksum) - } - } - - // Verify that we have added an annotation (and not overwritten them) - expectedAnnotationCount := 1 + len(test.configMap.Annotations) - if len(target.Annotations) != expectedAnnotationCount { - t.Errorf("unexpected number of annotations - got %d, expected %d", len(target.Annotations), expectedAnnotationCount) - } - }) - } -} - -func TestVerifyConfigMapSignature(t *testing.T) { - tests := []struct { - desc string - configMap *v1.ConfigMap - expectErr bool - }{ - { - desc: "correct signature is acknowledged", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "le-config", - Namespace: "le-namespace", - Labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, - Annotations: map[string]string{ - "annotation1": "value1", - "annotation2": "value2", - constants.ComponentConfigHashAnnotationKey: "sha256:c8f8b724728a6d6684106e5e64e94ce811c9965d19dd44dd073cf86cf43bc238", - }, - }, - Data: map[string]string{ - "foo": "bar", - }, - BinaryData: map[string][]byte{ - "bar": []byte("baz"), - }, - }, - }, - { - desc: "wrong checksum leads to failure", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "le-config", - Namespace: "le-namespace", - Labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, - Annotations: map[string]string{ - "annotation1": "value1", - "annotation2": "value2", - constants.ComponentConfigHashAnnotationKey: "sha256:832cb34fc68fc370dd44dd91d5699c118ec49e46e5e6014866d6a827427b8f8c", - }, - }, - Data: map[string]string{ - "foo": "bar", - }, - BinaryData: map[string][]byte{ - "bar": []byte("baz"), - }, - }, - expectErr: true, - }, - { - desc: "missing signature means error", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "le-config", - Namespace: "le-namespace", - Labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, - Annotations: map[string]string{ - "annotation1": "value1", - "annotation2": "value2", - }, - }, - Data: map[string]string{ - "foo": "bar", - }, - BinaryData: map[string][]byte{ - "bar": []byte("baz"), - }, - }, - expectErr: true, - }, - } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - result := VerifyConfigMapSignature(test.configMap) - if result != !test.expectErr { - t.Errorf("unexpected result - got %t, expected %t", result, !test.expectErr) - } - }) - } -} diff --git a/cmd/kubeadm/app/componentconfigs/configset.go b/cmd/kubeadm/app/componentconfigs/configset.go deleted file mode 100644 index 906d99fd8cd12..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/configset.go +++ /dev/null @@ -1,342 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "github.com/pkg/errors" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/validation/field" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// handler is a package internal type that handles component config factory and common functionality. -// Every component config group should have exactly one static instance of handler. -type handler struct { - // GroupVersion holds this handler's group name and preferred version - GroupVersion schema.GroupVersion - - // AddToScheme points to a func that should add the GV types to a schema - AddToScheme func(*runtime.Scheme) error - - // CreateEmpty returns an empty kubeadmapi.ComponentConfig (not even defaulted) - CreateEmpty func() kubeadmapi.ComponentConfig - - // fromCluster should load the component config from a config map on the cluster. - // Don't use this directly! Use FromCluster instead! - fromCluster func(*handler, clientset.Interface, *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) -} - -// FromDocumentMap looks in the document map for documents with this handler's group. -// If such are found a new component config is instantiated and the documents are loaded into it. -// No error is returned if no documents are found. -func (h *handler) FromDocumentMap(docmap kubeadmapi.DocumentMap) (kubeadmapi.ComponentConfig, error) { - for gvk := range docmap { - if gvk.Group == h.GroupVersion.Group { - cfg := h.CreateEmpty() - if err := cfg.Unmarshal(docmap); err != nil { - return nil, err - } - // consider all successfully loaded configs from a document map as user supplied - cfg.SetUserSupplied(true) - return cfg, nil - } - } - return nil, nil -} - -// fromConfigMap is an utility function, which will load the value of a key of a config map and use h.FromDocumentMap() to perform the parsing -// This is an utility func. Used by the component config support implementations. Don't use it outside of that context. -func (h *handler) fromConfigMap(client clientset.Interface, cmName, cmKey string, mustExist bool) (kubeadmapi.ComponentConfig, error) { - configMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, cmName) - if err != nil { - if !mustExist && (apierrors.IsNotFound(err) || apierrors.IsForbidden(err)) { - klog.Warningf("Warning: No %s config is loaded. Continuing without it: %v", h.GroupVersion, err) - return nil, nil - } - return nil, err - } - - configData, ok := configMap.Data[cmKey] - if !ok { - return nil, errors.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", cmName, cmKey) - } - - gvkmap, err := kubeadmutil.SplitYAMLDocuments([]byte(configData)) - if err != nil { - return nil, err - } - - // If the checksum comes up neatly we assume the config was generated - generatedConfig := VerifyConfigMapSignature(configMap) - - componentCfg, err := h.FromDocumentMap(gvkmap) - if err != nil { - // If the config was generated and we get UnsupportedConfigVersionError, we skip loading it. - // This will force us to use the generated default current version (effectively regenerating the config with the current version). - if _, ok := err.(*UnsupportedConfigVersionError); ok && generatedConfig { - return nil, nil - } - return nil, err - } - - if componentCfg != nil { - componentCfg.SetUserSupplied(!generatedConfig) - } - - return componentCfg, nil -} - -// FromCluster loads a component from a config map in the cluster -func (h *handler) FromCluster(clientset clientset.Interface, clusterCfg *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) { - return h.fromCluster(h, clientset, clusterCfg) -} - -// known holds the known component config handlers. Add new component configs here. -var known = []*handler{ - &kubeProxyHandler, - &kubeletHandler, -} - -// configBase is the base type for all component config implementations -type configBase struct { - // GroupVersion holds the supported GroupVersion for the inheriting config - GroupVersion schema.GroupVersion - - // userSupplied tells us if the config is user supplied (invalid checksum) or not - userSupplied bool -} - -func (cb *configBase) IsUserSupplied() bool { - return cb.userSupplied -} - -func (cb *configBase) SetUserSupplied(userSupplied bool) { - cb.userSupplied = userSupplied -} - -func (cb *configBase) DeepCopyInto(other *configBase) { - *other = *cb -} - -func cloneBytes(in []byte) []byte { - out := make([]byte, len(in)) - copy(out, in) - return out -} - -// Marshal is an utility function, used by the component config support implementations to marshal a runtime.Object to YAML with the -// correct group and version -func (cb *configBase) Marshal(object runtime.Object) ([]byte, error) { - return kubeadmutil.MarshalToYamlForCodecs(object, cb.GroupVersion, Codecs) -} - -// Unmarshal attempts to unmarshal a runtime.Object from a document map. If no object is found, no error is returned. -// If a matching group is found, but no matching version an error is returned indicating that users should do manual conversion. -func (cb *configBase) Unmarshal(from kubeadmapi.DocumentMap, into runtime.Object) error { - for gvk, yaml := range from { - // If this is a different group, we ignore it - if gvk.Group != cb.GroupVersion.Group { - continue - } - - if gvk.Version != cb.GroupVersion.Version { - return &UnsupportedConfigVersionError{ - OldVersion: gvk.GroupVersion(), - CurrentVersion: cb.GroupVersion, - Document: cloneBytes(yaml), - } - } - - // As long as we support only component configs with a single kind, this is allowed - return runtime.DecodeInto(Codecs.UniversalDecoder(), yaml, into) - } - - return nil -} - -// ensureInitializedComponentConfigs is an utility func to initialize the ComponentConfigMap in ClusterConfiguration prior to possible writes to it -func ensureInitializedComponentConfigs(clusterCfg *kubeadmapi.ClusterConfiguration) { - if clusterCfg.ComponentConfigs == nil { - clusterCfg.ComponentConfigs = kubeadmapi.ComponentConfigMap{} - } -} - -// Default sets up defaulted component configs in the supplied ClusterConfiguration -func Default(clusterCfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) { - ensureInitializedComponentConfigs(clusterCfg) - - for _, handler := range known { - // If the component config exists, simply default it. Otherwise, create it before defaulting. - group := handler.GroupVersion.Group - if componentCfg, ok := clusterCfg.ComponentConfigs[group]; ok { - componentCfg.Default(clusterCfg, localAPIEndpoint, nodeRegOpts) - } else { - componentCfg := handler.CreateEmpty() - componentCfg.Default(clusterCfg, localAPIEndpoint, nodeRegOpts) - clusterCfg.ComponentConfigs[group] = componentCfg - } - } -} - -// FetchFromCluster attempts to fetch all known component configs from their config maps and store them in the supplied ClusterConfiguration -func FetchFromCluster(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error { - ensureInitializedComponentConfigs(clusterCfg) - - for _, handler := range known { - componentCfg, err := handler.FromCluster(client, clusterCfg) - if err != nil { - return err - } - - if componentCfg != nil { - clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg - } - } - - return nil -} - -// FetchFromDocumentMap attempts to load all known component configs from a document map into the supplied ClusterConfiguration -func FetchFromDocumentMap(clusterCfg *kubeadmapi.ClusterConfiguration, docmap kubeadmapi.DocumentMap) error { - ensureInitializedComponentConfigs(clusterCfg) - - for _, handler := range known { - componentCfg, err := handler.FromDocumentMap(docmap) - if err != nil { - return err - } - - if componentCfg != nil { - clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg - } - } - - return nil -} - -// FetchFromClusterWithLocalOverwrites fetches component configs from a cluster and overwrites them locally with -// the ones present in the supplied document map. If any UnsupportedConfigVersionError are not handled by the configs -// in the document map, the function returns them all as a single UnsupportedConfigVersionsErrorMap. -// This function is normally called only in some specific cases during upgrade. -func FetchFromClusterWithLocalOverwrites(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, docmap kubeadmapi.DocumentMap) error { - ensureInitializedComponentConfigs(clusterCfg) - - oldVersionErrs := UnsupportedConfigVersionsErrorMap{} - - for _, handler := range known { - componentCfg, err := handler.FromCluster(client, clusterCfg) - if err != nil { - if vererr, ok := err.(*UnsupportedConfigVersionError); ok { - oldVersionErrs[handler.GroupVersion.Group] = vererr - } else { - return err - } - } else if componentCfg != nil { - clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg - } - } - - for _, handler := range known { - componentCfg, err := handler.FromDocumentMap(docmap) - if err != nil { - if vererr, ok := err.(*UnsupportedConfigVersionError); ok { - oldVersionErrs[handler.GroupVersion.Group] = vererr - } else { - return err - } - } else if componentCfg != nil { - clusterCfg.ComponentConfigs[handler.GroupVersion.Group] = componentCfg - delete(oldVersionErrs, handler.GroupVersion.Group) - } - } - - if len(oldVersionErrs) != 0 { - return oldVersionErrs - } - - return nil -} - -// GetVersionStates returns a slice of ComponentConfigVersionState structs -// describing all supported component config groups that were identified on the cluster -func GetVersionStates(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, docmap kubeadmapi.DocumentMap) ([]output.ComponentConfigVersionState, error) { - // We don't want to modify clusterCfg so we make a working deep copy of it. - // Also, we don't want the defaulted component configs so we get rid of them. - scratchClusterCfg := clusterCfg.DeepCopy() - scratchClusterCfg.ComponentConfigs = kubeadmapi.ComponentConfigMap{} - - // Call FetchFromClusterWithLocalOverwrites. This will populate the configs it can load and will return all - // UnsupportedConfigVersionError(s) in a sinle instance of a MultipleUnsupportedConfigVersionsError. - var multipleVerErrs UnsupportedConfigVersionsErrorMap - err := FetchFromClusterWithLocalOverwrites(scratchClusterCfg, client, docmap) - if err != nil { - if vererrs, ok := err.(UnsupportedConfigVersionsErrorMap); ok { - multipleVerErrs = vererrs - } else { - // This seems to be a genuine error so we end here - return nil, err - } - } - - results := []output.ComponentConfigVersionState{} - for _, handler := range known { - group := handler.GroupVersion.Group - if vererr, ok := multipleVerErrs[group]; ok { - // If there is an UnsupportedConfigVersionError then we are dealing with a case where the config was user - // supplied and requires manual upgrade - results = append(results, output.ComponentConfigVersionState{ - Group: group, - CurrentVersion: vererr.OldVersion.Version, - PreferredVersion: vererr.CurrentVersion.Version, - ManualUpgradeRequired: true, - }) - } else if _, ok := scratchClusterCfg.ComponentConfigs[group]; ok { - // Normally loaded component config. No manual upgrade required on behalf of users. - results = append(results, output.ComponentConfigVersionState{ - Group: group, - CurrentVersion: handler.GroupVersion.Version, // Currently kubeadm supports only one version per API - PreferredVersion: handler.GroupVersion.Version, // group so we can get away with these being the same - }) - } else { - // This config was either not present (user did not install an addon) or the config was unsupported kubeadm - // generated one and is therefore skipped so we can automatically re-generate it (no action required on - // behalf of the user). - results = append(results, output.ComponentConfigVersionState{ - Group: group, - PreferredVersion: handler.GroupVersion.Version, - }) - } - } - - return results, nil -} - -// Validate is a placeholder for performing a validation on an already loaded component configs in a ClusterConfiguration -// TODO: investigate if the function can be repurposed for validating component config via CLI -func Validate(clusterCfg *kubeadmapi.ClusterConfiguration) field.ErrorList { - return field.ErrorList{} -} diff --git a/cmd/kubeadm/app/componentconfigs/configset_test.go b/cmd/kubeadm/app/componentconfigs/configset_test.go deleted file mode 100644 index 7189e9f7263e8..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/configset_test.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "testing" - - "github.com/lithammer/dedent" - - "k8s.io/apimachinery/pkg/runtime" - clientsetfake "k8s.io/client-go/kubernetes/fake" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -func testClusterCfg() *kubeadmapi.ClusterConfiguration { - return &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: constants.CurrentKubernetesVersion.String(), - } -} - -func TestDefault(t *testing.T) { - clusterCfg := testClusterCfg() - localAPIEndpoint := &kubeadmapi.APIEndpoint{} - nodeRegOps := &kubeadmapi.NodeRegistrationOptions{} - - Default(clusterCfg, localAPIEndpoint, nodeRegOps) - - if len(clusterCfg.ComponentConfigs) != len(known) { - t.Errorf("missmatch between supported and defaulted type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(known)) - } -} - -func TestFromCluster(t *testing.T) { - objects := []runtime.Object{ - testKubeProxyConfigMap(` - apiVersion: kubeproxy.config.k8s.io/v1alpha1 - kind: KubeProxyConfiguration - `), - testKubeletConfigMap(` - apiVersion: kubelet.config.k8s.io/v1beta1 - kind: KubeletConfiguration - `), - } - client := clientsetfake.NewSimpleClientset(objects...) - clusterCfg := testClusterCfg() - - if err := FetchFromCluster(clusterCfg, client); err != nil { - t.Fatalf("FetchFromCluster failed: %v", err) - } - - if len(clusterCfg.ComponentConfigs) != len(objects) { - t.Fatalf("missmatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(objects)) - } -} - -func TestFetchFromDocumentMap(t *testing.T) { - test := dedent.Dedent(` - apiVersion: kubeproxy.config.k8s.io/v1alpha1 - kind: KubeProxyConfiguration - --- - apiVersion: kubelet.config.k8s.io/v1beta1 - kind: KubeletConfiguration - `) - gvkmap, err := kubeadmutil.SplitYAMLDocuments([]byte(test)) - if err != nil { - t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err) - } - - clusterCfg := testClusterCfg() - if err = FetchFromDocumentMap(clusterCfg, gvkmap); err != nil { - t.Fatalf("FetchFromDocumentMap failed: %v", err) - } - - if len(clusterCfg.ComponentConfigs) != len(gvkmap) { - t.Fatalf("missmatch between supplied and loaded type numbers:\n\tgot: %d\n\texpected: %d", len(clusterCfg.ComponentConfigs), len(gvkmap)) - } -} diff --git a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go deleted file mode 100644 index 4db2625f3eccc..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go +++ /dev/null @@ -1,722 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "crypto/sha256" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "github.com/lithammer/dedent" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientset "k8s.io/client-go/kubernetes" - clientsetfake "k8s.io/client-go/kubernetes/fake" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - outputapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -// All tests in this file use an alternative set of `known` component configs. -// In this case it's just one known config and it's kubeadm's very own ClusterConfiguration. -// ClusterConfiguration is normally not managed by this package. It's only used, because of the following: -// - It's a versioned API that is under the control of kubeadm maintainers. This enables us to test -// the componentconfigs package more thoroughly without having to have full and always up to date -// knowledge about the config of another component. -// - Other components often introduce new fields in their configs without bumping up the config version. -// This, often times, requires that the PR that introduces such new fields to touch kubeadm test code. -// Doing so, requires more work on the part of developers and reviewers. When kubeadm moves out of k/k -// this would allow for more sporadic breaks in kubeadm tests as PRs that merge in k/k and introduce -// new fields won't be able to fix the tests in kubeadm. -// - If we implement tests for all common functionality using the config of another component and it gets -// deprecated and/or we stop supporting it in production, we'll have to focus on a massive test refactoring -// or just continue importing this config just for test use. -// -// Thus, to reduce maintenance costs without sacrificing test coverage, we introduce this mini-framework -// and set of tests here which replace the normal component configs with a single one (ClusterConfiguration) -// and test the component config independent logic of this package. - -// clusterConfigHandler is the handler instance for the latest supported ClusterConfiguration to be used in tests -var clusterConfigHandler = handler{ - GroupVersion: kubeadmapiv1.SchemeGroupVersion, - AddToScheme: kubeadmapiv1.AddToScheme, - CreateEmpty: func() kubeadmapi.ComponentConfig { - return &clusterConfig{ - configBase: configBase{ - GroupVersion: kubeadmapiv1.SchemeGroupVersion, - }, - } - }, - fromCluster: clusterConfigFromCluster, -} - -func clusterConfigFromCluster(h *handler, clientset clientset.Interface, _ *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) { - return h.fromConfigMap(clientset, constants.KubeadmConfigConfigMap, constants.ClusterConfigurationConfigMapKey, true) -} - -type clusterConfig struct { - configBase - config kubeadmapiv1.ClusterConfiguration -} - -func (cc *clusterConfig) DeepCopy() kubeadmapi.ComponentConfig { - result := &clusterConfig{} - cc.configBase.DeepCopyInto(&result.configBase) - cc.config.DeepCopyInto(&result.config) - return result -} - -func (cc *clusterConfig) Marshal() ([]byte, error) { - return cc.configBase.Marshal(&cc.config) -} - -func (cc *clusterConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error { - return cc.configBase.Unmarshal(docmap, &cc.config) -} - -func (cc *clusterConfig) Default(_ *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, _ *kubeadmapi.NodeRegistrationOptions) { - cc.config.ClusterName = "foo" - cc.config.KubernetesVersion = "bar" -} - -// fakeKnown replaces temporarily during the execution of each test here known (in configset.go) -var fakeKnown = []*handler{ - &clusterConfigHandler, -} - -// fakeKnownContext is the func that houses the fake component config context. -// NOTE: It does not support concurrent test execution! -func fakeKnownContext(f func()) { - // Save the real values - realKnown := known - realScheme := Scheme - realCodecs := Codecs - - // Replace the context with the fake context - known = fakeKnown - Scheme = kubeadmscheme.Scheme - Codecs = kubeadmscheme.Codecs - - // Upon function exit, restore the real values - defer func() { - known = realKnown - Scheme = realScheme - Codecs = realCodecs - }() - - // Call f in the fake context - f() -} - -// testClusterConfigMap is a short hand for creating and possibly signing a test config map. -// This produces config maps that can be loaded by clusterConfigFromCluster -func testClusterConfigMap(yaml string, signIt bool) *v1.ConfigMap { - cm := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.KubeadmConfigConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - constants.ClusterConfigurationConfigMapKey: dedent.Dedent(yaml), - }, - } - - if signIt { - SignConfigMap(cm) - } - - return cm -} - -// oldClusterConfigVersion is used as an old unsupported version in tests throughout this file -const oldClusterConfigVersion = "v1alpha1" - -var ( - // currentClusterConfigVersion represents the current actively supported version of ClusterConfiguration - currentClusterConfigVersion = kubeadmapiv1.SchemeGroupVersion.Version - - // currentFooClusterConfig is a minimal currently supported ClusterConfiguration - // with a well known value of clusterName (in this case `foo`) - currentFooClusterConfig = fmt.Sprintf(` - apiVersion: %s - kind: ClusterConfiguration - clusterName: foo - `, kubeadmapiv1.SchemeGroupVersion) - - // oldFooClusterConfig is a minimal unsupported ClusterConfiguration - // with a well known value of clusterName (in this case `foo`) - oldFooClusterConfig = fmt.Sprintf(` - apiVersion: %s/%s - kind: ClusterConfiguration - clusterName: foo - `, kubeadmapiv1.GroupName, oldClusterConfigVersion) - - // currentBarClusterConfig is a minimal currently supported ClusterConfiguration - // with a well known value of clusterName (in this case `bar`) - currentBarClusterConfig = fmt.Sprintf(` - apiVersion: %s - kind: ClusterConfiguration - clusterName: bar - `, kubeadmapiv1.SchemeGroupVersion) - - // oldBarClusterConfig is a minimal unsupported ClusterConfiguration - // with a well known value of clusterName (in this case `bar`) - oldBarClusterConfig = fmt.Sprintf(` - apiVersion: %s/%s - kind: ClusterConfiguration - clusterName: bar - `, kubeadmapiv1.GroupName, oldClusterConfigVersion) - - // This is the "minimal" valid config that can be unmarshalled to and from YAML. - // Due to same static defaulting it's not exactly small in size. - validUnmarshallableClusterConfig = struct { - yaml string - obj kubeadmapiv1.ClusterConfiguration - }{ - yaml: dedent.Dedent(` - apiServer: - timeoutForControlPlane: 4m - apiVersion: kubeadm.k8s.io/v1beta2 - certificatesDir: /etc/kubernetes/pki - clusterName: LeCluster - controllerManager: {} - dns: - type: CoreDNS - etcd: - local: - dataDir: /var/lib/etcd - imageRepository: k8s.gcr.io - kind: ClusterConfiguration - kubernetesVersion: 1.2.3 - networking: - dnsDomain: cluster.local - serviceSubnet: 10.96.0.0/12 - scheduler: {} - `), - obj: kubeadmapiv1.ClusterConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: kubeadmapiv1.SchemeGroupVersion.String(), - Kind: "ClusterConfiguration", - }, - ClusterName: "LeCluster", - KubernetesVersion: "1.2.3", - CertificatesDir: "/etc/kubernetes/pki", - ImageRepository: "k8s.gcr.io", - Networking: kubeadmapiv1.Networking{ - DNSDomain: "cluster.local", - ServiceSubnet: "10.96.0.0/12", - }, - DNS: kubeadmapiv1.DNS{ - Type: kubeadmapiv1.CoreDNS, - }, - Etcd: kubeadmapiv1.Etcd{ - Local: &kubeadmapiv1.LocalEtcd{ - DataDir: "/var/lib/etcd", - }, - }, - APIServer: kubeadmapiv1.APIServer{ - TimeoutForControlPlane: &metav1.Duration{ - Duration: 4 * time.Minute, - }, - }, - }, - } -) - -func TestConfigBaseMarshal(t *testing.T) { - fakeKnownContext(func() { - cfg := &clusterConfig{ - configBase: configBase{ - GroupVersion: kubeadmapiv1.SchemeGroupVersion, - }, - config: kubeadmapiv1.ClusterConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: kubeadmapiv1.SchemeGroupVersion.String(), - Kind: "ClusterConfiguration", - }, - ClusterName: "LeCluster", - KubernetesVersion: "1.2.3", - }, - } - - b, err := cfg.Marshal() - if err != nil { - t.Fatalf("Marshal failed: %v", err) - } - - got := strings.TrimSpace(string(b)) - expected := strings.TrimSpace(dedent.Dedent(` - apiServer: {} - apiVersion: kubeadm.k8s.io/v1beta2 - clusterName: LeCluster - controllerManager: {} - dns: - type: "" - etcd: {} - kind: ClusterConfiguration - kubernetesVersion: 1.2.3 - networking: {} - scheduler: {} - `)) - - if expected != got { - t.Fatalf("Missmatch between expected and got:\nExpected:\n%s\n---\nGot:\n%s", expected, got) - } - }) -} - -func TestConfigBaseUnmarshal(t *testing.T) { - fakeKnownContext(func() { - expected := &clusterConfig{ - configBase: configBase{ - GroupVersion: kubeadmapiv1.SchemeGroupVersion, - }, - config: validUnmarshallableClusterConfig.obj, - } - - gvkmap, err := kubeadmutil.SplitYAMLDocuments([]byte(validUnmarshallableClusterConfig.yaml)) - if err != nil { - t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err) - } - - got := &clusterConfig{ - configBase: configBase{ - GroupVersion: kubeadmapiv1.SchemeGroupVersion, - }, - } - if err = got.Unmarshal(gvkmap); err != nil { - t.Fatalf("unexpected failure of Unmarshal: %v", err) - } - - if !reflect.DeepEqual(got, expected) { - t.Fatalf("Missmatch between expected and got:\nExpected:\n%v\n---\nGot:\n%v", expected, got) - } - }) -} - -func TestGeneratedConfigFromCluster(t *testing.T) { - fakeKnownContext(func() { - testYAML := dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: ClusterConfiguration - `) - testYAMLHash := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(testYAML))) - // The SHA256 sum of "The quick brown fox jumps over the lazy dog" - const mismatchHash = "sha256:d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592" - tests := []struct { - name string - hash string - userSupplied bool - }{ - { - name: "Matching hash means generated config", - hash: testYAMLHash, - }, - { - name: "Missmatching hash means user supplied config", - hash: mismatchHash, - userSupplied: true, - }, - { - name: "No hash means user supplied config", - userSupplied: true, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - configMap := testClusterConfigMap(testYAML, false) - if test.hash != "" { - configMap.Annotations = map[string]string{ - constants.ComponentConfigHashAnnotationKey: test.hash, - } - } - - client := clientsetfake.NewSimpleClientset(configMap) - cfg, err := clusterConfigHandler.FromCluster(client, testClusterCfg()) - if err != nil { - t.Fatalf("unexpected failure of FromCluster: %v", err) - } - - got := cfg.IsUserSupplied() - if got != test.userSupplied { - t.Fatalf("mismatch between expected and got:\n\tExpected: %t\n\tGot: %t", test.userSupplied, got) - } - }) - } - }) -} - -// runClusterConfigFromTest holds common test case data and evaluation code for handler.From* functions -func runClusterConfigFromTest(t *testing.T, perform func(t *testing.T, in string) (kubeadmapi.ComponentConfig, error)) { - fakeKnownContext(func() { - tests := []struct { - name string - in string - out *clusterConfig - expectErr bool - }{ - { - name: "Empty document map should return nothing successfully", - }, - { - name: "Non-empty document map without the proper API group returns nothing successfully", - in: dedent.Dedent(` - apiVersion: api.example.com/v1 - kind: Configuration - `), - }, - { - name: "Old config version returns an error", - in: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1alpha1 - kind: ClusterConfiguration - `), - expectErr: true, - }, - { - name: "Unknown kind returns an error", - in: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: Configuration - `), - expectErr: true, - }, - { - name: "Valid config gets loaded", - in: validUnmarshallableClusterConfig.yaml, - out: &clusterConfig{ - configBase: configBase{ - GroupVersion: clusterConfigHandler.GroupVersion, - userSupplied: true, - }, - config: validUnmarshallableClusterConfig.obj, - }, - }, - { - name: "Valid config gets loaded even if coupled with an extra document", - in: "apiVersion: api.example.com/v1\nkind: Configuration\n---\n" + validUnmarshallableClusterConfig.yaml, - out: &clusterConfig{ - configBase: configBase{ - GroupVersion: clusterConfigHandler.GroupVersion, - userSupplied: true, - }, - config: validUnmarshallableClusterConfig.obj, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - componentCfg, err := perform(t, test.in) - if err != nil { - if !test.expectErr { - t.Errorf("unexpected failure: %v", err) - } - } else { - if test.expectErr { - t.Error("unexpected success") - } else { - if componentCfg == nil { - if test.out != nil { - t.Error("unexpected nil result") - } - } else { - if got, ok := componentCfg.(*clusterConfig); !ok { - t.Error("different result type") - } else { - if test.out == nil { - t.Errorf("unexpected result: %v", got) - } else { - if !reflect.DeepEqual(test.out, got) { - t.Errorf("missmatch between expected and got:\nExpected:\n%v\n---\nGot:\n%v", test.out, got) - } - } - } - } - } - } - }) - } - }) -} - -func TestLoadingFromDocumentMap(t *testing.T) { - runClusterConfigFromTest(t, func(t *testing.T, in string) (kubeadmapi.ComponentConfig, error) { - gvkmap, err := kubeadmutil.SplitYAMLDocuments([]byte(in)) - if err != nil { - t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err) - } - - return clusterConfigHandler.FromDocumentMap(gvkmap) - }) -} - -func TestLoadingFromCluster(t *testing.T) { - runClusterConfigFromTest(t, func(t *testing.T, in string) (kubeadmapi.ComponentConfig, error) { - client := clientsetfake.NewSimpleClientset( - testClusterConfigMap(in, false), - ) - - return clusterConfigHandler.FromCluster(client, testClusterCfg()) - }) -} - -func TestFetchFromClusterWithLocalOverwrites(t *testing.T) { - fakeKnownContext(func() { - cases := []struct { - desc string - obj runtime.Object - config string - expectedValue string - isNotLoaded bool - expectedErr bool - }{ - { - desc: "appropriate cluster object without overwrite is used", - obj: testClusterConfigMap(currentFooClusterConfig, false), - expectedValue: "foo", - }, - { - desc: "appropriate cluster object with appropriate overwrite is overwritten", - obj: testClusterConfigMap(currentFooClusterConfig, false), - config: dedent.Dedent(currentBarClusterConfig), - expectedValue: "bar", - }, - { - desc: "appropriate cluster object with old overwrite returns an error", - obj: testClusterConfigMap(currentFooClusterConfig, false), - config: dedent.Dedent(oldBarClusterConfig), - expectedErr: true, - }, - { - desc: "old config without overwrite returns an error", - obj: testClusterConfigMap(oldFooClusterConfig, false), - expectedErr: true, - }, - { - desc: "old config with appropriate overwrite returns the substitute", - obj: testClusterConfigMap(oldFooClusterConfig, false), - config: dedent.Dedent(currentBarClusterConfig), - expectedValue: "bar", - }, - { - desc: "old config with old overwrite returns an error", - obj: testClusterConfigMap(oldFooClusterConfig, false), - config: dedent.Dedent(oldBarClusterConfig), - expectedErr: true, - }, - { - desc: "appropriate signed cluster object without overwrite is used", - obj: testClusterConfigMap(currentFooClusterConfig, true), - expectedValue: "foo", - }, - { - desc: "appropriate signed cluster object with appropriate overwrite is overwritten", - obj: testClusterConfigMap(currentFooClusterConfig, true), - config: dedent.Dedent(currentBarClusterConfig), - expectedValue: "bar", - }, - { - desc: "appropriate signed cluster object with old overwrite returns an error", - obj: testClusterConfigMap(currentFooClusterConfig, true), - config: dedent.Dedent(oldBarClusterConfig), - expectedErr: true, - }, - { - desc: "old signed config without an overwrite is not loaded", - obj: testClusterConfigMap(oldFooClusterConfig, true), - isNotLoaded: true, - }, - { - desc: "old signed config with appropriate overwrite returns the substitute", - obj: testClusterConfigMap(oldFooClusterConfig, true), - config: dedent.Dedent(currentBarClusterConfig), - expectedValue: "bar", - }, - { - desc: "old signed config with old overwrite returns an error", - obj: testClusterConfigMap(oldFooClusterConfig, true), - config: dedent.Dedent(oldBarClusterConfig), - expectedErr: true, - }, - } - - for _, test := range cases { - t.Run(test.desc, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset(test.obj) - - docmap, err := kubeadmutil.SplitYAMLDocuments([]byte(test.config)) - if err != nil { - t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err) - } - - clusterCfg := testClusterCfg() - - err = FetchFromClusterWithLocalOverwrites(clusterCfg, client, docmap) - if err != nil { - if !test.expectedErr { - t.Errorf("unexpected failure: %v", err) - } - } else { - if test.expectedErr { - t.Error("unexpected success") - } else { - clusterCfg, ok := clusterCfg.ComponentConfigs[kubeadmapiv1.GroupName] - if !ok { - if !test.isNotLoaded { - t.Error("no config was loaded when it should have been") - } - } else { - actualConfig, ok := clusterCfg.(*clusterConfig) - if !ok { - t.Error("the config is not of the expected type") - } else if actualConfig.config.ClusterName != test.expectedValue { - t.Errorf("unexpected value:\n\tgot: %q\n\texpected: %q", actualConfig.config.ClusterName, test.expectedValue) - } - } - } - } - }) - } - }) -} - -func TestGetVersionStates(t *testing.T) { - fakeKnownContext(func() { - versionStateCurrent := outputapi.ComponentConfigVersionState{ - Group: kubeadmapiv1.GroupName, - CurrentVersion: currentClusterConfigVersion, - PreferredVersion: currentClusterConfigVersion, - } - versionStateOld := outputapi.ComponentConfigVersionState{ - Group: kubeadmapiv1.GroupName, - CurrentVersion: oldClusterConfigVersion, - PreferredVersion: currentClusterConfigVersion, - ManualUpgradeRequired: true, - } - - cases := []struct { - desc string - obj runtime.Object - config string - expected outputapi.ComponentConfigVersionState - }{ - { - desc: "appropriate cluster object without overwrite", - obj: testClusterConfigMap(currentFooClusterConfig, false), - expected: versionStateCurrent, - }, - { - desc: "appropriate cluster object with appropriate overwrite", - obj: testClusterConfigMap(currentFooClusterConfig, false), - config: dedent.Dedent(currentBarClusterConfig), - expected: versionStateCurrent, - }, - { - desc: "appropriate cluster object with old overwrite", - obj: testClusterConfigMap(currentFooClusterConfig, false), - config: dedent.Dedent(oldBarClusterConfig), - expected: versionStateOld, - }, - { - desc: "old config without overwrite returns an error", - obj: testClusterConfigMap(oldFooClusterConfig, false), - expected: versionStateOld, - }, - { - desc: "old config with appropriate overwrite", - obj: testClusterConfigMap(oldFooClusterConfig, false), - config: dedent.Dedent(currentBarClusterConfig), - expected: versionStateCurrent, - }, - { - desc: "old config with old overwrite", - obj: testClusterConfigMap(oldFooClusterConfig, false), - config: dedent.Dedent(oldBarClusterConfig), - expected: versionStateOld, - }, - { - desc: "appropriate signed cluster object without overwrite", - obj: testClusterConfigMap(currentFooClusterConfig, true), - expected: versionStateCurrent, - }, - { - desc: "appropriate signed cluster object with appropriate overwrite", - obj: testClusterConfigMap(currentFooClusterConfig, true), - config: dedent.Dedent(currentBarClusterConfig), - expected: versionStateCurrent, - }, - { - desc: "appropriate signed cluster object with old overwrit", - obj: testClusterConfigMap(currentFooClusterConfig, true), - config: dedent.Dedent(oldBarClusterConfig), - expected: versionStateOld, - }, - { - desc: "old signed config without an overwrite", - obj: testClusterConfigMap(oldFooClusterConfig, true), - expected: outputapi.ComponentConfigVersionState{ - Group: kubeadmapiv1.GroupName, - CurrentVersion: "", // The config is treated as if it's missing - PreferredVersion: currentClusterConfigVersion, - }, - }, - { - desc: "old signed config with appropriate overwrite", - obj: testClusterConfigMap(oldFooClusterConfig, true), - config: dedent.Dedent(currentBarClusterConfig), - expected: versionStateCurrent, - }, - { - desc: "old signed config with old overwrite", - obj: testClusterConfigMap(oldFooClusterConfig, true), - config: dedent.Dedent(oldBarClusterConfig), - expected: versionStateOld, - }, - } - - for _, test := range cases { - t.Run(test.desc, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset(test.obj) - - docmap, err := kubeadmutil.SplitYAMLDocuments([]byte(test.config)) - if err != nil { - t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err) - } - - clusterCfg := testClusterCfg() - - got, err := GetVersionStates(clusterCfg, client, docmap) - if err != nil { - t.Errorf("unexpected error: %v", err) - } else if len(got) != 1 { - t.Errorf("got %d, but expected only a single result: %v", len(got), got) - } else if got[0] != test.expected { - t.Errorf("unexpected result:\n\texpected: %v\n\tgot: %v", test.expected, got[0]) - } - }) - } - }) -} diff --git a/cmd/kubeadm/app/componentconfigs/kubelet.go b/cmd/kubeadm/app/componentconfigs/kubelet.go deleted file mode 100644 index ae13e8822e8e2..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/kubelet.go +++ /dev/null @@ -1,233 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "path/filepath" - - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeletconfig "k8s.io/kubelet/config/v1beta1" - "k8s.io/kubernetes/cmd/kubeadm/app/util/initsystem" - utilsexec "k8s.io/utils/exec" - utilpointer "k8s.io/utils/pointer" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const ( - // KubeletGroup is a pointer to the used API group name for the kubelet config - KubeletGroup = kubeletconfig.GroupName - - // kubeletReadOnlyPort specifies the default insecure http server port - // 0 will disable insecure http server. - kubeletReadOnlyPort int32 = 0 - - // kubeletRotateCertificates specifies the default value to enable certificate rotation - kubeletRotateCertificates = true - - // kubeletAuthenticationAnonymousEnabled specifies the default value to disable anonymous access - kubeletAuthenticationAnonymousEnabled = false - - // kubeletAuthenticationWebhookEnabled set the default value to enable authentication webhook - kubeletAuthenticationWebhookEnabled = true - - // kubeletHealthzBindAddress specifies the default healthz bind address - kubeletHealthzBindAddress = "127.0.0.1" - - // kubeletSystemdResolverConfig specifies the default resolver config when systemd service is active - kubeletSystemdResolverConfig = "/run/systemd/resolve/resolv.conf" -) - -// kubeletHandler is the handler instance for the kubelet component config -var kubeletHandler = handler{ - GroupVersion: kubeletconfig.SchemeGroupVersion, - AddToScheme: kubeletconfig.AddToScheme, - CreateEmpty: func() kubeadmapi.ComponentConfig { - return &kubeletConfig{ - configBase: configBase{ - GroupVersion: kubeletconfig.SchemeGroupVersion, - }, - } - }, - fromCluster: kubeletConfigFromCluster, -} - -func kubeletConfigFromCluster(h *handler, clientset clientset.Interface, clusterCfg *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) { - // Read the ConfigMap from the cluster based on what version the kubelet is - k8sVersion, err := version.ParseGeneric(clusterCfg.KubernetesVersion) - if err != nil { - return nil, err - } - - configMapName := constants.GetKubeletConfigMapName(k8sVersion) - return h.fromConfigMap(clientset, configMapName, constants.KubeletBaseConfigurationConfigMapKey, true) -} - -// kubeletConfig implements the kubeadmapi.ComponentConfig interface for kubelet -type kubeletConfig struct { - configBase - config kubeletconfig.KubeletConfiguration -} - -func (kc *kubeletConfig) DeepCopy() kubeadmapi.ComponentConfig { - result := &kubeletConfig{} - kc.configBase.DeepCopyInto(&result.configBase) - kc.config.DeepCopyInto(&result.config) - return result -} - -func (kc *kubeletConfig) Marshal() ([]byte, error) { - return kc.configBase.Marshal(&kc.config) -} - -func (kc *kubeletConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error { - return kc.configBase.Unmarshal(docmap, &kc.config) -} - -func (kc *kubeletConfig) Default(cfg *kubeadmapi.ClusterConfiguration, _ *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) { - const kind = "KubeletConfiguration" - - if kc.config.FeatureGates == nil { - kc.config.FeatureGates = map[string]bool{} - } - - // TODO: The following code should be removed after dual-stack is GA. - // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. - if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present { - kc.config.FeatureGates[features.IPv6DualStack] = enabled - } - - if kc.config.StaticPodPath == "" { - kc.config.StaticPodPath = kubeadmapiv1beta2.DefaultManifestsDir - } else if kc.config.StaticPodPath != kubeadmapiv1beta2.DefaultManifestsDir { - warnDefaultComponentConfigValue(kind, "staticPodPath", kubeadmapiv1beta2.DefaultManifestsDir, kc.config.StaticPodPath) - } - - clusterDNS := "" - dnsIP, err := constants.GetDNSIP(cfg.Networking.ServiceSubnet, features.Enabled(cfg.FeatureGates, features.IPv6DualStack)) - if err != nil { - clusterDNS = kubeadmapiv1beta2.DefaultClusterDNSIP - } else { - clusterDNS = dnsIP.String() - } - - if kc.config.ClusterDNS == nil { - kc.config.ClusterDNS = []string{clusterDNS} - } else if len(kc.config.ClusterDNS) != 1 || kc.config.ClusterDNS[0] != clusterDNS { - warnDefaultComponentConfigValue(kind, "clusterDNS", []string{clusterDNS}, kc.config.ClusterDNS) - } - - if kc.config.ClusterDomain == "" { - kc.config.ClusterDomain = cfg.Networking.DNSDomain - } else if cfg.Networking.DNSDomain != "" && kc.config.ClusterDomain != cfg.Networking.DNSDomain { - warnDefaultComponentConfigValue(kind, "clusterDomain", cfg.Networking.DNSDomain, kc.config.ClusterDomain) - } - - // Require all clients to the kubelet API to have client certs signed by the cluster CA - clientCAFile := filepath.Join(cfg.CertificatesDir, constants.CACertName) - if kc.config.Authentication.X509.ClientCAFile == "" { - kc.config.Authentication.X509.ClientCAFile = clientCAFile - } else if kc.config.Authentication.X509.ClientCAFile != clientCAFile { - warnDefaultComponentConfigValue(kind, "authentication.x509.clientCAFile", clientCAFile, kc.config.Authentication.X509.ClientCAFile) - } - - if kc.config.Authentication.Anonymous.Enabled == nil { - kc.config.Authentication.Anonymous.Enabled = utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled) - } else if *kc.config.Authentication.Anonymous.Enabled != kubeletAuthenticationAnonymousEnabled { - warnDefaultComponentConfigValue(kind, "authentication.anonymous.enabled", kubeletAuthenticationAnonymousEnabled, *kc.config.Authentication.Anonymous.Enabled) - } - - // On every client request to the kubelet API, execute a webhook (SubjectAccessReview request) to the API server - // and ask it whether the client is authorized to access the kubelet API - if kc.config.Authorization.Mode == "" { - kc.config.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeWebhook - } else if kc.config.Authorization.Mode != kubeletconfig.KubeletAuthorizationModeWebhook { - warnDefaultComponentConfigValue(kind, "authorization.mode", kubeletconfig.KubeletAuthorizationModeWebhook, kc.config.Authorization.Mode) - } - - // Let clients using other authentication methods like ServiceAccount tokens also access the kubelet API - if kc.config.Authentication.Webhook.Enabled == nil { - kc.config.Authentication.Webhook.Enabled = utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled) - } else if *kc.config.Authentication.Webhook.Enabled != kubeletAuthenticationWebhookEnabled { - warnDefaultComponentConfigValue(kind, "authentication.webhook.enabled", kubeletAuthenticationWebhookEnabled, *kc.config.Authentication.Webhook.Enabled) - } - - // Serve a /healthz webserver on localhost:10248 that kubeadm can talk to - if kc.config.HealthzBindAddress == "" { - kc.config.HealthzBindAddress = kubeletHealthzBindAddress - } else if kc.config.HealthzBindAddress != kubeletHealthzBindAddress { - warnDefaultComponentConfigValue(kind, "healthzBindAddress", kubeletHealthzBindAddress, kc.config.HealthzBindAddress) - } - - if kc.config.HealthzPort == nil { - kc.config.HealthzPort = utilpointer.Int32Ptr(constants.KubeletHealthzPort) - } else if *kc.config.HealthzPort != constants.KubeletHealthzPort { - warnDefaultComponentConfigValue(kind, "healthzPort", constants.KubeletHealthzPort, *kc.config.HealthzPort) - } - - if kc.config.ReadOnlyPort != kubeletReadOnlyPort { - warnDefaultComponentConfigValue(kind, "readOnlyPort", kubeletReadOnlyPort, kc.config.ReadOnlyPort) - } - - // We cannot show a warning for RotateCertificates==false and we must hardcode it to true. - // There is no way to determine if the user has set this or not, given the field is a non-pointer. - kc.config.RotateCertificates = kubeletRotateCertificates - - // TODO: Conditionally set CgroupDriver to either `systemd` or `cgroupfs` for CRI other than Docker - if nodeRegOpts.CRISocket == constants.DefaultDockerCRISocket { - driver, err := kubeadmutil.GetCgroupDriverDocker(utilsexec.New()) - if err != nil { - klog.Warningf("cannot automatically set CgroupDriver when starting the Kubelet: %v", err) - } else { - // if we can parse the right cgroup driver from docker info, - // we should always override CgroupDriver here no matter user specifies this value explicitly or not - if kc.config.CgroupDriver != "" && kc.config.CgroupDriver != driver { - klog.Warningf("detected %q as the Docker cgroup driver, the provided value %q in %q will be overrided", driver, kc.config.CgroupDriver, kind) - } - kc.config.CgroupDriver = driver - } - } - - ok, err := isServiceActive("systemd-resolved") - if err != nil { - klog.Warningf("cannot determine if systemd-resolved is active: %v", err) - } - if ok { - if kc.config.ResolverConfig == "" { - kc.config.ResolverConfig = kubeletSystemdResolverConfig - } else { - if kc.config.ResolverConfig != kubeletSystemdResolverConfig { - warnDefaultComponentConfigValue(kind, "resolvConf", kubeletSystemdResolverConfig, kc.config.ResolverConfig) - } - } - } -} - -// isServiceActive checks whether the given service exists and is running -func isServiceActive(name string) (bool, error) { - initSystem, err := initsystem.GetInitSystem() - if err != nil { - return false, err - } - return initSystem.ServiceIsActive(name), nil -} diff --git a/cmd/kubeadm/app/componentconfigs/kubelet_test.go b/cmd/kubeadm/app/componentconfigs/kubelet_test.go deleted file mode 100644 index bf5d7ce8e6cd2..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/kubelet_test.go +++ /dev/null @@ -1,329 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "fmt" - "path/filepath" - "reflect" - "testing" - - "github.com/lithammer/dedent" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - clientsetfake "k8s.io/client-go/kubernetes/fake" - kubeletconfig "k8s.io/kubelet/config/v1beta1" - utilpointer "k8s.io/utils/pointer" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" -) - -func testKubeletConfigMap(contents string) *v1.ConfigMap { - return &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.GetKubeletConfigMapName(constants.CurrentKubernetesVersion), - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - constants.KubeletBaseConfigurationConfigMapKey: dedent.Dedent(contents), - }, - } -} - -func TestKubeletDefault(t *testing.T) { - var resolverConfig string - if isSystemdResolvedActive, _ := isServiceActive("systemd-resolved"); isSystemdResolvedActive { - // If systemd-resolved is active, we need to set the default resolver config - resolverConfig = kubeletSystemdResolverConfig - } - - tests := []struct { - name string - clusterCfg kubeadmapi.ClusterConfiguration - expected kubeletConfig - }{ - { - name: "No specific defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{}, - expected: kubeletConfig{ - config: kubeletconfig.KubeletConfiguration{ - FeatureGates: map[string]bool{}, - StaticPodPath: kubeadmapiv1beta2.DefaultManifestsDir, - ClusterDNS: []string{kubeadmapiv1beta2.DefaultClusterDNSIP}, - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: constants.CACertName, - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled), - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled), - }, - }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, - }, - HealthzBindAddress: kubeletHealthzBindAddress, - HealthzPort: utilpointer.Int32Ptr(constants.KubeletHealthzPort), - RotateCertificates: kubeletRotateCertificates, - ResolverConfig: resolverConfig, - }, - }, - }, - { - name: "Service subnet, no dual stack defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - ServiceSubnet: "192.168.0.0/16", - }, - }, - expected: kubeletConfig{ - config: kubeletconfig.KubeletConfiguration{ - FeatureGates: map[string]bool{}, - StaticPodPath: kubeadmapiv1beta2.DefaultManifestsDir, - ClusterDNS: []string{"192.168.0.10"}, - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: constants.CACertName, - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled), - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled), - }, - }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, - }, - HealthzBindAddress: kubeletHealthzBindAddress, - HealthzPort: utilpointer.Int32Ptr(constants.KubeletHealthzPort), - RotateCertificates: kubeletRotateCertificates, - ResolverConfig: resolverConfig, - }, - }, - }, - { - name: "Service subnet, explicitly disabled dual stack defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: false, - }, - Networking: kubeadmapi.Networking{ - ServiceSubnet: "192.168.0.0/16", - }, - }, - expected: kubeletConfig{ - config: kubeletconfig.KubeletConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: false, - }, - StaticPodPath: kubeadmapiv1beta2.DefaultManifestsDir, - ClusterDNS: []string{"192.168.0.10"}, - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: constants.CACertName, - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled), - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled), - }, - }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, - }, - HealthzBindAddress: kubeletHealthzBindAddress, - HealthzPort: utilpointer.Int32Ptr(constants.KubeletHealthzPort), - RotateCertificates: kubeletRotateCertificates, - ResolverConfig: resolverConfig, - }, - }, - }, - { - name: "Service subnet, enabled dual stack defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: true, - }, - Networking: kubeadmapi.Networking{ - ServiceSubnet: "192.168.0.0/16", - }, - }, - expected: kubeletConfig{ - config: kubeletconfig.KubeletConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: true, - }, - StaticPodPath: kubeadmapiv1beta2.DefaultManifestsDir, - ClusterDNS: []string{"192.168.0.10"}, - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: constants.CACertName, - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled), - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled), - }, - }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, - }, - HealthzBindAddress: kubeletHealthzBindAddress, - HealthzPort: utilpointer.Int32Ptr(constants.KubeletHealthzPort), - RotateCertificates: kubeletRotateCertificates, - ResolverConfig: resolverConfig, - }, - }, - }, - { - name: "DNS domain defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - DNSDomain: "example.com", - }, - }, - expected: kubeletConfig{ - config: kubeletconfig.KubeletConfiguration{ - FeatureGates: map[string]bool{}, - StaticPodPath: kubeadmapiv1beta2.DefaultManifestsDir, - ClusterDNS: []string{kubeadmapiv1beta2.DefaultClusterDNSIP}, - ClusterDomain: "example.com", - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: constants.CACertName, - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled), - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled), - }, - }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, - }, - HealthzBindAddress: kubeletHealthzBindAddress, - HealthzPort: utilpointer.Int32Ptr(constants.KubeletHealthzPort), - RotateCertificates: kubeletRotateCertificates, - ResolverConfig: resolverConfig, - }, - }, - }, - { - name: "CertificatesDir defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{ - CertificatesDir: "/path/to/certs", - }, - expected: kubeletConfig{ - config: kubeletconfig.KubeletConfiguration{ - FeatureGates: map[string]bool{}, - StaticPodPath: kubeadmapiv1beta2.DefaultManifestsDir, - ClusterDNS: []string{kubeadmapiv1beta2.DefaultClusterDNSIP}, - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: filepath.Join("/path/to/certs", constants.CACertName), - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationAnonymousEnabled), - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: utilpointer.BoolPtr(kubeletAuthenticationWebhookEnabled), - }, - }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, - }, - HealthzBindAddress: kubeletHealthzBindAddress, - HealthzPort: utilpointer.Int32Ptr(constants.KubeletHealthzPort), - RotateCertificates: kubeletRotateCertificates, - ResolverConfig: resolverConfig, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // This is the same for all test cases so we set it here - expected := test.expected - expected.configBase.GroupVersion = kubeletconfig.SchemeGroupVersion - - got := &kubeletConfig{ - configBase: configBase{ - GroupVersion: kubeletconfig.SchemeGroupVersion, - }, - } - got.Default(&test.clusterCfg, &kubeadmapi.APIEndpoint{}, &kubeadmapi.NodeRegistrationOptions{}) - - if !reflect.DeepEqual(got, &expected) { - t.Fatalf("Missmatch between expected and got:\nExpected:\n%v\n---\nGot:\n%v", expected, *got) - } - }) - } -} - -// runKubeletFromTest holds common test case data and evaluation code for kubeletHandler.From* functions -func runKubeletFromTest(t *testing.T, perform func(gvk schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error)) { - const ( - kind = "KubeletConfiguration" - clusterDomain = "foo.bar" - ) - - gvk := kubeletHandler.GroupVersion.WithKind(kind) - yaml := fmt.Sprintf("apiVersion: %s\nkind: %s\nclusterDomain: %s", kubeletHandler.GroupVersion, kind, clusterDomain) - - cfg, err := perform(gvk, yaml) - - if err != nil { - t.Fatalf("unexpected failure: %v", err) - } - if cfg == nil { - t.Fatal("no config loaded where it should have been") - } - if kubeletCfg, ok := cfg.(*kubeletConfig); !ok { - t.Fatalf("found different object type than expected: %s", reflect.TypeOf(cfg)) - } else if kubeletCfg.config.ClusterDomain != clusterDomain { - t.Fatalf("unexpected control value (clusterDomain):\n\tgot: %q\n\texpected: %q", kubeletCfg.config.ClusterDomain, clusterDomain) - } -} - -func TestKubeletFromDocumentMap(t *testing.T) { - runKubeletFromTest(t, func(gvk schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) { - return kubeletHandler.FromDocumentMap(kubeadmapi.DocumentMap{ - gvk: []byte(yaml), - }) - }) -} - -func TestKubeletFromCluster(t *testing.T) { - runKubeletFromTest(t, func(_ schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) { - client := clientsetfake.NewSimpleClientset( - testKubeletConfigMap(yaml), - ) - return kubeletHandler.FromCluster(client, testClusterCfg()) - }) -} diff --git a/cmd/kubeadm/app/componentconfigs/kubeproxy.go b/cmd/kubeadm/app/componentconfigs/kubeproxy.go deleted file mode 100644 index c53beccdce50c..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/kubeproxy.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "net" - - clientset "k8s.io/client-go/kubernetes" - kubeproxyconfig "k8s.io/kube-proxy/config/v1alpha1" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" -) - -const ( - // KubeProxyGroup is a pointer to the used API group name for the kube-proxy config - KubeProxyGroup = kubeproxyconfig.GroupName - - // kubeproxyKubeConfigFileName is used during defaulting. It's here so it can be accessed from the tests. - kubeproxyKubeConfigFileName = "/var/lib/kube-proxy/kubeconfig.conf" -) - -// kubeProxyHandler is the handler instance for the kube-proxy component config -var kubeProxyHandler = handler{ - GroupVersion: kubeproxyconfig.SchemeGroupVersion, - AddToScheme: kubeproxyconfig.AddToScheme, - CreateEmpty: func() kubeadmapi.ComponentConfig { - return &kubeProxyConfig{ - configBase: configBase{ - GroupVersion: kubeproxyconfig.SchemeGroupVersion, - }, - } - }, - fromCluster: kubeProxyConfigFromCluster, -} - -func kubeProxyConfigFromCluster(h *handler, clientset clientset.Interface, _ *kubeadmapi.ClusterConfiguration) (kubeadmapi.ComponentConfig, error) { - return h.fromConfigMap(clientset, kubeadmconstants.KubeProxyConfigMap, kubeadmconstants.KubeProxyConfigMapKey, false) -} - -// kubeProxyConfig implements the kubeadmapi.ComponentConfig interface for kube-proxy -type kubeProxyConfig struct { - configBase - config kubeproxyconfig.KubeProxyConfiguration -} - -func (kp *kubeProxyConfig) DeepCopy() kubeadmapi.ComponentConfig { - result := &kubeProxyConfig{} - kp.configBase.DeepCopyInto(&result.configBase) - kp.config.DeepCopyInto(&result.config) - return result -} - -func (kp *kubeProxyConfig) Marshal() ([]byte, error) { - return kp.configBase.Marshal(&kp.config) -} - -func (kp *kubeProxyConfig) Unmarshal(docmap kubeadmapi.DocumentMap) error { - return kp.configBase.Unmarshal(docmap, &kp.config) -} - -func kubeProxyDefaultBindAddress(localAdvertiseAddress string) string { - ip := net.ParseIP(localAdvertiseAddress) - if ip.To4() != nil { - return kubeadmapiv1beta2.DefaultProxyBindAddressv4 - } - return kubeadmapiv1beta2.DefaultProxyBindAddressv6 -} - -func (kp *kubeProxyConfig) Default(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint, _ *kubeadmapi.NodeRegistrationOptions) { - const kind = "KubeProxyConfiguration" - - // The below code is necessary because while KubeProxy may be defined, the user may not - // have defined any feature-gates, thus FeatureGates will be nil and the later insertion - // of any feature-gates (e.g. IPv6DualStack) will cause a panic. - if kp.config.FeatureGates == nil { - kp.config.FeatureGates = map[string]bool{} - } - - defaultBindAddress := kubeProxyDefaultBindAddress(localAPIEndpoint.AdvertiseAddress) - if kp.config.BindAddress == "" { - kp.config.BindAddress = defaultBindAddress - } else if kp.config.BindAddress != defaultBindAddress { - warnDefaultComponentConfigValue(kind, "bindAddress", kp.config.BindAddress, defaultBindAddress) - } - - if kp.config.ClusterCIDR == "" && cfg.Networking.PodSubnet != "" { - kp.config.ClusterCIDR = cfg.Networking.PodSubnet - } else if cfg.Networking.PodSubnet != "" && kp.config.ClusterCIDR != cfg.Networking.PodSubnet { - warnDefaultComponentConfigValue(kind, "clusterCIDR", cfg.Networking.PodSubnet, kp.config.ClusterCIDR) - } - - if kp.config.ClientConnection.Kubeconfig == "" { - kp.config.ClientConnection.Kubeconfig = kubeproxyKubeConfigFileName - } else if kp.config.ClientConnection.Kubeconfig != kubeproxyKubeConfigFileName { - warnDefaultComponentConfigValue(kind, "clientConnection.kubeconfig", kubeproxyKubeConfigFileName, kp.config.ClientConnection.Kubeconfig) - } - - // TODO: The following code should be removed after dual-stack is GA. - // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. - if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present { - kp.config.FeatureGates[features.IPv6DualStack] = enabled - } -} diff --git a/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go b/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go deleted file mode 100644 index d68c4c89e168c..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/kubeproxy_test.go +++ /dev/null @@ -1,209 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "fmt" - "reflect" - "testing" - - "github.com/lithammer/dedent" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientsetfake "k8s.io/client-go/kubernetes/fake" - componentbaseconfig "k8s.io/component-base/config/v1alpha1" - kubeproxyconfig "k8s.io/kube-proxy/config/v1alpha1" - - "k8s.io/apimachinery/pkg/runtime/schema" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" -) - -func testKubeProxyConfigMap(contents string) *v1.ConfigMap { - return &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.KubeProxyConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - constants.KubeProxyConfigMapKey: dedent.Dedent(contents), - }, - } -} - -func TestKubeProxyDefault(t *testing.T) { - tests := []struct { - name string - clusterCfg kubeadmapi.ClusterConfiguration - endpoint kubeadmapi.APIEndpoint - expected kubeProxyConfig - }{ - { - name: "No specific defaulting works", - clusterCfg: kubeadmapi.ClusterConfiguration{}, - endpoint: kubeadmapi.APIEndpoint{}, - expected: kubeProxyConfig{ - config: kubeproxyconfig.KubeProxyConfiguration{ - FeatureGates: map[string]bool{}, - BindAddress: kubeadmapiv1beta2.DefaultProxyBindAddressv6, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: kubeproxyKubeConfigFileName, - }, - }, - }, - }, - { - name: "IPv4 bind address", - clusterCfg: kubeadmapi.ClusterConfiguration{}, - endpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - }, - expected: kubeProxyConfig{ - config: kubeproxyconfig.KubeProxyConfiguration{ - FeatureGates: map[string]bool{}, - BindAddress: kubeadmapiv1beta2.DefaultProxyBindAddressv4, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: kubeproxyKubeConfigFileName, - }, - }, - }, - }, - { - name: "ClusterCIDR is fetched from PodSubnet", - clusterCfg: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - PodSubnet: "192.168.0.0/16", - }, - }, - endpoint: kubeadmapi.APIEndpoint{}, - expected: kubeProxyConfig{ - config: kubeproxyconfig.KubeProxyConfiguration{ - FeatureGates: map[string]bool{}, - BindAddress: kubeadmapiv1beta2.DefaultProxyBindAddressv6, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: kubeproxyKubeConfigFileName, - }, - ClusterCIDR: "192.168.0.0/16", - }, - }, - }, - { - name: "IPv6DualStack feature gate set to true", - clusterCfg: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: true, - }, - }, - endpoint: kubeadmapi.APIEndpoint{}, - expected: kubeProxyConfig{ - config: kubeproxyconfig.KubeProxyConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: true, - }, - BindAddress: kubeadmapiv1beta2.DefaultProxyBindAddressv6, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: kubeproxyKubeConfigFileName, - }, - }, - }, - }, - { - name: "IPv6DualStack feature gate set to false", - clusterCfg: kubeadmapi.ClusterConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: false, - }, - }, - endpoint: kubeadmapi.APIEndpoint{}, - expected: kubeProxyConfig{ - config: kubeproxyconfig.KubeProxyConfiguration{ - FeatureGates: map[string]bool{ - features.IPv6DualStack: false, - }, - BindAddress: kubeadmapiv1beta2.DefaultProxyBindAddressv6, - ClientConnection: componentbaseconfig.ClientConnectionConfiguration{ - Kubeconfig: kubeproxyKubeConfigFileName, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // This is the same for all test cases so we set it here - expected := test.expected - expected.configBase.GroupVersion = kubeproxyconfig.SchemeGroupVersion - - got := &kubeProxyConfig{ - configBase: configBase{ - GroupVersion: kubeproxyconfig.SchemeGroupVersion, - }, - } - got.Default(&test.clusterCfg, &test.endpoint, &kubeadmapi.NodeRegistrationOptions{}) - if !reflect.DeepEqual(got, &expected) { - t.Fatalf("Missmatch between expected and got:\nExpected:\n%v\n---\nGot:\n%v", expected, got) - } - }) - } -} - -// runKubeProxyFromTest holds common test case data and evaluation code for kubeProxyHandler.From* functions -func runKubeProxyFromTest(t *testing.T, perform func(gvk schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error)) { - const ( - kind = "KubeProxyConfiguration" - clusterCIDR = "1.2.3.4/16" - ) - - gvk := kubeProxyHandler.GroupVersion.WithKind(kind) - yaml := fmt.Sprintf("apiVersion: %s\nkind: %s\nclusterCIDR: %s", kubeProxyHandler.GroupVersion, kind, clusterCIDR) - - cfg, err := perform(gvk, yaml) - - if err != nil { - t.Fatalf("unexpected failure: %v", err) - } - if cfg == nil { - t.Fatal("no config loaded where it should have been") - } - if kubeproxyCfg, ok := cfg.(*kubeProxyConfig); !ok { - t.Fatalf("found different object type than expected: %s", reflect.TypeOf(cfg)) - } else if kubeproxyCfg.config.ClusterCIDR != clusterCIDR { - t.Fatalf("unexpected control value (clusterDomain):\n\tgot: %q\n\texpected: %q", kubeproxyCfg.config.ClusterCIDR, clusterCIDR) - } -} - -func TestKubeProxyFromDocumentMap(t *testing.T) { - runKubeProxyFromTest(t, func(gvk schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) { - return kubeProxyHandler.FromDocumentMap(kubeadmapi.DocumentMap{ - gvk: []byte(yaml), - }) - }) -} - -func TestKubeProxyFromCluster(t *testing.T) { - runKubeProxyFromTest(t, func(_ schema.GroupVersionKind, yaml string) (kubeadmapi.ComponentConfig, error) { - client := clientsetfake.NewSimpleClientset( - testKubeProxyConfigMap(yaml), - ) - - return kubeProxyHandler.FromCluster(client, testClusterCfg()) - }) -} diff --git a/cmd/kubeadm/app/componentconfigs/scheme.go b/cmd/kubeadm/app/componentconfigs/scheme.go deleted file mode 100644 index c50f79c66bbbe..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/scheme.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" -) - -// Scheme is the runtime.Scheme to which all supported kubeadm ComponentConfig API types are registered. -var Scheme = runtime.NewScheme() - -// Codecs provides access to encoding and decoding for the scheme. -var Codecs = serializer.NewCodecFactory(Scheme) - -func init() { - metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - AddToScheme(Scheme) -} - -// AddToScheme builds the kubeadm ComponentConfig scheme using all known ComponentConfig versions. -func AddToScheme(scheme *runtime.Scheme) { - for _, handler := range known { - utilruntime.Must(handler.AddToScheme(scheme)) - } -} diff --git a/cmd/kubeadm/app/componentconfigs/utils.go b/cmd/kubeadm/app/componentconfigs/utils.go deleted file mode 100644 index c77b78f3fc1b7..0000000000000 --- a/cmd/kubeadm/app/componentconfigs/utils.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package componentconfigs - -import ( - "fmt" - "sort" - "strings" - - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/klog/v2" -) - -// UnsupportedConfigVersionError is a special error type returned whenever we encounter too old config version -type UnsupportedConfigVersionError struct { - // OldVersion is the config version that is causing the problem - OldVersion schema.GroupVersion - - // CurrentVersion describes the natively supported config version - CurrentVersion schema.GroupVersion - - // Document points to the YAML/JSON document that caused the problem - Document []byte -} - -// Error implements the standard Golang error interface for UnsupportedConfigVersionError -func (err *UnsupportedConfigVersionError) Error() string { - return fmt.Sprintf("unsupported apiVersion %q, you may have to do manual conversion to %q and run kubeadm again", err.OldVersion, err.CurrentVersion) -} - -// UnsupportedConfigVersionsErrorMap is a cumulative version of the UnsupportedConfigVersionError type -type UnsupportedConfigVersionsErrorMap map[string]*UnsupportedConfigVersionError - -// Error implements the standard Golang error interface for UnsupportedConfigVersionsErrorMap -func (errs UnsupportedConfigVersionsErrorMap) Error() string { - // Make sure the error messages we print are predictable by sorting them by the group names involved - groups := make([]string, 0, len(errs)) - for group := range errs { - groups = append(groups, group) - } - sort.Strings(groups) - - msgs := make([]string, 1, 1+len(errs)) - msgs[0] = "multiple unsupported config version errors encountered:" - for _, group := range groups { - msgs = append(msgs, errs[group].Error()) - } - - return strings.Join(msgs, "\n\t- ") -} - -// warnDefaultComponentConfigValue prints a warning if the user modified a field in a certain -// CompomentConfig from the default recommended value in kubeadm. -func warnDefaultComponentConfigValue(componentConfigKind, paramName string, defaultValue, userValue interface{}) { - klog.Warningf("The recommended value for %q in %q is: %v; the provided value is: %v", - paramName, componentConfigKind, defaultValue, userValue) -} diff --git a/cmd/kubeadm/app/constants/BUILD b/cmd/kubeadm/app/constants/BUILD deleted file mode 100644 index c75c10130f73f..0000000000000 --- a/cmd/kubeadm/app/constants/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "constants.go", - "constants_unix.go", - "constants_windows.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/constants", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["constants_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go deleted file mode 100644 index bf9f3094d1d80..0000000000000 --- a/cmd/kubeadm/app/constants/constants.go +++ /dev/null @@ -1,669 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -import ( - "fmt" - "io/ioutil" - "net" - "os" - "path" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/apimachinery/pkg/util/wait" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - utilnet "k8s.io/utils/net" -) - -const ( - // KubernetesDir is the directory Kubernetes owns for storing various configuration files - KubernetesDir = "/etc/kubernetes" - // ManifestsSubDirName defines directory name to store manifests - ManifestsSubDirName = "manifests" - // TempDirForKubeadm defines temporary directory for kubeadm - // should be joined with KubernetesDir. - TempDirForKubeadm = "tmp" - - // CertificateValidity defines the validity for all the signed certificates generated by kubeadm - CertificateValidity = time.Hour * 24 * 365 - - // CACertAndKeyBaseName defines certificate authority base name - CACertAndKeyBaseName = "ca" - // CACertName defines certificate name - CACertName = "ca.crt" - // CAKeyName defines certificate name - CAKeyName = "ca.key" - - // APIServerCertAndKeyBaseName defines API's server certificate and key base name - APIServerCertAndKeyBaseName = "apiserver" - // APIServerCertName defines API's server certificate name - APIServerCertName = "apiserver.crt" - // APIServerKeyName defines API's server key name - APIServerKeyName = "apiserver.key" - // APIServerCertCommonName defines API's server certificate common name (CN) - APIServerCertCommonName = "kube-apiserver" - - // APIServerKubeletClientCertAndKeyBaseName defines kubelet client certificate and key base name - APIServerKubeletClientCertAndKeyBaseName = "apiserver-kubelet-client" - // APIServerKubeletClientCertName defines kubelet client certificate name - APIServerKubeletClientCertName = "apiserver-kubelet-client.crt" - // APIServerKubeletClientKeyName defines kubelet client key name - APIServerKubeletClientKeyName = "apiserver-kubelet-client.key" - // APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN) - APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client" - - // EtcdCACertAndKeyBaseName defines etcd's CA certificate and key base name - EtcdCACertAndKeyBaseName = "etcd/ca" - // EtcdCACertName defines etcd's CA certificate name - EtcdCACertName = "etcd/ca.crt" - // EtcdCAKeyName defines etcd's CA key name - EtcdCAKeyName = "etcd/ca.key" - - // EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name - EtcdServerCertAndKeyBaseName = "etcd/server" - // EtcdServerCertName defines etcd's server certificate name - EtcdServerCertName = "etcd/server.crt" - // EtcdServerKeyName defines etcd's server key name - EtcdServerKeyName = "etcd/server.key" - - // EtcdListenClientPort defines the port etcd listen on for client traffic - EtcdListenClientPort = 2379 - // EtcdMetricsPort is the port at which to obtain etcd metrics and health status - EtcdMetricsPort = 2381 - - // EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name - EtcdPeerCertAndKeyBaseName = "etcd/peer" - // EtcdPeerCertName defines etcd's peer certificate name - EtcdPeerCertName = "etcd/peer.crt" - // EtcdPeerKeyName defines etcd's peer key name - EtcdPeerKeyName = "etcd/peer.key" - - // EtcdListenPeerPort defines the port etcd listen on for peer traffic - EtcdListenPeerPort = 2380 - - // EtcdHealthcheckClientCertAndKeyBaseName defines etcd's healthcheck client certificate and key base name - EtcdHealthcheckClientCertAndKeyBaseName = "etcd/healthcheck-client" - // EtcdHealthcheckClientCertName defines etcd's healthcheck client certificate name - EtcdHealthcheckClientCertName = "etcd/healthcheck-client.crt" - // EtcdHealthcheckClientKeyName defines etcd's healthcheck client key name - EtcdHealthcheckClientKeyName = "etcd/healthcheck-client.key" - // EtcdHealthcheckClientCertCommonName defines etcd's healthcheck client certificate common name (CN) - EtcdHealthcheckClientCertCommonName = "kube-etcd-healthcheck-client" - - // APIServerEtcdClientCertAndKeyBaseName defines apiserver's etcd client certificate and key base name - APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client" - // APIServerEtcdClientCertName defines apiserver's etcd client certificate name - APIServerEtcdClientCertName = "apiserver-etcd-client.crt" - // APIServerEtcdClientKeyName defines apiserver's etcd client key name - APIServerEtcdClientKeyName = "apiserver-etcd-client.key" - // APIServerEtcdClientCertCommonName defines apiserver's etcd client certificate common name (CN) - APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client" - - // ServiceAccountKeyBaseName defines SA key base name - ServiceAccountKeyBaseName = "sa" - // ServiceAccountPublicKeyName defines SA public key base name - ServiceAccountPublicKeyName = "sa.pub" - // ServiceAccountPrivateKeyName defines SA private key base name - ServiceAccountPrivateKeyName = "sa.key" - - // FrontProxyCACertAndKeyBaseName defines front proxy CA certificate and key base name - FrontProxyCACertAndKeyBaseName = "front-proxy-ca" - // FrontProxyCACertName defines front proxy CA certificate name - FrontProxyCACertName = "front-proxy-ca.crt" - // FrontProxyCAKeyName defines front proxy CA key name - FrontProxyCAKeyName = "front-proxy-ca.key" - - // FrontProxyClientCertAndKeyBaseName defines front proxy certificate and key base name - FrontProxyClientCertAndKeyBaseName = "front-proxy-client" - // FrontProxyClientCertName defines front proxy certificate name - FrontProxyClientCertName = "front-proxy-client.crt" - // FrontProxyClientKeyName defines front proxy key name - FrontProxyClientKeyName = "front-proxy-client.key" - // FrontProxyClientCertCommonName defines front proxy certificate common name - FrontProxyClientCertCommonName = "front-proxy-client" //used as subject.commonname attribute (CN) - - // AdminKubeConfigFileName defines name for the kubeconfig aimed to be used by the superuser/admin of the cluster - AdminKubeConfigFileName = "admin.conf" - // KubeletBootstrapKubeConfigFileName defines the file name for the kubeconfig that the kubelet will use to do - // the TLS bootstrap to get itself an unique credential - KubeletBootstrapKubeConfigFileName = "bootstrap-kubelet.conf" - - // KubeletKubeConfigFileName defines the file name for the kubeconfig that the control-plane kubelet will use for talking - // to the API server - KubeletKubeConfigFileName = "kubelet.conf" - // ControllerManagerKubeConfigFileName defines the file name for the controller manager's kubeconfig file - ControllerManagerKubeConfigFileName = "controller-manager.conf" - // SchedulerKubeConfigFileName defines the file name for the scheduler's kubeconfig file - SchedulerKubeConfigFileName = "scheduler.conf" - - // Some well-known users and groups in the core Kubernetes authorization system - - // ControllerManagerUser defines the well-known user the controller-manager should be authenticated as - ControllerManagerUser = "system:kube-controller-manager" - // SchedulerUser defines the well-known user the scheduler should be authenticated as - SchedulerUser = "system:kube-scheduler" - // SystemPrivilegedGroup defines the well-known group for the apiservers. This group is also superuser by default - // (i.e. bound to the cluster-admin ClusterRole) - SystemPrivilegedGroup = "system:masters" - // NodesGroup defines the well-known group for all nodes. - NodesGroup = "system:nodes" - // NodesUserPrefix defines the user name prefix as requested by the Node authorizer. - NodesUserPrefix = "system:node:" - // NodesClusterRoleBinding defines the well-known ClusterRoleBinding which binds the too permissive system:node - // ClusterRole to the system:nodes group. Since kubeadm is using the Node Authorizer, this ClusterRoleBinding's - // system:nodes group subject is removed if present. - NodesClusterRoleBinding = "system:node" - - // APICallRetryInterval defines how long kubeadm should wait before retrying a failed API operation - APICallRetryInterval = 500 * time.Millisecond - // DiscoveryRetryInterval specifies how long kubeadm should wait before retrying to connect to the control-plane when doing discovery - DiscoveryRetryInterval = 5 * time.Second - // PatchNodeTimeout specifies how long kubeadm should wait for applying the label and taint on the control-plane before timing out - PatchNodeTimeout = 2 * time.Minute - // TLSBootstrapTimeout specifies how long kubeadm should wait for the kubelet to perform the TLS Bootstrap - TLSBootstrapTimeout = 5 * time.Minute - // TLSBootstrapRetryInterval specifies how long kubeadm should wait before retrying the TLS Bootstrap check - TLSBootstrapRetryInterval = 5 * time.Second - // APICallWithWriteTimeout specifies how long kubeadm should wait for api calls with at least one write - APICallWithWriteTimeout = 40 * time.Second - // APICallWithReadTimeout specifies how long kubeadm should wait for api calls with only reads - APICallWithReadTimeout = 15 * time.Second - // PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed - PullImageRetry = 5 - - // DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm - DefaultControlPlaneTimeout = 4 * time.Minute - - // MinimumAddressesInServiceSubnet defines minimum amount of nodes the Service subnet should allow. - // We need at least ten, because the DNS service is always at the tenth cluster clusterIP - MinimumAddressesInServiceSubnet = 10 - - // MaximumBitsForServiceSubnet defines maximum possible size of the service subnet in terms of bits. - // For example, if the value is 20, then the largest supported service subnet is /12 for IPv4 and /108 for IPv6. - // Note however that anything in between /108 and /112 will be clamped to /112 due to the limitations of the underlying allocation logic. - // TODO: https://github.com/kubernetes/enhancements/pull/1881 - MaximumBitsForServiceSubnet = 20 - - // MinimumAddressesInPodSubnet defines minimum amount of pods in the cluster. - // We need at least more than services, an IPv4 /28 or IPv6 /128 subnet means 14 util addresses - MinimumAddressesInPodSubnet = 14 - - // PodSubnetNodeMaskMaxDiff is limited to 16 due to an issue with uncompressed IP bitmap in core: - // xref: #44918 - // The node subnet mask size must be no more than the pod subnet mask size + 16 - PodSubnetNodeMaskMaxDiff = 16 - - // DefaultTokenDuration specifies the default amount of time that a bootstrap token will be valid - // Default behaviour is 24 hours - DefaultTokenDuration = 24 * time.Hour - - // DefaultCertTokenDuration specifies the default amount of time that the token used by upload certs will be valid - // Default behaviour is 2 hours - DefaultCertTokenDuration = 2 * time.Hour - - // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase - CertificateKeySize = 32 - - // LabelNodeRoleOldControlPlane specifies that a node hosts control-plane components - // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 - LabelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" - - // LabelNodeRoleControlPlane specifies that a node hosts control-plane components - LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" - - // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at - // init/join time for use later. kubeadm annotates the node object with this information - AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" - - // KubeadmConfigConfigMap specifies in what ConfigMap in the kube-system namespace the `kubeadm init` configuration should be stored - KubeadmConfigConfigMap = "kubeadm-config" - - // ClusterConfigurationConfigMapKey specifies in what ConfigMap key the cluster configuration should be stored - ClusterConfigurationConfigMapKey = "ClusterConfiguration" - - // ClusterStatusConfigMapKey specifies in what ConfigMap key the cluster status should be stored - ClusterStatusConfigMapKey = "ClusterStatus" - - // KubeProxyConfigMap specifies in what ConfigMap in the kube-system namespace the kube-proxy configuration should be stored - KubeProxyConfigMap = "kube-proxy" - - // KubeProxyConfigMapKey specifies in what ConfigMap key the component config of kube-proxy should be stored - KubeProxyConfigMapKey = "config.conf" - - // KubeletBaseConfigurationConfigMapPrefix specifies in what ConfigMap in the kube-system namespace the initial remote configuration of kubelet should be stored - KubeletBaseConfigurationConfigMapPrefix = "kubelet-config-" - - // KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the initial remote configuration of kubelet should be stored - KubeletBaseConfigurationConfigMapKey = "kubelet" - - // KubeletBaseConfigMapRolePrefix defines the base kubelet configuration ConfigMap. - KubeletBaseConfigMapRolePrefix = "kubeadm:kubelet-config-" - - // KubeletRunDirectory specifies the directory where the kubelet runtime information is stored. - KubeletRunDirectory = "/var/lib/kubelet" - - // KubeletConfigurationFileName specifies the file name on the node which stores initial remote configuration of kubelet - // This file should exist under KubeletRunDirectory - KubeletConfigurationFileName = "config.yaml" - - // KubeletEnvFileName is a file "kubeadm init" writes at runtime. Using that interface, kubeadm can customize certain - // kubelet flags conditionally based on the environment at runtime. Also, parameters given to the configuration file - // might be passed through this file. "kubeadm init" writes one variable, with the name ${KubeletEnvFileVariableName}. - // This file should exist under KubeletRunDirectory - KubeletEnvFileName = "kubeadm-flags.env" - - // KubeletEnvFileVariableName specifies the shell script variable name "kubeadm init" should write a value to in KubeletEnvFile - KubeletEnvFileVariableName = "KUBELET_KUBEADM_ARGS" - - // KubeletHealthzPort is the port of the kubelet healthz endpoint - KubeletHealthzPort = 10248 - - // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports - MinExternalEtcdVersion = "3.2.18" - - // DefaultEtcdVersion indicates the default etcd version that kubeadm uses - DefaultEtcdVersion = "3.4.13-0" - - // Etcd defines variable used internally when referring to etcd component - Etcd = "etcd" - // KubeAPIServer defines variable used internally when referring to kube-apiserver component - KubeAPIServer = "kube-apiserver" - // KubeControllerManager defines variable used internally when referring to kube-controller-manager component - KubeControllerManager = "kube-controller-manager" - // KubeScheduler defines variable used internally when referring to kube-scheduler component - KubeScheduler = "kube-scheduler" - // KubeProxy defines variable used internally when referring to kube-proxy component - KubeProxy = "kube-proxy" - // HyperKube defines variable used internally when referring to the hyperkube image - HyperKube = "hyperkube" - // CoreDNS defines variable used internally when referring to the CoreDNS component - CoreDNS = "CoreDNS" - // KubeDNS defines variable used internally when referring to the KubeDNS component - KubeDNS = "kube-dns" - // Kubelet defines variable used internally when referring to the Kubelet - Kubelet = "kubelet" - - // SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has - SelfHostingPrefix = "self-hosted-" - - // KubeCertificatesVolumeName specifies the name for the Volume that is used for injecting certificates to control plane components (can be both a hostPath volume or a projected, all-in-one volume) - KubeCertificatesVolumeName = "k8s-certs" - - // KubeConfigVolumeName specifies the name for the Volume that is used for injecting the kubeconfig to talk securely to the api server for a control plane component if applicable - KubeConfigVolumeName = "kubeconfig" - - // NodeBootstrapTokenAuthGroup specifies which group a Node Bootstrap Token should be authenticated in - NodeBootstrapTokenAuthGroup = "system:bootstrappers:kubeadm:default-node-token" - - // DefaultCIImageRepository points to image registry where CI uploads images from ci-cross build job - DefaultCIImageRepository = "gcr.io/kubernetes-ci-images" - - // CoreDNSConfigMap specifies in what ConfigMap in the kube-system namespace the CoreDNS config should be stored - CoreDNSConfigMap = "coredns" - - // CoreDNSDeploymentName specifies the name of the Deployment for CoreDNS add-on - CoreDNSDeploymentName = "coredns" - - // CoreDNSImageName specifies the name of the image for CoreDNS add-on - CoreDNSImageName = "coredns" - - // KubeDNSConfigMap specifies in what ConfigMap in the kube-system namespace the kube-dns config should be stored - KubeDNSConfigMap = "kube-dns" - - // KubeDNSDeploymentName specifies the name of the Deployment for kube-dns add-on - KubeDNSDeploymentName = "kube-dns" - - // KubeDNSKubeDNSImageName specifies the name of the image for the kubedns container in the kube-dns add-on - KubeDNSKubeDNSImageName = "k8s-dns-kube-dns" - - // KubeDNSSidecarImageName specifies the name of the image for the sidecar container in the kube-dns add-on - KubeDNSSidecarImageName = "k8s-dns-sidecar" - - // KubeDNSDnsMasqNannyImageName specifies the name of the image for the dnsmasq container in the kube-dns add-on - KubeDNSDnsMasqNannyImageName = "k8s-dns-dnsmasq-nanny" - - // KubeDNSVersion is the version of kube-dns to be deployed if it is used - KubeDNSVersion = "1.14.13" - - // CoreDNSVersion is the version of CoreDNS to be deployed if it is used - CoreDNSVersion = "1.7.0" - - // ClusterConfigurationKind is the string kind value for the ClusterConfiguration struct - ClusterConfigurationKind = "ClusterConfiguration" - - // InitConfigurationKind is the string kind value for the InitConfiguration struct - InitConfigurationKind = "InitConfiguration" - - // JoinConfigurationKind is the string kind value for the JoinConfiguration struct - JoinConfigurationKind = "JoinConfiguration" - - // YAMLDocumentSeparator is the separator for YAML documents - // TODO: Find a better place for this constant - YAMLDocumentSeparator = "---\n" - - // DefaultAPIServerBindAddress is the default bind address for the API Server - DefaultAPIServerBindAddress = "0.0.0.0" - - // ControlPlaneNumCPU is the number of CPUs required on control-plane - ControlPlaneNumCPU = 2 - - // ControlPlaneMem is the number of megabytes of memory required on the control-plane - // Below that amount of RAM running a stable control plane would be difficult. - ControlPlaneMem = 1700 - - // KubeadmCertsSecret specifies in what Secret in the kube-system namespace the certificates should be stored - KubeadmCertsSecret = "kubeadm-certs" - - // KubeletPort is the default port for the kubelet server on each host machine. - // May be overridden by a flag at startup. - KubeletPort = 10250 - // KubeSchedulerPort is the default port for the scheduler status server. - // May be overridden by a flag at startup. - KubeSchedulerPort = 10259 - // KubeControllerManagerPort is the default port for the controller manager status server. - // May be overridden by a flag at startup. - KubeControllerManagerPort = 10257 - - // EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the - // advertise client URLs - EtcdAdvertiseClientUrlsAnnotationKey = "kubeadm.kubernetes.io/etcd.advertise-client-urls" - // KubeAPIServerAdvertiseAddressEndpointAnnotationKey is the annotation key on every apiserver pod, - // describing the API endpoint (advertise address and bind port of the api server) - KubeAPIServerAdvertiseAddressEndpointAnnotationKey = "kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint" - // ComponentConfigHashAnnotationKey holds the config map annotation key that kubeadm uses to store - // a SHA256 sum to check for user changes - ComponentConfigHashAnnotationKey = "kubeadm.kubernetes.io/component-config.hash" - - // ControlPlaneTier is the value used in the tier label to identify control plane components - ControlPlaneTier = "control-plane" - - // Mode* constants were copied from pkg/kubeapiserver/authorizer/modes - // to avoid kubeadm dependency on the internal module - // TODO: share Mode* constants in component config - - // ModeAlwaysAllow is the mode to set all requests as authorized - ModeAlwaysAllow string = "AlwaysAllow" - // ModeAlwaysDeny is the mode to set no requests as authorized - ModeAlwaysDeny string = "AlwaysDeny" - // ModeABAC is the mode to use Attribute Based Access Control to authorize - ModeABAC string = "ABAC" - // ModeWebhook is the mode to make an external webhook call to authorize - ModeWebhook string = "Webhook" - // ModeRBAC is the mode to use Role Based Access Control to authorize - ModeRBAC string = "RBAC" - // ModeNode is an authorization mode that authorizes API requests made by kubelets. - ModeNode string = "Node" -) - -var ( - // OldControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane - // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 - OldControlPlaneTaint = v1.Taint{ - Key: LabelNodeRoleOldControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // OldControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane - // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 - OldControlPlaneToleration = v1.Toleration{ - Key: LabelNodeRoleOldControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // ControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane - ControlPlaneTaint = v1.Taint{ - Key: LabelNodeRoleControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // ControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane - ControlPlaneToleration = v1.Toleration{ - Key: LabelNodeRoleControlPlane, - Effect: v1.TaintEffectNoSchedule, - } - - // DefaultTokenUsages specifies the default functions a token will get - DefaultTokenUsages = bootstrapapi.KnownTokenUsages - - // DefaultTokenGroups specifies the default groups that this token will authenticate as when used for authentication - DefaultTokenGroups = []string{NodeBootstrapTokenAuthGroup} - - // ControlPlaneComponents defines the control-plane component names - ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler} - - // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy - MinimumControlPlaneVersion = version.MustParseSemantic("v1.19.0") - - // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports - MinimumKubeletVersion = version.MustParseSemantic("v1.19.0") - - // CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm - CurrentKubernetesVersion = version.MustParseSemantic("v1.20.0") - - // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases - SupportedEtcdVersion = map[uint8]string{ - 13: "3.2.24", - 14: "3.3.10", - 15: "3.3.10", - 16: "3.3.17-0", - 17: "3.4.3-0", - 18: "3.4.3-0", - 19: "3.4.9-1", - 20: "3.4.13-0", - 21: "3.4.13-0", - } - - // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows - // the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane - KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret) - - // StaticPodMirroringDefaultRetry is used a backoff strategy for - // waiting for static pods to be mirrored to the apiserver. - StaticPodMirroringDefaultRetry = wait.Backoff{ - Steps: 30, - Duration: 1 * time.Second, - Factor: 1.0, - Jitter: 0.1, - } -) - -// EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release -// If passed version is not in the given list, the function returns the nearest version with a warning -func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) { - kubernetesVersion, err := version.ParseSemantic(versionString) - if err != nil { - return nil, nil, err - } - desiredVersion, etcdStringVersion := uint8(kubernetesVersion.Minor()), "" - - min, max := ^uint8(0), uint8(0) - for k, v := range supportedEtcdVersion { - if desiredVersion == k { - etcdStringVersion = v - break - } - if k < min { - min = k - } - if k > max { - max = k - } - } - - if len(etcdStringVersion) == 0 { - if desiredVersion < min { - etcdStringVersion = supportedEtcdVersion[min] - } - if desiredVersion > max { - etcdStringVersion = supportedEtcdVersion[max] - } - warning = fmt.Errorf("could not find officially supported version of etcd for Kubernetes %s, falling back to the nearest etcd version (%s)", - versionString, etcdStringVersion) - } - - etcdVersion, err = version.ParseSemantic(etcdStringVersion) - if err != nil { - return nil, nil, err - } - - return etcdVersion, warning, nil -} - -// GetStaticPodDirectory returns the location on the disk where the Static Pod should be present -func GetStaticPodDirectory() string { - return filepath.Join(KubernetesDir, ManifestsSubDirName) -} - -// GetStaticPodFilepath returns the location on the disk where the Static Pod should be present -func GetStaticPodFilepath(componentName, manifestsDir string) string { - return filepath.Join(manifestsDir, componentName+".yaml") -} - -// GetAdminKubeConfigPath returns the location on the disk where admin kubeconfig is located by default -func GetAdminKubeConfigPath() string { - return filepath.Join(KubernetesDir, AdminKubeConfigFileName) -} - -// GetBootstrapKubeletKubeConfigPath returns the location on the disk where bootstrap kubelet kubeconfig is located by default -func GetBootstrapKubeletKubeConfigPath() string { - return filepath.Join(KubernetesDir, KubeletBootstrapKubeConfigFileName) -} - -// GetKubeletKubeConfigPath returns the location on the disk where kubelet kubeconfig is located by default -func GetKubeletKubeConfigPath() string { - return filepath.Join(KubernetesDir, KubeletKubeConfigFileName) -} - -// AddSelfHostedPrefix adds the self-hosted- prefix to the component name -func AddSelfHostedPrefix(componentName string) string { - return fmt.Sprintf("%s%s", SelfHostingPrefix, componentName) -} - -// CreateTempDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp (not using /tmp as that would potentially be dangerous) -func CreateTempDirForKubeadm(kubernetesDir, dirName string) (string, error) { - tempDir := path.Join(KubernetesDir, TempDirForKubeadm) - if len(kubernetesDir) != 0 { - tempDir = path.Join(kubernetesDir, TempDirForKubeadm) - } - - // creates target folder if not already exists - if err := os.MkdirAll(tempDir, 0700); err != nil { - return "", errors.Wrapf(err, "failed to create directory %q", tempDir) - } - - tempDir, err := ioutil.TempDir(tempDir, dirName) - if err != nil { - return "", errors.Wrap(err, "couldn't create a temporary directory") - } - return tempDir, nil -} - -// CreateTimestampDirForKubeadm is a function that creates a temporary directory under /etc/kubernetes/tmp formatted with the current date -func CreateTimestampDirForKubeadm(kubernetesDir, dirName string) (string, error) { - tempDir := path.Join(KubernetesDir, TempDirForKubeadm) - if len(kubernetesDir) != 0 { - tempDir = path.Join(kubernetesDir, TempDirForKubeadm) - } - - // creates target folder if not already exists - if err := os.MkdirAll(tempDir, 0700); err != nil { - return "", errors.Wrapf(err, "failed to create directory %q", tempDir) - } - - timestampDirName := fmt.Sprintf("%s-%s", dirName, time.Now().Format("2006-01-02-15-04-05")) - timestampDir := path.Join(tempDir, timestampDirName) - if err := os.Mkdir(timestampDir, 0700); err != nil { - return "", errors.Wrap(err, "could not create timestamp directory") - } - - return timestampDir, nil -} - -// GetDNSIP returns a dnsIP, which is 10th IP in svcSubnet CIDR range -func GetDNSIP(svcSubnetList string, isDualStack bool) (net.IP, error) { - // Get the service subnet CIDR - svcSubnetCIDR, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) - if err != nil { - return nil, errors.Wrapf(err, "unable to get internal Kubernetes Service IP from the given service CIDR (%s)", svcSubnetList) - } - - // Selects the 10th IP in service subnet CIDR range as dnsIP - dnsIP, err := utilnet.GetIndexedIP(svcSubnetCIDR, 10) - if err != nil { - return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") - } - - return dnsIP, nil -} - -// GetKubernetesServiceCIDR returns the default Service CIDR for the Kubernetes internal service -func GetKubernetesServiceCIDR(svcSubnetList string, isDualStack bool) (*net.IPNet, error) { - if isDualStack { - // The default service address family for the cluster is the address family of the first - // service cluster IP range configured via the `--service-cluster-ip-range` flag - // of the kube-controller-manager and kube-apiserver. - svcSubnets, err := utilnet.ParseCIDRs(strings.Split(svcSubnetList, ",")) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) - } - if len(svcSubnets) == 0 { - return nil, errors.New("received empty ServiceSubnet for dual-stack") - } - return svcSubnets[0], nil - } - // internal IP address for the API server - _, svcSubnet, err := net.ParseCIDR(svcSubnetList) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse ServiceSubnet %v", svcSubnetList) - } - return svcSubnet, nil -} - -// GetAPIServerVirtualIP returns the IP of the internal Kubernetes API service -func GetAPIServerVirtualIP(svcSubnetList string, isDualStack bool) (net.IP, error) { - svcSubnet, err := GetKubernetesServiceCIDR(svcSubnetList, isDualStack) - if err != nil { - return nil, errors.Wrap(err, "unable to get internal Kubernetes Service IP from the given service CIDR") - } - internalAPIServerVirtualIP, err := utilnet.GetIndexedIP(svcSubnet, 1) - if err != nil { - return nil, errors.Wrapf(err, "unable to get the first IP address from the given CIDR: %s", svcSubnet.String()) - } - return internalAPIServerVirtualIP, nil -} - -// GetDNSVersion is a handy function that returns the DNS version by DNS type -func GetDNSVersion(dnsType kubeadmapi.DNSAddOnType) string { - switch dnsType { - case kubeadmapi.KubeDNS: - return KubeDNSVersion - default: - return CoreDNSVersion - } -} - -// GetKubeletConfigMapName returns the right ConfigMap name for the right branch of k8s -func GetKubeletConfigMapName(k8sVersion *version.Version) string { - return fmt.Sprintf("%s%d.%d", KubeletBaseConfigurationConfigMapPrefix, k8sVersion.Major(), k8sVersion.Minor()) -} diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go deleted file mode 100644 index 83bc8814d63bc..0000000000000 --- a/cmd/kubeadm/app/constants/constants_test.go +++ /dev/null @@ -1,303 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -import ( - "path/filepath" - "testing" - - "k8s.io/apimachinery/pkg/util/version" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestGetStaticPodDirectory(t *testing.T) { - expected := "/etc/kubernetes/manifests" - actual := GetStaticPodDirectory() - - if actual != expected { - t.Errorf( - "failed GetStaticPodDirectory:\n\texpected: %s\n\t actual: %s", - expected, - actual, - ) - } -} - -func TestGetAdminKubeConfigPath(t *testing.T) { - expected := filepath.Join(KubernetesDir, AdminKubeConfigFileName) - actual := GetAdminKubeConfigPath() - - if actual != expected { - t.Errorf( - "failed GetAdminKubeConfigPath:\n\texpected: %s\n\t actual: %s", - expected, - actual, - ) - } -} - -func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) { - expected := "/etc/kubernetes/bootstrap-kubelet.conf" - actual := GetBootstrapKubeletKubeConfigPath() - - if actual != expected { - t.Errorf( - "failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", - expected, - actual, - ) - } -} - -func TestGetKubeletKubeConfigPath(t *testing.T) { - expected := "/etc/kubernetes/kubelet.conf" - actual := GetKubeletKubeConfigPath() - - if actual != expected { - t.Errorf( - "failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t actual: %s", - expected, - actual, - ) - } -} - -func TestGetStaticPodFilepath(t *testing.T) { - var tests = []struct { - componentName, manifestsDir, expected string - }{ - { - componentName: "kube-apiserver", - manifestsDir: "/etc/kubernetes/manifests", - expected: "/etc/kubernetes/manifests/kube-apiserver.yaml", - }, - { - componentName: "kube-controller-manager", - manifestsDir: "/etc/kubernetes/manifests/", - expected: "/etc/kubernetes/manifests/kube-controller-manager.yaml", - }, - { - componentName: "foo", - manifestsDir: "/etc/bar/", - expected: "/etc/bar/foo.yaml", - }, - } - for _, rt := range tests { - t.Run(rt.componentName, func(t *testing.T) { - actual := GetStaticPodFilepath(rt.componentName, rt.manifestsDir) - if actual != rt.expected { - t.Errorf( - "failed GetStaticPodFilepath:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - }) - } -} - -func TestAddSelfHostedPrefix(t *testing.T) { - var tests = []struct { - componentName, expected string - }{ - { - componentName: "kube-apiserver", - expected: "self-hosted-kube-apiserver", - }, - { - componentName: "kube-controller-manager", - expected: "self-hosted-kube-controller-manager", - }, - { - componentName: "kube-scheduler", - expected: "self-hosted-kube-scheduler", - }, - { - componentName: "foo", - expected: "self-hosted-foo", - }, - } - for _, rt := range tests { - t.Run(rt.componentName, func(t *testing.T) { - actual := AddSelfHostedPrefix(rt.componentName) - if actual != rt.expected { - t.Errorf( - "failed AddSelfHostedPrefix:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - }) - } -} - -func TestEtcdSupportedVersion(t *testing.T) { - var supportedEtcdVersion = map[uint8]string{ - 13: "3.2.24", - 14: "3.3.10", - 15: "3.3.10", - 16: "3.3.17-0", - 17: "3.4.3-0", - 18: "3.4.3-0", - } - var tests = []struct { - kubernetesVersion string - expectedVersion *version.Version - expectedWarning bool - expectedError bool - }{ - { - kubernetesVersion: "1.x.1", - expectedVersion: nil, - expectedWarning: false, - expectedError: true, - }, - { - kubernetesVersion: "1.10.1", - expectedVersion: version.MustParseSemantic("3.2.24"), - expectedWarning: true, - expectedError: false, - }, - { - kubernetesVersion: "1.99.0", - expectedVersion: version.MustParseSemantic("3.4.3-0"), - expectedWarning: true, - expectedError: false, - }, - { - kubernetesVersion: "v1.16.0", - expectedVersion: version.MustParseSemantic("3.3.17-0"), - expectedWarning: false, - expectedError: false, - }, - { - kubernetesVersion: "1.17.2", - expectedVersion: version.MustParseSemantic("3.4.3-0"), - expectedWarning: false, - expectedError: false, - }, - } - for _, rt := range tests { - t.Run(rt.kubernetesVersion, func(t *testing.T) { - actualVersion, actualWarning, actualError := EtcdSupportedVersion(supportedEtcdVersion, rt.kubernetesVersion) - if (actualError != nil) != rt.expectedError { - t.Fatalf("expected error %v, got %v", rt.expectedError, actualError != nil) - } - if (actualWarning != nil) != rt.expectedWarning { - t.Fatalf("expected warning %v, got %v", rt.expectedWarning, actualWarning != nil) - } - if actualError == nil && actualVersion.String() != rt.expectedVersion.String() { - t.Errorf("expected version %s, got %s", rt.expectedVersion.String(), actualVersion.String()) - } - }) - } -} - -func TestGetKubeDNSVersion(t *testing.T) { - var tests = []struct { - dns kubeadmapi.DNSAddOnType - expected string - }{ - { - dns: kubeadmapi.KubeDNS, - expected: KubeDNSVersion, - }, - { - dns: kubeadmapi.CoreDNS, - expected: CoreDNSVersion, - }, - } - for _, rt := range tests { - t.Run(string(rt.dns), func(t *testing.T) { - actualDNSVersion := GetDNSVersion(rt.dns) - if actualDNSVersion != rt.expected { - t.Errorf( - "failed GetDNSVersion:\n\texpected: %s\n\t actual: %s", - rt.expected, - actualDNSVersion, - ) - } - }) - } -} - -func TestGetKubernetesServiceCIDR(t *testing.T) { - var tests = []struct { - svcSubnetList string - isDualStack bool - expected string - expectedError bool - name string - }{ - { - svcSubnetList: "192.168.10.0/24", - isDualStack: false, - expected: "192.168.10.0/24", - expectedError: false, - name: "valid: valid IPv4 range from single-stack", - }, - { - svcSubnetList: "fd03::/112", - isDualStack: false, - expected: "fd03::/112", - expectedError: false, - name: "valid: valid IPv6 range from single-stack", - }, - { - svcSubnetList: "192.168.10.0/24,fd03::/112", - isDualStack: true, - expected: "192.168.10.0/24", - expectedError: false, - name: "valid: valid ranges from dual-stack", - }, - { - svcSubnetList: "fd03::/112,192.168.10.0/24", - isDualStack: true, - expected: "fd03::/112", - expectedError: false, - name: "valid: valid ranges from dual-stack", - }, - { - svcSubnetList: "192.168.10.0/24,fd03:x::/112", - isDualStack: true, - expected: "", - expectedError: true, - name: "invalid: failed to parse subnet range for dual-stack", - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual, actualError := GetKubernetesServiceCIDR(rt.svcSubnetList, rt.isDualStack) - if rt.expectedError { - if actualError == nil { - t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected error, but got no error") - } - } else if !rt.expectedError && actualError != nil { - t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected no error, but got: %v", actualError) - } else { - if actual.String() != rt.expected { - t.Errorf( - "failed GetKubernetesServiceCIDR:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual.String(), - ) - } - } - }) - } -} diff --git a/cmd/kubeadm/app/constants/constants_unix.go b/cmd/kubeadm/app/constants/constants_unix.go deleted file mode 100644 index b024348c29a12..0000000000000 --- a/cmd/kubeadm/app/constants/constants_unix.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build !windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -const ( - // DefaultDockerCRISocket defines the default Docker CRI socket - DefaultDockerCRISocket = "/var/run/dockershim.sock" - - // PauseVersion indicates the default pause image version for kubeadm - PauseVersion = "3.2" -) diff --git a/cmd/kubeadm/app/constants/constants_windows.go b/cmd/kubeadm/app/constants/constants_windows.go deleted file mode 100644 index e8b2494490a70..0000000000000 --- a/cmd/kubeadm/app/constants/constants_windows.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package constants - -const ( - // DefaultDockerCRISocket defines the default Docker CRI socket - DefaultDockerCRISocket = "npipe:////./pipe/docker_engine" - - // PauseVersion indicates the default pause image version for kubeadm - PauseVersion = "1.4.0" -) diff --git a/cmd/kubeadm/app/discovery/BUILD b/cmd/kubeadm/app/discovery/BUILD deleted file mode 100644 index de9e0ef0d90b8..0000000000000 --- a/cmd/kubeadm/app/discovery/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["discovery.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/discovery", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/discovery/file:go_default_library", - "//cmd/kubeadm/app/discovery/https:go_default_library", - "//cmd/kubeadm/app/discovery/token:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["discovery_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/discovery/file:all-srcs", - "//cmd/kubeadm/app/discovery/https:all-srcs", - "//cmd/kubeadm/app/discovery/token:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/discovery/discovery.go b/cmd/kubeadm/app/discovery/discovery.go deleted file mode 100644 index 010c3c3a058bd..0000000000000 --- a/cmd/kubeadm/app/discovery/discovery.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package discovery - -import ( - "net/url" - - "github.com/pkg/errors" - - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/discovery/file" - "k8s.io/kubernetes/cmd/kubeadm/app/discovery/https" - "k8s.io/kubernetes/cmd/kubeadm/app/discovery/token" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -// TokenUser defines token user -const TokenUser = "tls-bootstrap-token-user" - -// For returns a kubeconfig object that can be used for doing the TLS Bootstrap with the right credentials -// Also, before returning anything, it makes sure it can trust the API Server -func For(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) { - // TODO: Print summary info about the CA certificate, along with the checksum signature - // we also need an ability for the user to configure the client to validate received CA cert against a checksum - config, err := DiscoverValidatedKubeConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "couldn't validate the identity of the API Server") - } - - // If the users has provided a TLSBootstrapToken use it for the join process. - // This is usually the case of Token discovery, but it can also be used with a discovery file - // without embedded authentication credentials. - if len(cfg.Discovery.TLSBootstrapToken) != 0 { - klog.V(1).Info("[discovery] Using provided TLSBootstrapToken as authentication credentials for the join process") - - clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(config) - return kubeconfigutil.CreateWithToken( - clusterinfo.Server, - kubeadmapiv1beta2.DefaultClusterName, - TokenUser, - clusterinfo.CertificateAuthorityData, - cfg.Discovery.TLSBootstrapToken, - ), nil - } - - // if the config returned from discovery has authentication credentials, proceed with the TLS boostrap process - if kubeconfigutil.HasAuthenticationCredentials(config) { - return config, nil - } - - // if there are no authentication credentials (nor in the config returned from discovery, nor in the TLSBootstrapToken), fail - return nil, errors.New("couldn't find authentication credentials for the TLS boostrap process. Please use Token discovery, a discovery file with embedded authentication credentials or a discovery file without authentication credentials but with the TLSBootstrapToken flag") -} - -// DiscoverValidatedKubeConfig returns a validated Config object that specifies where the cluster is and the CA cert to trust -func DiscoverValidatedKubeConfig(cfg *kubeadmapi.JoinConfiguration) (*clientcmdapi.Config, error) { - switch { - case cfg.Discovery.File != nil: - kubeConfigPath := cfg.Discovery.File.KubeConfigPath - if isHTTPSURL(kubeConfigPath) { - return https.RetrieveValidatedConfigInfo(kubeConfigPath, kubeadmapiv1beta2.DefaultClusterName, cfg.Discovery.Timeout.Duration) - } - return file.RetrieveValidatedConfigInfo(kubeConfigPath, kubeadmapiv1beta2.DefaultClusterName, cfg.Discovery.Timeout.Duration) - case cfg.Discovery.BootstrapToken != nil: - return token.RetrieveValidatedConfigInfo(&cfg.Discovery) - default: - return nil, errors.New("couldn't find a valid discovery configuration") - } -} - -// isHTTPSURL checks whether the string is parsable as a URL and whether the Scheme is https -func isHTTPSURL(s string) bool { - u, err := url.Parse(s) - return err == nil && u.Scheme == "https" -} diff --git a/cmd/kubeadm/app/discovery/discovery_test.go b/cmd/kubeadm/app/discovery/discovery_test.go deleted file mode 100644 index f6daeffddba56..0000000000000 --- a/cmd/kubeadm/app/discovery/discovery_test.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package discovery - -import ( - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestFor(t *testing.T) { - tests := []struct { - name string - d kubeadm.JoinConfiguration - expect bool - }{ - { - name: "default Discovery", - d: kubeadm.JoinConfiguration{}, - expect: false, - }, - { - name: "file Discovery with a path", - d: kubeadm.JoinConfiguration{ - Discovery: kubeadm.Discovery{ - File: &kubeadm.FileDiscovery{ - KubeConfigPath: "notnil", - }, - }, - }, - expect: false, - }, - { - name: "file Discovery with an url", - d: kubeadm.JoinConfiguration{ - Discovery: kubeadm.Discovery{ - File: &kubeadm.FileDiscovery{ - KubeConfigPath: "https://localhost", - }, - }, - }, - expect: false, - }, - { - name: "BootstrapTokenDiscovery", - d: kubeadm.JoinConfiguration{ - Discovery: kubeadm.Discovery{ - BootstrapToken: &kubeadm.BootstrapTokenDiscovery{ - Token: "foo.bar@foobar", - }, - }, - }, - expect: false, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - config := rt.d - config.Discovery.Timeout = &metav1.Duration{Duration: 5 * time.Minute} - _, actual := For(&config) - if (actual == nil) != rt.expect { - t.Errorf( - "failed For:\n\texpected: %t\n\t actual: %t", - rt.expect, - (actual == nil), - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/discovery/file/BUILD b/cmd/kubeadm/app/discovery/file/BUILD deleted file mode 100644 index 2833c16c537f3..0000000000000 --- a/cmd/kubeadm/app/discovery/file/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["file.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/discovery/file", - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/discovery/file/file.go b/cmd/kubeadm/app/discovery/file/file.go deleted file mode 100644 index cbc6240d4fa03..0000000000000 --- a/cmd/kubeadm/app/discovery/file/file.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package file - -import ( - "context" - "time" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" -) - -// RetrieveValidatedConfigInfo connects to the API Server and makes sure it can talk -// securely to the API Server using the provided CA cert and -// optionally refreshes the cluster-info information from the cluster-info ConfigMap -func RetrieveValidatedConfigInfo(filepath, clustername string, discoveryTimeout time.Duration) (*clientcmdapi.Config, error) { - config, err := clientcmd.LoadFromFile(filepath) - if err != nil { - return nil, err - } - return ValidateConfigInfo(config, clustername, discoveryTimeout) -} - -// ValidateConfigInfo connects to the API Server and makes sure it can talk -// securely to the API Server using the provided CA cert/client certificates and -// optionally refreshes the cluster-info information from the cluster-info ConfigMap -func ValidateConfigInfo(config *clientcmdapi.Config, clustername string, discoveryTimeout time.Duration) (*clientcmdapi.Config, error) { - if len(config.Clusters) < 1 { - return nil, errors.New("the provided kubeconfig file must have at least one Cluster defined") - } - currentCluster := kubeconfigutil.GetClusterFromKubeConfig(config) - if currentCluster == nil { - return nil, errors.New("the provided kubeconfig file must have a unnamed Cluster or a CurrentContext that specifies a non-nil Cluster") - } - if err := clientcmd.Validate(*config); err != nil { - return nil, err - } - - // If the kubeconfig points to a file for the CA, make sure the CA file contents are embedded - if err := kubeconfigutil.EnsureCertificateAuthorityIsEmbedded(currentCluster); err != nil { - return nil, err - } - - // If the discovery file config contains authentication credentials - if kubeconfigutil.HasAuthenticationCredentials(config) { - klog.V(1).Info("[discovery] Using authentication credentials from the discovery file for validating TLS connection") - - // We should ensure that all the authentication info is embedded in config file, so everything will work also when - // the kubeconfig file will be stored in /etc/kubernetes/boostrap-kubelet.conf - if err := kubeconfigutil.EnsureAuthenticationInfoAreEmbedded(config); err != nil { - return nil, err - } - } else { - // If the discovery file config does not contains authentication credentials - klog.V(1).Info("[discovery] Discovery file does not contains authentication credentials, using unauthenticated request for validating TLS connection") - - // Create a new kubeconfig object from the discovery file config, with only the server and the CA cert. - // NB. We do this in order to not pick up other possible misconfigurations in the clusterinfo file - config = kubeconfigutil.CreateBasic( - currentCluster.Server, - clustername, - "", // no user provided - currentCluster.CertificateAuthorityData, - ) - } - - // Try to read the cluster-info config map; this step was required by the original design in order - // to validate the TLS connection to the server early in the process - client, err := kubeconfigutil.ToClientSet(config) - if err != nil { - return nil, err - } - - klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q\n", currentCluster.Server) - - var clusterinfoCM *v1.ConfigMap - - err = wait.Poll(constants.DiscoveryRetryInterval, discoveryTimeout, func() (bool, error) { - var err error - clusterinfoCM, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.TODO(), bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - if apierrors.IsForbidden(err) { - // If the request is unauthorized, the cluster admin has not granted access to the cluster info configmap for unauthenticated users - // In that case, trust the cluster admin and do not refresh the cluster-info data - klog.Warningf("[discovery] Could not access the %s ConfigMap for refreshing the cluster-info information, but the TLS cert is valid so proceeding...\n", bootstrapapi.ConfigMapClusterInfo) - return true, nil - } - klog.V(1).Infof("[discovery] Error reading the %s ConfigMap, will try again: %v\n", bootstrapapi.ConfigMapClusterInfo, err) - return false, nil - } - return true, nil - }) - if err == wait.ErrWaitTimeout { - return nil, errors.Errorf("Abort reading the %s ConfigMap after timeout of %v", bootstrapapi.ConfigMapClusterInfo, discoveryTimeout) - } - - // If we couldn't fetch the cluster-info ConfigMap, just return the cluster-info object the user provided - if clusterinfoCM == nil { - return config, nil - } - - // We somehow got hold of the ConfigMap, try to read some data from it. If we can't, fallback on the user-provided file - refreshedBaseKubeConfig, err := tryParseClusterInfoFromConfigMap(clusterinfoCM) - if err != nil { - klog.V(1).Infof("[discovery] The %s ConfigMap isn't set up properly (%v), but the TLS cert is valid so proceeding...\n", bootstrapapi.ConfigMapClusterInfo, err) - return config, nil - } - - refreshedCluster := kubeconfigutil.GetClusterFromKubeConfig(refreshedBaseKubeConfig) - currentCluster.Server = refreshedCluster.Server - currentCluster.CertificateAuthorityData = refreshedCluster.CertificateAuthorityData - - klog.V(1).Infof("[discovery] Synced Server and CertificateAuthorityData from the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - return config, nil -} - -// tryParseClusterInfoFromConfigMap tries to parse a kubeconfig file from a ConfigMap key -func tryParseClusterInfoFromConfigMap(cm *v1.ConfigMap) (*clientcmdapi.Config, error) { - kubeConfigString, ok := cm.Data[bootstrapapi.KubeConfigKey] - if !ok || len(kubeConfigString) == 0 { - return nil, errors.Errorf("no %s key in ConfigMap", bootstrapapi.KubeConfigKey) - } - parsedKubeConfig, err := clientcmd.Load([]byte(kubeConfigString)) - if err != nil { - return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - } - return parsedKubeConfig, nil -} diff --git a/cmd/kubeadm/app/discovery/https/BUILD b/cmd/kubeadm/app/discovery/https/BUILD deleted file mode 100644 index 7893f4e2e4eb1..0000000000000 --- a/cmd/kubeadm/app/discovery/https/BUILD +++ /dev/null @@ -1,31 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["https.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/discovery/https", - deps = [ - "//cmd/kubeadm/app/discovery/file:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/discovery/https/https.go b/cmd/kubeadm/app/discovery/https/https.go deleted file mode 100644 index 2767eeddf1dda..0000000000000 --- a/cmd/kubeadm/app/discovery/https/https.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package https - -import ( - "io/ioutil" - "net/http" - "time" - - netutil "k8s.io/apimachinery/pkg/util/net" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/kubernetes/cmd/kubeadm/app/discovery/file" -) - -// RetrieveValidatedConfigInfo connects to the API Server and makes sure it can talk -// securely to the API Server using the provided CA cert and -// optionally refreshes the cluster-info information from the cluster-info ConfigMap -func RetrieveValidatedConfigInfo(httpsURL, clustername string, discoveryTimeout time.Duration) (*clientcmdapi.Config, error) { - client := &http.Client{Transport: netutil.SetOldTransportDefaults(&http.Transport{})} - response, err := client.Get(httpsURL) - if err != nil { - return nil, err - } - defer response.Body.Close() - - kubeconfig, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, err - } - - config, err := clientcmd.Load(kubeconfig) - if err != nil { - return nil, err - } - return file.ValidateConfigInfo(config, clustername, discoveryTimeout) -} diff --git a/cmd/kubeadm/app/discovery/token/BUILD b/cmd/kubeadm/app/discovery/token/BUILD deleted file mode 100644 index f62c349e04e74..0000000000000 --- a/cmd/kubeadm/app/discovery/token/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["token.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/discovery/token", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/pubkeypin:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/jws:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["token_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/jws:go_default_library", - "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/discovery/token/token.go b/cmd/kubeadm/app/discovery/token/token.go deleted file mode 100644 index cd56f5bf36860..0000000000000 --- a/cmd/kubeadm/app/discovery/token/token.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package token - -import ( - "bytes" - "context" - "fmt" - "time" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - bootstrap "k8s.io/cluster-bootstrap/token/jws" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin" -) - -// BootstrapUser defines bootstrap user name -const BootstrapUser = "token-bootstrap-client" - -// RetrieveValidatedConfigInfo connects to the API Server and tries to fetch the cluster-info ConfigMap -// It then makes sure it can trust the API Server by looking at the JWS-signed tokens and (if CACertHashes is not empty) -// validating the cluster CA against a set of pinned public keys -func RetrieveValidatedConfigInfo(cfg *kubeadmapi.Discovery) (*clientcmdapi.Config, error) { - return retrieveValidatedConfigInfo(nil, cfg, constants.DiscoveryRetryInterval) -} - -// retrieveValidatedConfigInfo is a private implementation of RetrieveValidatedConfigInfo. -// It accepts an optional clientset that can be used for testing purposes. -func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Discovery, interval time.Duration) (*clientcmdapi.Config, error) { - token, err := kubeadmapi.NewBootstrapTokenString(cfg.BootstrapToken.Token) - if err != nil { - return nil, err - } - - // Load the CACertHashes into a pubkeypin.Set - pubKeyPins := pubkeypin.NewSet() - if err = pubKeyPins.Allow(cfg.BootstrapToken.CACertHashes...); err != nil { - return nil, err - } - - duration := cfg.Timeout.Duration - // Make sure the interval is not bigger than the duration - if interval > duration { - interval = duration - } - - endpoint := cfg.BootstrapToken.APIServerEndpoint - insecureBootstrapConfig := buildInsecureBootstrapKubeConfig(endpoint, kubeadmapiv1beta2.DefaultClusterName) - clusterName := insecureBootstrapConfig.Contexts[insecureBootstrapConfig.CurrentContext].Cluster - - klog.V(1).Infof("[discovery] Created cluster-info discovery client, requesting info from %q", endpoint) - insecureClusterInfo, err := getClusterInfo(client, insecureBootstrapConfig, token, interval, duration) - if err != nil { - return nil, err - } - - // Validate the token in the cluster info - insecureKubeconfigBytes, err := validateClusterInfoToken(insecureClusterInfo, token) - if err != nil { - return nil, err - } - - // Load the insecure config - insecureConfig, err := clientcmd.Load(insecureKubeconfigBytes) - if err != nil { - return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - } - - // The ConfigMap should contain a single cluster - if len(insecureConfig.Clusters) != 1 { - return nil, errors.Errorf("expected the kubeconfig file in the %s ConfigMap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters)) - } - - // If no TLS root CA pinning was specified, we're done - if pubKeyPins.Empty() { - klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and no TLS pinning was specified, will use API Server %q", endpoint) - return insecureConfig, nil - } - - // Load and validate the cluster CA from the insecure kubeconfig - clusterCABytes, err := validateClusterCA(insecureConfig, pubKeyPins) - if err != nil { - return nil, err - } - - // Now that we know the cluster CA, connect back a second time validating with that CA - secureBootstrapConfig := buildSecureBootstrapKubeConfig(endpoint, clusterCABytes, clusterName) - - klog.V(1).Infof("[discovery] Requesting info from %q again to validate TLS against the pinned public key", endpoint) - secureClusterInfo, err := getClusterInfo(client, secureBootstrapConfig, token, interval, duration) - if err != nil { - return nil, err - } - - // Pull the kubeconfig from the securely-obtained ConfigMap and validate that it's the same as what we found the first time - secureKubeconfigBytes := []byte(secureClusterInfo.Data[bootstrapapi.KubeConfigKey]) - if !bytes.Equal(secureKubeconfigBytes, insecureKubeconfigBytes) { - return nil, errors.Errorf("the second kubeconfig from the %s ConfigMap (using validated TLS) was different from the first", bootstrapapi.ConfigMapClusterInfo) - } - - secureKubeconfig, err := clientcmd.Load(secureKubeconfigBytes) - if err != nil { - return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - } - - klog.V(1).Infof("[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server %q", endpoint) - - return secureKubeconfig, nil -} - -// buildInsecureBootstrapKubeConfig makes a kubeconfig object that connects insecurely to the API Server for bootstrapping purposes -func buildInsecureBootstrapKubeConfig(endpoint, clustername string) *clientcmdapi.Config { - controlPlaneEndpoint := fmt.Sprintf("https://%s", endpoint) - bootstrapConfig := kubeconfigutil.CreateBasic(controlPlaneEndpoint, clustername, BootstrapUser, []byte{}) - bootstrapConfig.Clusters[clustername].InsecureSkipTLSVerify = true - return bootstrapConfig -} - -// buildSecureBootstrapKubeConfig makes a kubeconfig object that connects securely to the API Server for bootstrapping purposes (validating with the specified CA) -func buildSecureBootstrapKubeConfig(endpoint string, caCert []byte, clustername string) *clientcmdapi.Config { - controlPlaneEndpoint := fmt.Sprintf("https://%s", endpoint) - bootstrapConfig := kubeconfigutil.CreateBasic(controlPlaneEndpoint, clustername, BootstrapUser, caCert) - return bootstrapConfig -} - -// validateClusterInfoToken validates that the JWS token present in the cluster info ConfigMap is valid -func validateClusterInfoToken(insecureClusterInfo *v1.ConfigMap, token *kubeadmapi.BootstrapTokenString) ([]byte, error) { - insecureKubeconfigString, ok := insecureClusterInfo.Data[bootstrapapi.KubeConfigKey] - if !ok || len(insecureKubeconfigString) == 0 { - return nil, errors.Errorf("there is no %s key in the %s ConfigMap. This API Server isn't set up for token bootstrapping, can't connect", - bootstrapapi.KubeConfigKey, bootstrapapi.ConfigMapClusterInfo) - } - - detachedJWSToken, ok := insecureClusterInfo.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID] - if !ok || len(detachedJWSToken) == 0 { - return nil, errors.Errorf("token id %q is invalid for this cluster or it has expired. Use \"kubeadm token create\" on the control-plane node to create a new valid token", token.ID) - } - - if !bootstrap.DetachedTokenIsValid(detachedJWSToken, insecureKubeconfigString, token.ID, token.Secret) { - return nil, errors.New("failed to verify JWS signature of received cluster info object, can't trust this API Server") - } - - return []byte(insecureKubeconfigString), nil -} - -// validateClusterCA validates the cluster CA found in the insecure kubeconfig -func validateClusterCA(insecureConfig *clientcmdapi.Config, pubKeyPins *pubkeypin.Set) ([]byte, error) { - var clusterCABytes []byte - for _, cluster := range insecureConfig.Clusters { - clusterCABytes = cluster.CertificateAuthorityData - } - - clusterCAs, err := certutil.ParseCertsPEM(clusterCABytes) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse cluster CA from the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - } - - // Validate the cluster CA public key against the pinned set - err = pubKeyPins.CheckAny(clusterCAs) - if err != nil { - return nil, errors.Wrapf(err, "cluster CA found in %s ConfigMap is invalid", bootstrapapi.ConfigMapClusterInfo) - } - - return clusterCABytes, nil -} - -// getClusterInfo creates a client from the given kubeconfig if the given client is nil, -// and requests the cluster info ConfigMap using PollImmediate. -// If a client is provided it will be used instead. -func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *kubeadmapi.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) { - var cm *v1.ConfigMap - var err error - - // Create client from kubeconfig - if client == nil { - client, err = kubeconfigutil.ToClientSet(kubeconfig) - if err != nil { - return nil, err - } - } - - ctx, cancel := context.WithTimeout(context.TODO(), duration) - defer cancel() - - wait.JitterUntil(func() { - cm, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.TODO(), bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{}) - if err != nil { - klog.V(1).Infof("[discovery] Failed to request cluster-info, will try again: %v", err) - return - } - // Even if the ConfigMap is available the JWS signature is patched-in a bit later. - // Make sure we retry util then. - if _, ok := cm.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]; !ok { - klog.V(1).Infof("[discovery] The cluster-info ConfigMap does not yet contain a JWS signature for token ID %q, will try again", token.ID) - err = errors.Errorf("could not find a JWS signature in the cluster-info ConfigMap for token ID %q", token.ID) - return - } - // Cancel the context on success - cancel() - }, interval, 0.3, true, ctx.Done()) - - if err != nil { - return nil, err - } - - return cm, nil -} diff --git a/cmd/kubeadm/app/discovery/token/token_test.go b/cmd/kubeadm/app/discovery/token/token_test.go deleted file mode 100644 index 3f27812d2c1fa..0000000000000 --- a/cmd/kubeadm/app/discovery/token/token_test.go +++ /dev/null @@ -1,312 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package token - -import ( - "testing" - "time" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - fakeclient "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/clientcmd" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - tokenjws "k8s.io/cluster-bootstrap/token/jws" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - - "github.com/pmezard/go-difflib/difflib" -) - -func TestRetrieveValidatedConfigInfo(t *testing.T) { - const ( - caCert = `-----BEGIN CERTIFICATE----- -MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl -cm5ldGVzMB4XDTE5MTEyMDAwNDk0MloXDTI5MTExNzAwNDk0MlowFTETMBEGA1UE -AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMqQ -ctECzA8yFSuVYupOUYgrTmfQeKe/9BaDWagaq7ow9+I2IvsfWFvlrD8QQr8sea6q -xjq7TV67Vb4RxBaoYDA+yI5vIcujWUxULun64lu3Q6iC1sj2UnmUpIdgazRXXEkZ -vxA6EbAnoxA0+lBOn1CZWl23IQ4s70o2hZ7wIp/vevB88RRRjqtvgc5elsjsbmDF -LS7L1Zuye8c6gS93bR+VjVmSIfr1IEq0748tIIyXjAVCWPVCvuP41MlfPc/JVpZD -uD2+pO6ZYREcdAnOf2eD4/eLOMKko4L1dSFy9JKM5PLnOC0Zk0AYOd1vS8DTAfxj -XPEIY8OBYFhlsxf4TE8CAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB -/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH/OYq8zyl1+zSTmuow3yI/15PL1 -dl8hB7IKnZNWmC/LTdm/+noh3Sb1IdRv6HkKg/GUn0UMuRUngLhju3EO4ozJPQcX -quaxzgmTKNWJ6ErDvRvWhGX0ZcbdBfZv+dowyRqzd5nlJ49hC+NrtFFQq6P05BYn -7SemguqeXmXwIj2Sa+1DeR6lRm9o8shAYjnyThUFqaMn18kI3SANJ5vk/3DFrPEO -CKC9EzFku2kuxg2dM12PbRGZQ2o0K6HEZgrrIKTPOy3ocb8r9M0aSFhjOV/NqGA4 -SaupXSW6XfvIi/UHoIbU3pNcsnUJGnQfQvip95XKk/gqcUr+m50vxgumxtA= ------END CERTIFICATE-----` - - caCertHash = "sha256:98be2e6d4d8a89aa308fb15de0c07e2531ce549c68dec1687cdd5c06f0826658" - - expectedKubeconfig = `apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1URXlNREF3TkRrME1sb1hEVEk1TVRFeE56QXdORGswTWxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTXFRCmN0RUN6QTh5RlN1Vll1cE9VWWdyVG1mUWVLZS85QmFEV2FnYXE3b3c5K0kySXZzZldGdmxyRDhRUXI4c2VhNnEKeGpxN1RWNjdWYjRSeEJhb1lEQSt5STV2SWN1aldVeFVMdW42NGx1M1E2aUMxc2oyVW5tVXBJZGdhelJYWEVrWgp2eEE2RWJBbm94QTArbEJPbjFDWldsMjNJUTRzNzBvMmhaN3dJcC92ZXZCODhSUlJqcXR2Z2M1ZWxzanNibURGCkxTN0wxWnV5ZThjNmdTOTNiUitWalZtU0lmcjFJRXEwNzQ4dElJeVhqQVZDV1BWQ3Z1UDQxTWxmUGMvSlZwWkQKdUQyK3BPNlpZUkVjZEFuT2YyZUQ0L2VMT01La280TDFkU0Z5OUpLTTVQTG5PQzBaazBBWU9kMXZTOERUQWZ4agpYUEVJWThPQllGaGxzeGY0VEU4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFIL09ZcTh6eWwxK3pTVG11b3czeUkvMTVQTDEKZGw4aEI3SUtuWk5XbUMvTFRkbS8rbm9oM1NiMUlkUnY2SGtLZy9HVW4wVU11UlVuZ0xoanUzRU80b3pKUFFjWApxdWF4emdtVEtOV0o2RXJEdlJ2V2hHWDBaY2JkQmZaditkb3d5UnF6ZDVubEo0OWhDK05ydEZGUXE2UDA1QlluCjdTZW1ndXFlWG1Yd0lqMlNhKzFEZVI2bFJtOW84c2hBWWpueVRoVUZxYU1uMThrSTNTQU5KNXZrLzNERnJQRU8KQ0tDOUV6Rmt1Mmt1eGcyZE0xMlBiUkdaUTJvMEs2SEVaZ3JySUtUUE95M29jYjhyOU0wYVNGaGpPVi9OcUdBNApTYXVwWFNXNlhmdklpL1VIb0liVTNwTmNzblVKR25RZlF2aXA5NVhLay9ncWNVcittNTB2eGd1bXh0QT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== - server: https://127.0.0.1 - name: somecluster -contexts: -- context: - cluster: somecluster - user: token-bootstrap-client - name: token-bootstrap-client@somecluster -current-context: token-bootstrap-client@somecluster -kind: Config -preferences: {} -users: null -` - ) - - tests := []struct { - name string - tokenID string - tokenSecret string - cfg *kubeadmapi.Discovery - configMap *fakeConfigMap - delayedJWSSignaturePatch bool - expectedError bool - }{ - { - // This is the default behavior. The JWS signature is patched after the cluster-info ConfigMap is created - name: "valid: retrieve a valid kubeconfig with CA verification and delayed JWS signature", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - CACertHashes: []string{caCertHash}, - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: map[string]string{}, - }, - delayedJWSSignaturePatch: true, - }, - { - // Same as above expect this test creates the ConfigMap with the JWS signature - name: "valid: retrieve a valid kubeconfig with CA verification", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - CACertHashes: []string{caCertHash}, - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: nil, - }, - }, - { - // Skipping CA verification is also supported - name: "valid: retrieve a valid kubeconfig without CA verification", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: nil, - }, - }, - { - name: "invalid: token format is invalid", - tokenID: "foo", - tokenSecret: "bar", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "foo.bar", - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: nil, - }, - expectedError: true, - }, - { - name: "invalid: missing cluster-info ConfigMap", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - }, - }, - configMap: &fakeConfigMap{ - name: "baz", - data: nil, - }, - expectedError: true, - }, - { - name: "invalid: wrong JWS signature", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: map[string]string{ - bootstrapapi.KubeConfigKey: "foo", - bootstrapapi.JWSSignatureKeyPrefix + "123456": "bar", - }, - }, - expectedError: true, - }, - { - name: "invalid: missing key for JWSSignatureKeyPrefix", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: map[string]string{ - bootstrapapi.KubeConfigKey: "foo", - }, - }, - expectedError: true, - }, - { - name: "invalid: wrong CA cert hash", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - CACertHashes: []string{"foo"}, - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: nil, - }, - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - kubeconfig := buildSecureBootstrapKubeConfig("127.0.0.1", []byte(caCert), "somecluster") - kubeconfigBytes, err := clientcmd.Write(*kubeconfig) - if err != nil { - t.Fatalf("cannot marshal kubeconfig %v", err) - } - - // Generate signature of the insecure kubeconfig - sig, err := tokenjws.ComputeDetachedSignature(string(kubeconfigBytes), test.tokenID, test.tokenSecret) - if err != nil { - t.Fatalf("cannot compute detached JWS signature: %v", err) - } - - // If the JWS signature is delayed, only add the kubeconfig - if test.delayedJWSSignaturePatch { - test.configMap.data = map[string]string{} - test.configMap.data[bootstrapapi.KubeConfigKey] = string(kubeconfigBytes) - } - - // Populate the default cluster-info data - if test.configMap.data == nil { - test.configMap.data = map[string]string{} - test.configMap.data[bootstrapapi.KubeConfigKey] = string(kubeconfigBytes) - test.configMap.data[bootstrapapi.JWSSignatureKeyPrefix+test.tokenID] = sig - } - - // Create a fake client and create the cluster-info ConfigMap - client := fakeclient.NewSimpleClientset() - if err = test.configMap.createOrUpdate(client); err != nil { - t.Fatalf("could not create ConfigMap: %v", err) - } - - // Set arbitrary discovery timeout and retry interval - test.cfg.Timeout = &metav1.Duration{Duration: time.Millisecond * 200} - interval := time.Millisecond * 20 - - // Patch the JWS signature after a short delay - if test.delayedJWSSignaturePatch { - test.configMap.data[bootstrapapi.JWSSignatureKeyPrefix+test.tokenID] = sig - go func() { - time.Sleep(time.Millisecond * 60) - if err := test.configMap.createOrUpdate(client); err != nil { - t.Errorf("could not update the cluster-info ConfigMap with a JWS signature: %v", err) - } - }() - } - - // Retrieve validated configuration - kubeconfig, err = retrieveValidatedConfigInfo(client, test.cfg, interval) - if (err != nil) != test.expectedError { - t.Errorf("expected error %v, got %v, error: %v", test.expectedError, err != nil, err) - } - - // Return if an error is expected - if test.expectedError { - return - } - - // Validate the resulted kubeconfig - kubeconfigBytes, err = clientcmd.Write(*kubeconfig) - if err != nil { - t.Fatalf("cannot marshal resulted kubeconfig %v", err) - } - if string(kubeconfigBytes) != expectedKubeconfig { - t.Error("unexpected kubeconfig") - diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(expectedKubeconfig), - B: difflib.SplitLines(string(kubeconfigBytes)), - FromFile: "expected", - ToFile: "got", - Context: 10, - } - diffstr, err := difflib.GetUnifiedDiffString(diff) - if err != nil { - t.Fatalf("error generating unified diff string: %v", err) - } - t.Errorf("\n%s", diffstr) - } - }) - } -} - -type fakeConfigMap struct { - name string - data map[string]string -} - -func (c *fakeConfigMap) createOrUpdate(client clientset.Interface) error { - return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.name, - Namespace: metav1.NamespacePublic, - }, - Data: c.data, - }) -} diff --git a/cmd/kubeadm/app/features/BUILD b/cmd/kubeadm/app/features/BUILD deleted file mode 100644 index 075c4d59772d4..0000000000000 --- a/cmd/kubeadm/app/features/BUILD +++ /dev/null @@ -1,41 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["features.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/features", - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/component-base/featuregate:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["features_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/component-base/featuregate:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go deleted file mode 100644 index 0d2299581e554..0000000000000 --- a/cmd/kubeadm/app/features/features.go +++ /dev/null @@ -1,178 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package features - -import ( - "fmt" - "sort" - "strconv" - "strings" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/featuregate" -) - -const ( - // IPv6DualStack is expected to be alpha in v1.16 - IPv6DualStack = "IPv6DualStack" - // PublicKeysECDSA is expected to be alpha in v1.19 - PublicKeysECDSA = "PublicKeysECDSA" -) - -// InitFeatureGates are the default feature gates for the init command -var InitFeatureGates = FeatureList{ - IPv6DualStack: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, - PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, -} - -// Feature represents a feature being gated -type Feature struct { - featuregate.FeatureSpec - MinimumVersion *version.Version - HiddenInHelpText bool - DeprecationMessage string -} - -// FeatureList represents a list of feature gates -type FeatureList map[string]Feature - -// ValidateVersion ensures that a feature gate list is compatible with the chosen Kubernetes version -func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error { - if requestedVersion == "" { - return nil - } - parsedExpVersion, err := version.ParseSemantic(requestedVersion) - if err != nil { - return errors.Wrapf(err, "error parsing version %s", requestedVersion) - } - for k := range requestedFeatures { - if minVersion := allFeatures[k].MinimumVersion; minVersion != nil { - if !parsedExpVersion.AtLeast(minVersion) { - return errors.Errorf( - "the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum", - requestedVersion, k, minVersion) - } - } - } - return nil -} - -// Enabled indicates whether a feature name has been enabled -func Enabled(featureList map[string]bool, featureName string) bool { - if enabled, ok := featureList[string(featureName)]; ok { - return enabled - } - return InitFeatureGates[string(featureName)].Default -} - -// Supports indicates whether a feature name is supported on the given -// feature set -func Supports(featureList FeatureList, featureName string) bool { - for k, v := range featureList { - if featureName == string(k) { - return v.PreRelease != featuregate.Deprecated - } - } - return false -} - -// Keys returns a slice of feature names for a given feature set -func Keys(featureList FeatureList) []string { - var list []string - for k := range featureList { - list = append(list, string(k)) - } - return list -} - -// KnownFeatures returns a slice of strings describing the FeatureList features. -func KnownFeatures(f *FeatureList) []string { - var known []string - for k, v := range *f { - if v.HiddenInHelpText { - continue - } - - pre := "" - if v.PreRelease != featuregate.GA { - pre = fmt.Sprintf("%s - ", v.PreRelease) - } - known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) - } - sort.Strings(known) - return known -} - -// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a -// map[string]bool of known keys or returns an error. -func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { - featureGate := map[string]bool{} - for _, s := range strings.Split(value, ",") { - if len(s) == 0 { - continue - } - - arr := strings.SplitN(s, "=", 2) - if len(arr) != 2 { - return nil, errors.Errorf("missing bool value for feature-gate key:%s", s) - } - - k := strings.TrimSpace(arr[0]) - v := strings.TrimSpace(arr[1]) - - featureSpec, ok := (*f)[k] - if !ok { - return nil, errors.Errorf("unrecognized feature-gate key: %s", k) - } - - if featureSpec.PreRelease == featuregate.Deprecated { - return nil, errors.Errorf("feature-gate key is deprecated: %s", k) - } - - boolValue, err := strconv.ParseBool(v) - if err != nil { - return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) - } - featureGate[k] = boolValue - } - - return featureGate, nil -} - -// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set. -// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might -// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm -func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string { - deprecatedMsg := map[string]string{} - for k := range features { - featureSpec, ok := (*f)[k] - if !ok { - // This case should never happen, it is implemented only as a sentinel - // for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove) - deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k) - } - - if featureSpec.PreRelease == featuregate.Deprecated { - if _, ok := deprecatedMsg[k]; !ok { - deprecatedMsg[k] = featureSpec.DeprecationMessage - } - } - } - - return deprecatedMsg -} diff --git a/cmd/kubeadm/app/features/features_test.go b/cmd/kubeadm/app/features/features_test.go deleted file mode 100644 index 81d29f3e6a824..0000000000000 --- a/cmd/kubeadm/app/features/features_test.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package features - -import ( - "reflect" - "testing" - - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/featuregate" -) - -func TestKnownFeatures(t *testing.T) { - var someFeatures = FeatureList{ - "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "feature3": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}}, - "hidden": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.GA}, HiddenInHelpText: true}, - } - - r := KnownFeatures(&someFeatures) - - if len(r) != 3 { - t.Errorf("KnownFeatures returned %d values, expected 3", len(r)) - } - - // check the first value is feature1 (the list should be sorted); prerelease and default should be present - f1 := "feature1=true|false (BETA - default=false)" - if r[0] != f1 { - t.Errorf("KnownFeatures returned %s values, expected %s", r[0], f1) - } - // check the second value is feature2; prerelease and default should be present - f2 := "feature2=true|false (ALPHA - default=true)" - if r[1] != f2 { - t.Errorf("KnownFeatures returned %s values, expected %s", r[1], f2) - } - // check the second value is feature3; prerelease should not be shown for GA features; default should be present - f3 := "feature3=true|false (default=false)" - if r[2] != f3 { - t.Errorf("KnownFeatures returned %s values, expected %s", r[2], f3) - } -} - -func TestNewFeatureGate(t *testing.T) { - var someFeatures = FeatureList{ - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}}, - "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}}, - } - - var tests = []struct { - value string - expectedError bool - expectedFeaturesGate map[string]bool - }{ - { //invalid value (missing =) - value: "invalidValue", - expectedError: true, - }, - { //invalid value (missing =) - value: "feature1=true,invalidValue", - expectedError: true, - }, - { //invalid value (not a boolean) - value: "feature1=notABoolean", - expectedError: true, - }, - { //invalid value (not a boolean) - value: "feature1=true,feature2=notABoolean", - expectedError: true, - }, - { //unrecognized feature-gate key - value: "unknownFeature=false", - expectedError: true, - }, - { //unrecognized feature-gate key - value: "feature1=true,unknownFeature=false", - expectedError: true, - }, - { //deprecated feature-gate key - value: "deprecated=true", - expectedError: true, - }, - { //one feature - value: "feature1=true", - expectedError: false, - expectedFeaturesGate: map[string]bool{"feature1": true}, - }, - { //two features - value: "feature1=true,feature2=false", - expectedError: false, - expectedFeaturesGate: map[string]bool{"feature1": true, "feature2": false}, - }, - } - - for _, test := range tests { - t.Run(test.value, func(t *testing.T) { - r, err := NewFeatureGate(&someFeatures, test.value) - - if !test.expectedError && err != nil { - t.Errorf("NewFeatureGate failed when not expected: %v", err) - return - } else if test.expectedError && err == nil { - t.Error("NewFeatureGate didn't failed when expected") - return - } - - if !reflect.DeepEqual(r, test.expectedFeaturesGate) { - t.Errorf("NewFeatureGate returned a unexpected value") - } - }) - } -} - -func TestValidateVersion(t *testing.T) { - var someFeatures = FeatureList{ - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "feature2": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Alpha}, MinimumVersion: version.MustParseSemantic("v1.17.0").WithPreRelease("alpha.1")}, - } - - var tests = []struct { - name string - requestedVersion string - requestedFeatures map[string]bool - expectedError bool - }{ - { - name: "no min version", - requestedFeatures: map[string]bool{"feature1": true}, - expectedError: false, - }, - { - name: "min version but correct value given", - requestedFeatures: map[string]bool{"feature2": true}, - requestedVersion: "v1.17.0", - expectedError: false, - }, - { - name: "min version and incorrect value given", - requestedFeatures: map[string]bool{"feature2": true}, - requestedVersion: "v1.11.2", - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := ValidateVersion(someFeatures, test.requestedFeatures, test.requestedVersion) - if !test.expectedError && err != nil { - t.Errorf("ValidateVersion failed when not expected: %v", err) - return - } else if test.expectedError && err == nil { - t.Error("ValidateVersion didn't failed when expected") - return - } - }) - } -} - -// TestEnabledDefaults tests that Enabled returns the default values for -// each feature gate when no feature gates are specified. -func TestEnabledDefaults(t *testing.T) { - for featureName, feature := range InitFeatureGates { - featureList := make(map[string]bool) - - enabled := Enabled(featureList, featureName) - if enabled != feature.Default { - t.Errorf("Enabled returned %v instead of default value %v for feature %s", enabled, feature.Default, featureName) - } - } -} - -func TestCheckDeprecatedFlags(t *testing.T) { - dummyMessage := "dummy message" - var someFeatures = FeatureList{ - "feature1": {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Beta}}, - "deprecated": {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Deprecated}, DeprecationMessage: dummyMessage}, - } - - var tests = []struct { - name string - features map[string]bool - expectedMsg map[string]string - }{ - { - name: "deprecated feature", - features: map[string]bool{"deprecated": true}, - expectedMsg: map[string]string{"deprecated": dummyMessage}, - }, - { - name: "valid feature", - features: map[string]bool{"feature1": true}, - expectedMsg: map[string]string{}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - msg := CheckDeprecatedFlags(&someFeatures, test.features) - if !reflect.DeepEqual(test.expectedMsg, msg) { - t.Error("CheckDeprecatedFlags didn't returned expected message") - } - }) - } -} diff --git a/cmd/kubeadm/app/images/BUILD b/cmd/kubeadm/app/images/BUILD deleted file mode 100644 index cafd5e84f43fa..0000000000000 --- a/cmd/kubeadm/app/images/BUILD +++ /dev/null @@ -1,51 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "images.go", - "images_unix.go", - "images_windows.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/images", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:windows": [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - ], - "//conditions:default": [], - }), -) - -go_test( - name = "go_default_test", - srcs = ["images_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go deleted file mode 100644 index ddb2a61579070..0000000000000 --- a/cmd/kubeadm/app/images/images.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package images - -import ( - "fmt" - - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const extraHyperKubeNote = ` The "useHyperKubeImage" field will be removed from future kubeadm config versions and possibly ignored in future releases.` - -// GetGenericImage generates and returns a platform agnostic image (backed by manifest list) -func GetGenericImage(prefix, image, tag string) string { - return fmt.Sprintf("%s/%s:%s", prefix, image, tag) -} - -// GetKubernetesImage generates and returns the image for the components managed in the Kubernetes main repository, -// including the control-plane components and kube-proxy. If specified, the HyperKube image will be used. -func GetKubernetesImage(image string, cfg *kubeadmapi.ClusterConfiguration) string { - if cfg.UseHyperKubeImage && image != constants.HyperKube { - klog.Warningf(`WARNING: DEPRECATED use of the "hyperkube" image in place of %q.`+extraHyperKubeNote, image) - image = constants.HyperKube - } - repoPrefix := cfg.GetControlPlaneImageRepository() - kubernetesImageTag := kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion) - return GetGenericImage(repoPrefix, image, kubernetesImageTag) -} - -// GetDNSImage generates and returns the image for the DNS, that can be CoreDNS or kube-dns. -// Given that kube-dns uses 3 containers, an additional imageName parameter was added -func GetDNSImage(cfg *kubeadmapi.ClusterConfiguration, imageName string) string { - // DNS uses default image repository by default - dnsImageRepository := cfg.ImageRepository - // unless an override is specified - if cfg.DNS.ImageRepository != "" { - dnsImageRepository = cfg.DNS.ImageRepository - } - // DNS uses an imageTag that corresponds to the DNS version matching the Kubernetes version - dnsImageTag := constants.GetDNSVersion(cfg.DNS.Type) - - // unless an override is specified - if cfg.DNS.ImageTag != "" { - dnsImageTag = cfg.DNS.ImageTag - } - return GetGenericImage(dnsImageRepository, imageName, dnsImageTag) -} - -// GetEtcdImage generates and returns the image for etcd -func GetEtcdImage(cfg *kubeadmapi.ClusterConfiguration) string { - // Etcd uses default image repository by default - etcdImageRepository := cfg.ImageRepository - // unless an override is specified - if cfg.Etcd.Local != nil && cfg.Etcd.Local.ImageRepository != "" { - etcdImageRepository = cfg.Etcd.Local.ImageRepository - } - // Etcd uses an imageTag that corresponds to the etcd version matching the Kubernetes version - etcdImageTag := constants.DefaultEtcdVersion - etcdVersion, warning, err := constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, cfg.KubernetesVersion) - if err == nil { - etcdImageTag = etcdVersion.String() - } - if warning != nil { - klog.Warningln(warning) - } - // unless an override is specified - if cfg.Etcd.Local != nil && cfg.Etcd.Local.ImageTag != "" { - etcdImageTag = cfg.Etcd.Local.ImageTag - } - return GetGenericImage(etcdImageRepository, constants.Etcd, etcdImageTag) -} - -// GetControlPlaneImages returns a list of container images kubeadm expects to use on a control plane node -func GetControlPlaneImages(cfg *kubeadmapi.ClusterConfiguration) []string { - imgs := []string{} - - // start with core kubernetes images - if cfg.UseHyperKubeImage { - klog.Warningln(`WARNING: DEPRECATED use of the "hyperkube" image for the Kubernetes control plane.` + extraHyperKubeNote) - imgs = append(imgs, GetKubernetesImage(constants.HyperKube, cfg)) - } else { - imgs = append(imgs, GetKubernetesImage(constants.KubeAPIServer, cfg)) - imgs = append(imgs, GetKubernetesImage(constants.KubeControllerManager, cfg)) - imgs = append(imgs, GetKubernetesImage(constants.KubeScheduler, cfg)) - imgs = append(imgs, GetKubernetesImage(constants.KubeProxy, cfg)) - } - - // pause is not available on the ci image repository so use the default image repository. - imgs = append(imgs, GetPauseImage(cfg)) - - // if etcd is not external then add the image as it will be required - if cfg.Etcd.Local != nil { - imgs = append(imgs, GetEtcdImage(cfg)) - } - - // Append the appropriate DNS images - if cfg.DNS.Type == kubeadmapi.CoreDNS { - imgs = append(imgs, GetDNSImage(cfg, constants.CoreDNSImageName)) - } else { - imgs = append(imgs, GetDNSImage(cfg, constants.KubeDNSKubeDNSImageName)) - imgs = append(imgs, GetDNSImage(cfg, constants.KubeDNSSidecarImageName)) - imgs = append(imgs, GetDNSImage(cfg, constants.KubeDNSDnsMasqNannyImageName)) - } - - return imgs -} diff --git a/cmd/kubeadm/app/images/images_test.go b/cmd/kubeadm/app/images/images_test.go deleted file mode 100644 index 7af0de3a7c363..0000000000000 --- a/cmd/kubeadm/app/images/images_test.go +++ /dev/null @@ -1,266 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package images - -import ( - "fmt" - "strings" - "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -const ( - testversion = "v10.1.2-alpha.1.100+0123456789abcdef+SOMETHING" - expected = "v10.1.2-alpha.1.100_0123456789abcdef_SOMETHING" - gcrPrefix = "k8s.gcr.io" -) - -func TestGetGenericImage(t *testing.T) { - const ( - prefix = "foo" - image = "bar" - tag = "baz" - ) - expected := fmt.Sprintf("%s/%s:%s", prefix, image, tag) - actual := GetGenericImage(prefix, image, tag) - if actual != expected { - t.Errorf("failed GetGenericImage:\n\texpected: %s\n\t actual: %s", expected, actual) - } -} - -func TestGetKubernetesImage(t *testing.T) { - var tests = []struct { - image string - expected string - cfg *kubeadmapi.ClusterConfiguration - }{ - { - expected: GetGenericImage(gcrPrefix, constants.HyperKube, expected), - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: gcrPrefix, - KubernetesVersion: testversion, - UseHyperKubeImage: true, - }, - }, - { - image: constants.KubeAPIServer, - expected: GetGenericImage(gcrPrefix, "kube-apiserver", expected), - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: gcrPrefix, - KubernetesVersion: testversion, - }, - }, - { - image: constants.KubeControllerManager, - expected: GetGenericImage(gcrPrefix, "kube-controller-manager", expected), - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: gcrPrefix, - KubernetesVersion: testversion, - }, - }, - { - image: constants.KubeScheduler, - expected: GetGenericImage(gcrPrefix, "kube-scheduler", expected), - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: gcrPrefix, - KubernetesVersion: testversion, - }, - }, - } - for _, rt := range tests { - actual := GetKubernetesImage(rt.image, rt.cfg) - if actual != rt.expected { - t.Errorf( - "failed GetKubernetesImage:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - } -} - -func TestGetEtcdImage(t *testing.T) { - var tests = []struct { - expected string - cfg *kubeadmapi.ClusterConfiguration - }{ - { - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: "real.repo", - KubernetesVersion: "1.16.0", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{}, - }, - }, - expected: "real.repo/etcd:3.3.17-0", - }, - { - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: "real.repo", - KubernetesVersion: "1.16.0", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ImageMeta: kubeadmapi.ImageMeta{ - ImageTag: "override", - }, - }, - }, - }, - expected: "real.repo/etcd:override", - }, - { - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: "real.repo", - KubernetesVersion: "1.16.0", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ImageMeta: kubeadmapi.ImageMeta{ - ImageRepository: "override", - }, - }, - }, - }, - expected: "override/etcd:3.3.17-0", - }, - { - expected: GetGenericImage(gcrPrefix, "etcd", constants.DefaultEtcdVersion), - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: gcrPrefix, - KubernetesVersion: testversion, - }, - }, - } - for _, rt := range tests { - actual := GetEtcdImage(rt.cfg) - if actual != rt.expected { - t.Errorf( - "failed GetEtcdImage:\n\texpected: %s\n\t actual: %s", - rt.expected, - actual, - ) - } - } -} - -func TestGetPauseImage(t *testing.T) { - testcases := []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - expected string - }{ - { - name: "pause image defined", - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: "test.repo", - }, - expected: "test.repo/pause:" + constants.PauseVersion, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - actual := GetPauseImage(tc.cfg) - if actual != tc.expected { - t.Fatalf( - "failed GetPauseImage:\n\texpected: %s\n\t actual: %s", - tc.expected, - actual, - ) - } - }) - } -} - -func TestGetAllImages(t *testing.T) { - testcases := []struct { - name string - expect string - cfg *kubeadmapi.ClusterConfiguration - }{ - { - name: "defined CIImageRepository", - cfg: &kubeadmapi.ClusterConfiguration{ - CIImageRepository: "test.repo", - }, - expect: "test.repo", - }, - { - name: "undefined CIImagerRepository should contain the default image prefix", - cfg: &kubeadmapi.ClusterConfiguration{ - ImageRepository: "real.repo", - }, - expect: "real.repo", - }, - { - name: "test that etcd is returned when it is not external", - cfg: &kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{}, - }, - }, - expect: constants.Etcd, - }, - { - name: "CoreDNS image is returned", - cfg: &kubeadmapi.ClusterConfiguration{ - DNS: kubeadmapi.DNS{ - Type: kubeadmapi.CoreDNS, - }, - }, - expect: constants.CoreDNSImageName, - }, - { - name: "main kube-dns image is returned", - cfg: &kubeadmapi.ClusterConfiguration{ - DNS: kubeadmapi.DNS{ - Type: kubeadmapi.KubeDNS, - }, - }, - expect: constants.KubeDNSKubeDNSImageName, - }, - { - name: "kube-dns sidecar image is returned", - cfg: &kubeadmapi.ClusterConfiguration{ - DNS: kubeadmapi.DNS{ - Type: kubeadmapi.KubeDNS, - }, - }, - expect: constants.KubeDNSSidecarImageName, - }, - { - name: "kube-dns dnsmasq-nanny image is returned", - cfg: &kubeadmapi.ClusterConfiguration{ - DNS: kubeadmapi.DNS{ - Type: kubeadmapi.KubeDNS, - }, - }, - expect: constants.KubeDNSDnsMasqNannyImageName, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - imgs := GetControlPlaneImages(tc.cfg) - for _, img := range imgs { - if strings.Contains(img, tc.expect) { - return - } - } - t.Fatalf("did not find %q in %q", tc.expect, imgs) - }) - } -} diff --git a/cmd/kubeadm/app/images/images_unix.go b/cmd/kubeadm/app/images/images_unix.go deleted file mode 100644 index 43fa63e88cf8e..0000000000000 --- a/cmd/kubeadm/app/images/images_unix.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build !windows - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package images - -import ( - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// GetPauseImage returns the image for the "pause" container -func GetPauseImage(cfg *kubeadmapi.ClusterConfiguration) string { - return GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion) -} diff --git a/cmd/kubeadm/app/images/images_windows.go b/cmd/kubeadm/app/images/images_windows.go deleted file mode 100644 index f3c084a9483bf..0000000000000 --- a/cmd/kubeadm/app/images/images_windows.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build windows - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package images - -import ( - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// GetPauseImage returns the image for the "pause" container -func GetPauseImage(cfg *kubeadmapi.ClusterConfiguration) string { - //If user has configured the cluster to use a different image repository, use that for the Windows pause image. - if cfg.ImageRepository != kubeadmapiv1beta2.DefaultImageRepository { - return GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion) - } - return GetGenericImage("mcr.microsoft.com/oss/kubernetes", "pause", constants.PauseVersion) -} diff --git a/cmd/kubeadm/app/kubeadm.go b/cmd/kubeadm/app/kubeadm.go deleted file mode 100644 index 77f067b1a1e9d..0000000000000 --- a/cmd/kubeadm/app/kubeadm.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "flag" - "os" - - "github.com/spf13/pflag" - - "k8s.io/klog/v2" - - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/kubernetes/cmd/kubeadm/app/cmd" -) - -// Run creates and executes new kubeadm command -func Run() error { - klog.InitFlags(nil) - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - - pflag.Set("logtostderr", "true") - // We do not want these flags to show up in --help - // These MarkHidden calls must be after the lines above - pflag.CommandLine.MarkHidden("version") - pflag.CommandLine.MarkHidden("log-flush-frequency") - pflag.CommandLine.MarkHidden("alsologtostderr") - pflag.CommandLine.MarkHidden("log-backtrace-at") - pflag.CommandLine.MarkHidden("log-dir") - pflag.CommandLine.MarkHidden("logtostderr") - pflag.CommandLine.MarkHidden("stderrthreshold") - pflag.CommandLine.MarkHidden("vmodule") - - cmd := cmd.NewKubeadmCommand(os.Stdin, os.Stdout, os.Stderr) - return cmd.Execute() -} diff --git a/cmd/kubeadm/app/phases/addons/dns/BUILD b/cmd/kubeadm/app/phases/addons/dns/BUILD deleted file mode 100644 index 6bb59f38856ca..0000000000000 --- a/cmd/kubeadm/app/phases/addons/dns/BUILD +++ /dev/null @@ -1,70 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["dns_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "dns.go", - "manifests.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/github.com/caddyserver/caddy/caddyfile:go_default_library", - "//vendor/github.com/coredns/corefile-migration/migration:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/addons/dns/dns.go b/cmd/kubeadm/app/phases/addons/dns/dns.go deleted file mode 100644 index 76c9a2d247fb4..0000000000000 --- a/cmd/kubeadm/app/phases/addons/dns/dns.go +++ /dev/null @@ -1,587 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package dns - -import ( - "context" - "encoding/json" - "fmt" - "net" - "strings" - - "github.com/caddyserver/caddy/caddyfile" - "github.com/coredns/corefile-migration/migration" - "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kuberuntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - clientset "k8s.io/client-go/kubernetes" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - utilsnet "k8s.io/utils/net" -) - -const ( - // KubeDNSServiceAccountName describes the name of the ServiceAccount for the kube-dns addon - KubeDNSServiceAccountName = "kube-dns" - kubeDNSStubDomain = "stubDomains" - kubeDNSUpstreamNameservers = "upstreamNameservers" - unableToDecodeCoreDNS = "unable to decode CoreDNS" - coreDNSReplicas = 2 - kubeDNSReplicas = 1 -) - -// DeployedDNSAddon returns the type of DNS addon currently deployed -func DeployedDNSAddon(client clientset.Interface) (kubeadmapi.DNSAddOnType, string, error) { - deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem) - deployments, err := deploymentsClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"}) - if err != nil { - return "", "", errors.Wrap(err, "couldn't retrieve DNS addon deployments") - } - - switch len(deployments.Items) { - case 0: - return "", "", nil - case 1: - addonName := deployments.Items[0].Name - addonType := kubeadmapi.CoreDNS - if addonName == kubeadmconstants.KubeDNSDeploymentName { - addonType = kubeadmapi.KubeDNS - } - addonImage := deployments.Items[0].Spec.Template.Spec.Containers[0].Image - addonImageParts := strings.Split(addonImage, ":") - addonVersion := addonImageParts[len(addonImageParts)-1] - return addonType, addonVersion, nil - default: - return "", "", errors.Errorf("multiple DNS addon deployments found: %v", deployments.Items) - } -} - -// deployedDNSReplicas returns the replica count for the current DNS deployment -func deployedDNSReplicas(client clientset.Interface, replicas int32) (*int32, error) { - deploymentsClient := client.AppsV1().Deployments(metav1.NamespaceSystem) - deployments, err := deploymentsClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "k8s-app=kube-dns"}) - if err != nil { - return &replicas, errors.Wrap(err, "couldn't retrieve DNS addon deployments") - } - switch len(deployments.Items) { - case 0: - return &replicas, nil - case 1: - return deployments.Items[0].Spec.Replicas, nil - default: - return &replicas, errors.Errorf("multiple DNS addon deployments found: %v", deployments.Items) - } -} - -// EnsureDNSAddon creates the kube-dns or CoreDNS addon -func EnsureDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error { - if cfg.DNS.Type == kubeadmapi.CoreDNS { - replicas, err := deployedDNSReplicas(client, coreDNSReplicas) - if err != nil { - return err - } - return coreDNSAddon(cfg, client, replicas) - } - replicas, err := deployedDNSReplicas(client, kubeDNSReplicas) - if err != nil { - return err - } - return kubeDNSAddon(cfg, client, replicas) -} - -func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error { - if err := CreateServiceAccount(client); err != nil { - return err - } - - dnsip, err := kubeadmconstants.GetDNSIP(cfg.Networking.ServiceSubnet, features.Enabled(cfg.FeatureGates, features.IPv6DualStack)) - if err != nil { - return err - } - - var dnsBindAddr, dnsProbeAddr string - if utilsnet.IsIPv6(dnsip) { - dnsBindAddr = "::1" - dnsProbeAddr = "[" + dnsBindAddr + "]" - } else { - dnsBindAddr = "127.0.0.1" - dnsProbeAddr = dnsBindAddr - } - - dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, - struct { - DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, OldControlPlaneTaintKey, ControlPlaneTaintKey string - Replicas *int32 - }{ - DeploymentName: kubeadmconstants.KubeDNSDeploymentName, - KubeDNSImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSKubeDNSImageName), - DNSMasqImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSDnsMasqNannyImageName), - SidecarImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSSidecarImageName), - DNSBindAddr: dnsBindAddr, - DNSProbeAddr: dnsProbeAddr, - DNSDomain: cfg.Networking.DNSDomain, - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - OldControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleOldControlPlane, - ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleControlPlane, - Replicas: replicas, - }) - if err != nil { - return errors.Wrap(err, "error when parsing kube-dns deployment template") - } - - dnsServiceBytes, err := kubeadmutil.ParseTemplate(KubeDNSService, struct{ DNSIP string }{ - DNSIP: dnsip.String(), - }) - if err != nil { - return errors.Wrap(err, "error when parsing kube-proxy configmap template") - } - - if err := createKubeDNSAddon(dnsDeploymentBytes, dnsServiceBytes, client); err != nil { - return err - } - fmt.Println("[addons] WARNING: kube-dns is deprecated and will not be supported in a future version") - fmt.Println("[addons] Applied essential addon: kube-dns") - return nil -} - -// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist. -func CreateServiceAccount(client clientset.Interface) error { - - return apiclient.CreateOrUpdateServiceAccount(client, &v1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: KubeDNSServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }) -} - -func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.Interface) error { - kubednsDeployment := &apps.Deployment{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), deploymentBytes, kubednsDeployment); err != nil { - return errors.Wrap(err, "unable to decode kube-dns deployment") - } - - // Create the Deployment for kube-dns or update it in case it already exists - if err := apiclient.CreateOrUpdateDeployment(client, kubednsDeployment); err != nil { - return err - } - - kubednsService := &v1.Service{} - return createDNSService(kubednsService, serviceBytes, client) -} - -func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error { - // Get the YAML manifest - coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct { - DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey string - Replicas *int32 - }{ - DeploymentName: kubeadmconstants.CoreDNSDeploymentName, - Image: images.GetDNSImage(cfg, kubeadmconstants.CoreDNSImageName), - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - OldControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleOldControlPlane, - ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleControlPlane, - Replicas: replicas, - }) - if err != nil { - return errors.Wrap(err, "error when parsing CoreDNS deployment template") - } - - // Get the kube-dns ConfigMap for translation to equivalent CoreDNS Config. - kubeDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.KubeDNSConfigMap, metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return err - } - - stubDomain, err := translateStubDomainOfKubeDNSToForwardCoreDNS(kubeDNSStubDomain, kubeDNSConfigMap) - if err != nil { - return err - } - - upstreamNameserver, err := translateUpstreamNameServerOfKubeDNSToUpstreamForwardCoreDNS(kubeDNSUpstreamNameservers, kubeDNSConfigMap) - if err != nil { - return err - } - coreDNSDomain := cfg.Networking.DNSDomain - - // Get the config file for CoreDNS - coreDNSConfigMapBytes, err := kubeadmutil.ParseTemplate(CoreDNSConfigMap, struct{ DNSDomain, UpstreamNameserver, StubDomain string }{ - DNSDomain: coreDNSDomain, - UpstreamNameserver: upstreamNameserver, - StubDomain: stubDomain, - }) - if err != nil { - return errors.Wrap(err, "error when parsing CoreDNS configMap template") - } - - dnsip, err := kubeadmconstants.GetDNSIP(cfg.Networking.ServiceSubnet, features.Enabled(cfg.FeatureGates, features.IPv6DualStack)) - if err != nil { - return err - } - - coreDNSServiceBytes, err := kubeadmutil.ParseTemplate(KubeDNSService, struct{ DNSIP string }{ - DNSIP: dnsip.String(), - }) - - if err != nil { - return errors.Wrap(err, "error when parsing CoreDNS service template") - } - - if err := createCoreDNSAddon(coreDNSDeploymentBytes, coreDNSServiceBytes, coreDNSConfigMapBytes, client); err != nil { - return err - } - fmt.Println("[addons] Applied essential addon: CoreDNS") - return nil -} - -func createCoreDNSAddon(deploymentBytes, serviceBytes, configBytes []byte, client clientset.Interface) error { - coreDNSConfigMap := &v1.ConfigMap{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), configBytes, coreDNSConfigMap); err != nil { - return errors.Wrapf(err, "%s ConfigMap", unableToDecodeCoreDNS) - } - - // Create the ConfigMap for CoreDNS or update/migrate it in case it already exists - _, corefile, currentInstalledCoreDNSVersion, err := GetCoreDNSInfo(client) - if err != nil { - return errors.Wrap(err, "unable to fetch CoreDNS current installed version and ConfigMap.") - } - - corefileMigrationRequired, err := isCoreDNSConfigMapMigrationRequired(corefile, currentInstalledCoreDNSVersion) - if err != nil { - return err - } - - // Assume that migration is always possible, rely on migrateCoreDNSCorefile() to fail if not. - canMigrateCorefile := true - - if corefile == "" || migration.Default("", corefile) { - // If the Corefile is empty or default, the latest default Corefile will be applied - if err := apiclient.CreateOrUpdateConfigMap(client, coreDNSConfigMap); err != nil { - return err - } - } else if corefileMigrationRequired { - // If migration is required, try and migrate the Corefile - if err := migrateCoreDNSCorefile(client, coreDNSConfigMap, corefile, currentInstalledCoreDNSVersion); err != nil { - // Errors in Corefile Migration is verified during preflight checks. This part will be executed when a user has chosen - // to ignore preflight check errors. - canMigrateCorefile = false - klog.Warningf("the CoreDNS Configuration was not migrated: %v. The existing CoreDNS Corefile configuration has been retained.", err) - if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { - return err - } - } - } else { - // If the Corefile is modified and doesn't require any migration, it'll be retained for the benefit of the user - if err := apiclient.CreateOrRetainConfigMap(client, coreDNSConfigMap, kubeadmconstants.CoreDNSConfigMap); err != nil { - return err - } - } - - coreDNSClusterRoles := &rbac.ClusterRole{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSClusterRole), coreDNSClusterRoles); err != nil { - return errors.Wrapf(err, "%s ClusterRole", unableToDecodeCoreDNS) - } - - // Create the Clusterroles for CoreDNS or update it in case it already exists - if err := apiclient.CreateOrUpdateClusterRole(client, coreDNSClusterRoles); err != nil { - return err - } - - coreDNSClusterRolesBinding := &rbac.ClusterRoleBinding{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSClusterRoleBinding), coreDNSClusterRolesBinding); err != nil { - return errors.Wrapf(err, "%s ClusterRoleBinding", unableToDecodeCoreDNS) - } - - // Create the Clusterrolebindings for CoreDNS or update it in case it already exists - if err := apiclient.CreateOrUpdateClusterRoleBinding(client, coreDNSClusterRolesBinding); err != nil { - return err - } - - coreDNSServiceAccount := &v1.ServiceAccount{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), []byte(CoreDNSServiceAccount), coreDNSServiceAccount); err != nil { - return errors.Wrapf(err, "%s ServiceAccount", unableToDecodeCoreDNS) - } - - // Create the ConfigMap for CoreDNS or update it in case it already exists - if err := apiclient.CreateOrUpdateServiceAccount(client, coreDNSServiceAccount); err != nil { - return err - } - - coreDNSDeployment := &apps.Deployment{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), deploymentBytes, coreDNSDeployment); err != nil { - return errors.Wrapf(err, "%s Deployment", unableToDecodeCoreDNS) - } - - // Create the deployment for CoreDNS or retain it in case the CoreDNS migration has failed during upgrade - if !canMigrateCorefile { - if err := apiclient.CreateOrRetainDeployment(client, coreDNSDeployment, kubeadmconstants.CoreDNSDeploymentName); err != nil { - return err - } - } else { - // Create the Deployment for CoreDNS or update it in case it already exists - if err := apiclient.CreateOrUpdateDeployment(client, coreDNSDeployment); err != nil { - return err - } - } - - coreDNSService := &v1.Service{} - return createDNSService(coreDNSService, serviceBytes, client) -} - -func createDNSService(dnsService *v1.Service, serviceBytes []byte, client clientset.Interface) error { - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), serviceBytes, dnsService); err != nil { - return errors.Wrap(err, "unable to decode the DNS service") - } - - // Can't use a generic apiclient helper func here as we have to tolerate more than AlreadyExists. - if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Create(context.TODO(), dnsService, metav1.CreateOptions{}); err != nil { - // Ignore if the Service is invalid with this error message: - // Service "kube-dns" is invalid: spec.clusterIP: Invalid value: "10.96.0.10": provided IP is already allocated - - if !apierrors.IsAlreadyExists(err) && !apierrors.IsInvalid(err) { - return errors.Wrap(err, "unable to create a new DNS service") - } - - if _, err := client.CoreV1().Services(metav1.NamespaceSystem).Update(context.TODO(), dnsService, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to create/update the DNS service") - } - } - return nil -} - -// isCoreDNSConfigMapMigrationRequired checks if a migration of the CoreDNS ConfigMap is required. -func isCoreDNSConfigMapMigrationRequired(corefile, currentInstalledCoreDNSVersion string) (bool, error) { - var isMigrationRequired bool - - // Current installed version is expected to be empty for init - if currentInstalledCoreDNSVersion == "" { - return isMigrationRequired, nil - } - deprecated, err := migration.Deprecated(currentInstalledCoreDNSVersion, kubeadmconstants.CoreDNSVersion, corefile) - if err != nil { - return isMigrationRequired, errors.Wrap(err, "unable to get list of changes to the configuration.") - } - - // Check if there are any plugins/options which needs to be removed or is a new default - for _, dep := range deprecated { - if dep.Severity == "removed" || dep.Severity == "newdefault" { - isMigrationRequired = true - } - } - - return isMigrationRequired, nil -} - -func migrateCoreDNSCorefile(client clientset.Interface, cm *v1.ConfigMap, corefile, currentInstalledCoreDNSVersion string) error { - // Since the current configuration present is not the default version, try and migrate it. - updatedCorefile, err := migration.Migrate(currentInstalledCoreDNSVersion, kubeadmconstants.CoreDNSVersion, corefile, false) - if err != nil { - return errors.Wrap(err, "unable to migrate CoreDNS ConfigMap") - } - - // Take a copy of the existing Corefile data as `Corefile-backup` and update the ConfigMap - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(context.TODO(), &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.CoreDNSConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - "Corefile": updatedCorefile, - "Corefile-backup": corefile, - }, - }, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update the CoreDNS ConfigMap") - } - - // Point the CoreDNS deployment to the `Corefile-backup` data. - if err := setCorefile(client, "Corefile-backup"); err != nil { - return err - } - - fmt.Println("[addons] Migrating CoreDNS Corefile") - changes, err := migration.Deprecated(currentInstalledCoreDNSVersion, kubeadmconstants.CoreDNSVersion, corefile) - if err != nil { - return errors.Wrap(err, "unable to get list of changes to the configuration.") - } - // show the migration changes - klog.V(2).Infof("the CoreDNS configuration has been migrated and applied: %v.", updatedCorefile) - klog.V(2).Infoln("the old migration has been saved in the CoreDNS ConfigMap under the name [Corefile-backup]") - klog.V(2).Infoln("The changes in the new CoreDNS Configuration are as follows:") - for _, change := range changes { - klog.V(2).Infof("%v", change.ToString()) - } - return nil -} - -// GetCoreDNSInfo gets the current CoreDNS installed and the current Corefile Configuration of CoreDNS. -func GetCoreDNSInfo(client clientset.Interface) (*v1.ConfigMap, string, string, error) { - coreDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.CoreDNSConfigMap, metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return nil, "", "", err - } - if apierrors.IsNotFound(err) { - return nil, "", "", nil - } - corefile, ok := coreDNSConfigMap.Data["Corefile"] - if !ok { - return nil, "", "", errors.New("unable to find the CoreDNS Corefile data") - } - - _, currentCoreDNSversion, err := DeployedDNSAddon(client) - if err != nil { - return nil, "", "", err - } - - return coreDNSConfigMap, corefile, currentCoreDNSversion, nil -} - -func setCorefile(client clientset.Interface, coreDNSCorefileName string) error { - dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.CoreDNSDeploymentName, metav1.GetOptions{}) - if err != nil { - return err - } - patch := fmt.Sprintf(`{"spec":{"template":{"spec":{"volumes":[{"name": "config-volume", "configMap":{"name": "coredns", "items":[{"key": "%s", "path": "Corefile"}]}}]}}}}`, coreDNSCorefileName) - - if _, err := client.AppsV1().Deployments(dnsDeployment.ObjectMeta.Namespace).Patch(context.TODO(), dnsDeployment.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil { - return errors.Wrap(err, "unable to patch the CoreDNS deployment") - } - return nil -} - -// translateStubDomainOfKubeDNSToForwardCoreDNS translates StubDomain Data in kube-dns ConfigMap -// in the form of Proxy for the CoreDNS Corefile. -func translateStubDomainOfKubeDNSToForwardCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) { - if kubeDNSConfigMap == nil { - return "", nil - } - - if proxy, ok := kubeDNSConfigMap.Data[dataField]; ok { - stubDomainData := make(map[string][]string) - err := json.Unmarshal([]byte(proxy), &stubDomainData) - if err != nil { - return "", errors.Wrap(err, "failed to parse JSON from 'kube-dns ConfigMap") - } - - var proxyStanza []interface{} - for domain, proxyHosts := range stubDomainData { - proxyIP, err := omitHostnameInTranslation(proxyHosts) - if err != nil { - return "", errors.Wrap(err, "invalid format to parse for proxy") - } - if len(proxyIP) == 0 { - continue - } - - pStanza := map[string]interface{}{} - pStanza["keys"] = []string{domain + ":53"} - pStanza["body"] = [][]string{ - {"errors"}, - {"cache", "30"}, - {"loop"}, - append([]string{"forward", "."}, proxyIP...), - } - proxyStanza = append(proxyStanza, pStanza) - } - stanzasBytes, err := json.Marshal(proxyStanza) - if err != nil { - return "", err - } - - corefileStanza, err := caddyfile.FromJSON(stanzasBytes) - if err != nil { - return "", err - } - - return prepCorefileFormat(string(corefileStanza), 4), nil - } - return "", nil -} - -// translateUpstreamNameServerOfKubeDNSToUpstreamForwardCoreDNS translates UpstreamNameServer Data in kube-dns ConfigMap -// in the form of Proxy for the CoreDNS Corefile. -func translateUpstreamNameServerOfKubeDNSToUpstreamForwardCoreDNS(dataField string, kubeDNSConfigMap *v1.ConfigMap) (string, error) { - if kubeDNSConfigMap == nil { - return "/etc/resolv.conf", nil - } - - if upstreamValues, ok := kubeDNSConfigMap.Data[dataField]; ok { - var upstreamProxyValues []string - - err := json.Unmarshal([]byte(upstreamValues), &upstreamProxyValues) - if err != nil { - return "", errors.Wrap(err, "failed to parse JSON from 'kube-dns ConfigMap") - } - - upstreamProxyValues, err = omitHostnameInTranslation(upstreamProxyValues) - if err != nil { - return "", errors.Wrap(err, "invalid format to parse for proxy") - } - - coreDNSProxyStanzaList := strings.Join(upstreamProxyValues, " ") - return coreDNSProxyStanzaList, nil - } - return "/etc/resolv.conf", nil -} - -// prepCorefileFormat indents the output of the Corefile caddytext and replaces tabs with spaces -// to neatly format the configmap, making it readable. -func prepCorefileFormat(s string, indentation int) string { - var r []string - if s == "" { - return "" - } - for _, line := range strings.Split(s, "\n") { - indented := strings.Repeat(" ", indentation) + line - r = append(r, indented) - } - corefile := strings.Join(r, "\n") - return "\n" + strings.Replace(corefile, "\t", " ", -1) -} - -// omitHostnameInTranslation checks if the data extracted from the kube-dns ConfigMap contains a valid -// IP address. Hostname to nameservers is not supported on CoreDNS and will -// skip that particular instance, if there is any hostname present. -func omitHostnameInTranslation(forwardIPs []string) ([]string, error) { - index := 0 - for _, value := range forwardIPs { - proxyHost, _, err := kubeadmutil.ParseHostPort(value) - if err != nil { - return nil, err - } - parseIP := net.ParseIP(proxyHost) - if parseIP == nil { - klog.Warningf("your kube-dns configuration contains a hostname %v. It will be omitted in the translation to CoreDNS as hostnames are unsupported", proxyHost) - } else { - forwardIPs[index] = value - index++ - } - } - forwardIPs = forwardIPs[:index] - - return forwardIPs, nil -} diff --git a/cmd/kubeadm/app/phases/addons/dns/dns_test.go b/cmd/kubeadm/app/phases/addons/dns/dns_test.go deleted file mode 100644 index d0476d1e330af..0000000000000 --- a/cmd/kubeadm/app/phases/addons/dns/dns_test.go +++ /dev/null @@ -1,954 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package dns - -import ( - "context" - "strings" - "testing" - - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - clientsetfake "k8s.io/client-go/kubernetes/fake" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" - core "k8s.io/client-go/testing" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -func TestCreateServiceAccount(t *testing.T) { - tests := []struct { - name string - createErr error - expectErr bool - }{ - { - "error-free case", - nil, - false, - }, - { - "duplication errors should be ignored", - apierrors.NewAlreadyExists(schema.GroupResource{}, ""), - false, - }, - { - "unexpected errors should be returned", - apierrors.NewUnauthorized(""), - true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - if tc.createErr != nil { - client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, tc.createErr - }) - } - - err := CreateServiceAccount(client) - if tc.expectErr { - if err == nil { - t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name) - } - return - } else if !tc.expectErr && err != nil { - t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err) - } - - wantResourcesCreated := 1 - if len(client.Actions()) != wantResourcesCreated { - t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions())) - } - - for _, action := range client.Actions() { - if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" { - t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]", - tc.name, action.GetVerb(), action.GetResource().Resource) - } - } - }) - } -} - -func TestCompileManifests(t *testing.T) { - replicas := int32(coreDNSReplicas) - var tests = []struct { - name string - manifest string - data interface{} - }{ - { - name: "KubeDNSDeployment manifest", - manifest: KubeDNSDeployment, - data: struct { - DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, OldControlPlaneTaintKey, ControlPlaneTaintKey string - Replicas *int32 - }{ - DeploymentName: "foo", - KubeDNSImage: "foo", - DNSMasqImage: "foo", - SidecarImage: "foo", - DNSBindAddr: "foo", - DNSProbeAddr: "foo", - DNSDomain: "foo", - OldControlPlaneTaintKey: "foo", - ControlPlaneTaintKey: "foo", - Replicas: &replicas, - }, - }, - { - name: "KubeDNSService manifest", - manifest: KubeDNSService, - data: struct{ DNSIP string }{ - DNSIP: "foo", - }, - }, - { - name: "CoreDNSDeployment manifest", - manifest: CoreDNSDeployment, - data: struct { - DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey string - Replicas *int32 - }{ - DeploymentName: "foo", - Image: "foo", - OldControlPlaneTaintKey: "foo", - ControlPlaneTaintKey: "foo", - Replicas: &replicas, - }, - }, - { - name: "CoreDNSConfigMap manifest", - manifest: CoreDNSConfigMap, - data: struct{ DNSDomain, UpstreamNameserver, StubDomain string }{ - DNSDomain: "foo", - UpstreamNameserver: "foo", - StubDomain: "foo", - }, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - _, err := kubeadmutil.ParseTemplate(rt.manifest, rt.data) - if err != nil { - t.Errorf("unexpected ParseTemplate failure: %+v", err) - } - }) - } -} - -func TestGetDNSIP(t *testing.T) { - var tests = []struct { - name, svcSubnet, expectedDNSIP string - isDualStack bool - }{ - { - name: "subnet mask 12", - svcSubnet: "10.96.0.0/12", - expectedDNSIP: "10.96.0.10", - isDualStack: false, - }, - { - name: "subnet mask 26", - svcSubnet: "10.87.116.64/26", - expectedDNSIP: "10.87.116.74", - isDualStack: false, - }, - { - name: "dual-stack ipv4 primary, subnet mask 26", - svcSubnet: "10.87.116.64/26,fd03::/112", - expectedDNSIP: "10.87.116.74", - isDualStack: true, - }, - { - name: "dual-stack ipv6 primary, subnet mask 112", - svcSubnet: "fd03::/112,10.87.116.64/26", - expectedDNSIP: "fd03::a", - isDualStack: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - dnsIP, err := kubeadmconstants.GetDNSIP(rt.svcSubnet, rt.isDualStack) - if err != nil { - t.Fatalf("couldn't get dnsIP : %v", err) - } - - actualDNSIP := dnsIP.String() - if actualDNSIP != rt.expectedDNSIP { - t.Errorf( - "failed GetDNSIP\n\texpected: %s\n\t actual: %s", - rt.expectedDNSIP, - actualDNSIP, - ) - } - }) - } -} - -func TestTranslateStubDomainKubeDNSToCoreDNS(t *testing.T) { - testCases := []struct { - name string - configMap *v1.ConfigMap - expectOne string - expectTwo string - }{ - { - name: "valid call with multiple IPs", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": `{"foo.com" : ["1.2.3.4:5300","3.3.3.3"], "my.cluster.local" : ["2.3.4.5"]}`, - "upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`, - }, - }, - - expectOne: ` - foo.com:53 { - errors - cache 30 - loop - forward . 1.2.3.4:5300 3.3.3.3 - } - - my.cluster.local:53 { - errors - cache 30 - loop - forward . 2.3.4.5 - }`, - expectTwo: ` - my.cluster.local:53 { - errors - cache 30 - loop - forward . 2.3.4.5 - } - - foo.com:53 { - errors - cache 30 - loop - forward . 1.2.3.4:5300 3.3.3.3 - }`, - }, - { - name: "empty call", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubedns", - Namespace: "kube-system", - }, - }, - - expectOne: "", - }, - { - name: "valid call", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": `{"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`, - "upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`, - }, - }, - - expectOne: ` - foo.com:53 { - errors - cache 30 - loop - forward . 1.2.3.4:5300 - } - - my.cluster.local:53 { - errors - cache 30 - loop - forward . 2.3.4.5 - }`, - expectTwo: ` - my.cluster.local:53 { - errors - cache 30 - loop - forward . 2.3.4.5 - } - - foo.com:53 { - errors - cache 30 - loop - forward . 1.2.3.4:5300 - }`, - }, - { - name: "If Hostname present: Omit Hostname", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": `{"bar.com" : ["1.2.3.4:5300","service.consul"], "my.cluster.local" : ["2.3.4.5"], "foo.com" : ["service.consul"]}`, - "upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`, - }, - }, - - expectOne: ` - bar.com:53 { - errors - cache 30 - loop - forward . 1.2.3.4:5300 - } - - my.cluster.local:53 { - errors - cache 30 - loop - forward . 2.3.4.5 - }`, - expectTwo: ` - my.cluster.local:53 { - errors - cache 30 - loop - forward . 2.3.4.5 - } - - bar.com:53 { - errors - cache 30 - loop - forward . 1.2.3.4:5300 - }`, - }, - { - name: "All hostname: return empty", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": `{"foo.com" : ["service.consul"], "my.cluster.local" : ["ns.foo.com"]}`, - "upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`, - }, - }, - - expectOne: "", - expectTwo: "", - }, - { - name: "missing stubDomains", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`, - }, - }, - - expectOne: "", - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - out, err := translateStubDomainOfKubeDNSToForwardCoreDNS(kubeDNSStubDomain, testCase.configMap) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if !strings.EqualFold(out, testCase.expectOne) && !strings.EqualFold(out, testCase.expectTwo) { - t.Errorf("expected to find %q or %q in output: %q", testCase.expectOne, testCase.expectTwo, out) - } - }) - } -} - -func TestTranslateUpstreamKubeDNSToCoreDNS(t *testing.T) { - testCases := []struct { - name string - configMap *v1.ConfigMap - expect string - }{ - { - name: "expect resolv.conf", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - }, - - expect: "/etc/resolv.conf", - }, - { - name: "expect list of Name Server IP addresses", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubedns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "stubDomains": ` {"foo.com" : ["1.2.3.4:5300"], "my.cluster.local" : ["2.3.4.5"]}`, - "upstreamNameservers": `["8.8.8.8", "8.8.4.4", "4.4.4.4"]`, - }, - }, - - expect: "8.8.8.8 8.8.4.4 4.4.4.4", - }, - { - name: "no stubDomains: expect list of Name Server IP addresses", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubedns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "upstreamNameservers": `["8.8.8.8", "8.8.4.4"]`, - }, - }, - - expect: "8.8.8.8 8.8.4.4", - }, - { - name: "Hostname present: expect NameServer to omit the hostname", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubedns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "upstreamNameservers": `["service.consul", "ns.foo.com", "8.8.4.4", "ns.moo.com", "ns.bar.com"]`, - }, - }, - - expect: "8.8.4.4", - }, - { - name: "All hostnames: return empty", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-dns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "upstreamNameservers": `["service.consul", "ns.foo.com"]`, - }, - }, - - expect: "", - }, - { - name: "IPv6: expect list of Name Server IP addresses", - configMap: &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubedns", - Namespace: "kube-system", - }, - Data: map[string]string{ - "upstreamNameservers": `["[2003::1]:53", "8.8.4.4"]`, - }, - }, - - expect: "[2003::1]:53 8.8.4.4", - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - out, err := translateUpstreamNameServerOfKubeDNSToUpstreamForwardCoreDNS(kubeDNSUpstreamNameservers, testCase.configMap) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if !strings.EqualFold(out, testCase.expect) { - t.Errorf("expected to find %q in output: %q", testCase.expect, out) - } - }) - } -} - -func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) { - replicas := int32(coreDNSReplicas) - testCases := []struct { - name string - manifest string - data interface{} - }{ - { - name: "KubeDNSDeployment", - manifest: KubeDNSDeployment, - data: struct { - DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, OldControlPlaneTaintKey, ControlPlaneTaintKey string - Replicas *int32 - }{ - DeploymentName: "foo", - KubeDNSImage: "foo", - DNSMasqImage: "foo", - SidecarImage: "foo", - DNSBindAddr: "foo", - DNSProbeAddr: "foo", - DNSDomain: "foo", - OldControlPlaneTaintKey: "foo", - ControlPlaneTaintKey: "foo", - Replicas: &replicas, - }, - }, - { - name: "CoreDNSDeployment", - manifest: CoreDNSDeployment, - data: struct { - DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey, CoreDNSConfigMapName string - Replicas *int32 - }{ - DeploymentName: "foo", - Image: "foo", - OldControlPlaneTaintKey: "foo", - ControlPlaneTaintKey: "foo", - CoreDNSConfigMapName: "foo", - Replicas: &replicas, - }, - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - deploymentBytes, _ := kubeadmutil.ParseTemplate(testCase.manifest, testCase.data) - deployment := &apps.Deployment{} - if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), deploymentBytes, deployment); err != nil { - t.Errorf("unexpected error: %v", err) - } - if deployment.Spec.Template.Spec.PriorityClassName != "system-cluster-critical" { - t.Errorf("expected to see system-cluster-critical priority class name. Got %q instead", deployment.Spec.Template.Spec.PriorityClassName) - } - }) - } -} - -func TestCreateCoreDNSAddon(t *testing.T) { - tests := []struct { - name string - initialCorefileData string - expectedCorefileData string - coreDNSVersion string - }{ - { - name: "Empty Corefile", - initialCorefileData: "", - expectedCorefileData: `.:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance -} -`, - coreDNSVersion: "1.6.7", - }, - { - name: "Default Corefile", - initialCorefileData: `.:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf - cache 30 - loop - reload - loadbalance - } -`, - expectedCorefileData: `.:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance -} -`, - coreDNSVersion: "1.6.7", - }, - { - name: "Modified Corefile with only newdefaults needed", - initialCorefileData: `.:53 { - errors - log - health - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf - cache 30 - loop - reload - loadbalance - } -`, - expectedCorefileData: `.:53 { - errors - log - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance -} -`, - coreDNSVersion: "1.6.2", - }, - { - name: "Default Corefile with rearranged plugins", - initialCorefileData: `.:53 { - errors - cache 30 - prometheus :9153 - forward . /etc/resolv.conf - loop - reload - loadbalance - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - upstream - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - health - } -`, - expectedCorefileData: `.:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance -} -`, - coreDNSVersion: "1.3.1", - }, - { - name: "Remove Deprecated options", - initialCorefileData: `.:53 { - errors - logs - health - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - upstream - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf - cache 30 - loop - reload - loadbalance - }`, - expectedCorefileData: `.:53 { - errors - logs - health { - lameduck 5s - } - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance - ready -} -`, - coreDNSVersion: "1.3.1", - }, - { - name: "Update proxy plugin to forward plugin", - initialCorefileData: `.:53 { - errors - health - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - upstream - fallthrough in-addr.arpa ip6.arpa - } - prometheus :9153 - proxy . /etc/resolv.conf - k8s_external example.com - cache 30 - loop - reload - loadbalance - }`, - expectedCorefileData: `.:53 { - errors - health { - lameduck 5s - } - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - k8s_external example.com - cache 30 - loop - reload - loadbalance - ready -} -`, - coreDNSVersion: "1.3.1", - }, - { - name: "Modified Corefile with no migration required", - initialCorefileData: `consul { - errors - forward . 10.10.96.16:8600 10.10.96.17:8600 10.10.96.18:8600 { - max_concurrent 1000 - } - loadbalance - cache 5 - reload - } - domain.int { - errors - forward . 10.10.0.140 10.10.0.240 10.10.51.40 { - max_concurrent 1000 - } - loadbalance - cache 3600 - reload - } - .:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - } - prometheus :9153 - forward . /etc/resolv.conf { - prefer_udp - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance - } -`, - expectedCorefileData: `consul { - errors - forward . 10.10.96.16:8600 10.10.96.17:8600 10.10.96.18:8600 { - max_concurrent 1000 - } - loadbalance - cache 5 - reload - } - domain.int { - errors - forward . 10.10.0.140 10.10.0.240 10.10.51.40 { - max_concurrent 1000 - } - loadbalance - cache 3600 - reload - } - .:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - } - prometheus :9153 - forward . /etc/resolv.conf { - prefer_udp - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance - } -`, - coreDNSVersion: "1.6.7", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client := createClientAndCoreDNSManifest(t, tc.initialCorefileData, tc.coreDNSVersion) - - configMapBytes, err := kubeadmutil.ParseTemplate(CoreDNSConfigMap, struct{ DNSDomain, UpstreamNameserver, StubDomain string }{ - DNSDomain: "cluster.local", - UpstreamNameserver: "/etc/resolv.conf", - StubDomain: "", - }) - if err != nil { - t.Errorf("unexpected ParseTemplate failure: %+v", err) - } - - err = createCoreDNSAddon(nil, nil, configMapBytes, client) - if err != nil { - t.Fatalf("error creating the CoreDNS Addon: %v", err) - } - migratedConfigMap, _ := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.CoreDNSConfigMap, metav1.GetOptions{}) - if !strings.EqualFold(migratedConfigMap.Data["Corefile"], tc.expectedCorefileData) { - t.Fatalf("expected to get %v, but got %v", tc.expectedCorefileData, migratedConfigMap.Data["Corefile"]) - } - }) - } -} - -func createClientAndCoreDNSManifest(t *testing.T, corefile, coreDNSVersion string) *clientsetfake.Clientset { - client := clientsetfake.NewSimpleClientset() - _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(context.TODO(), &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.CoreDNSConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - "Corefile": corefile, - }, - }, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("error creating ConfigMap: %v", err) - } - _, err = client.AppsV1().Deployments(metav1.NamespaceSystem).Create(context.TODO(), &apps.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.CoreDNSConfigMap, - Namespace: metav1.NamespaceSystem, - Labels: map[string]string{ - "k8s-app": "kube-dns", - }, - }, - Spec: apps.DeploymentSpec{ - Template: v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Image: "test:" + coreDNSVersion, - }, - }, - }, - }, - }, - }, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("error creating deployment: %v", err) - } - return client -} diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go deleted file mode 100644 index 014cbd773c22d..0000000000000 --- a/cmd/kubeadm/app/phases/addons/dns/manifests.go +++ /dev/null @@ -1,383 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package dns - -const ( - // KubeDNSDeployment is the kube-dns Deployment manifest for the kube-dns manifest for v1.7+ - KubeDNSDeployment = ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .DeploymentName }} - namespace: kube-system - labels: - k8s-app: kube-dns -spec: - replicas: {{ .Replicas }} - strategy: - rollingUpdate: - maxSurge: 10% - maxUnavailable: 0 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - spec: - priorityClassName: system-cluster-critical - volumes: - - name: kube-dns-config - configMap: - name: kube-dns - optional: true - containers: - - name: kubedns - image: {{ .KubeDNSImage }} - imagePullPolicy: IfNotPresent - resources: - # TODO: Set memory limits when we've profiled the container for large - # clusters, then set request = limit to keep this container in - # guaranteed class. Currently, this container falls into the - # "burstable" category so the kubelet doesn't backoff from restarting it. - limits: - memory: 170Mi - requests: - cpu: 100m - memory: 70Mi - livenessProbe: - httpGet: - path: /healthcheck/kubedns - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /readiness - port: 8081 - scheme: HTTP - # we poll on pod startup for the Kubernetes control-plane service and - # only setup the /readiness HTTP server once that's available. - initialDelaySeconds: 3 - timeoutSeconds: 5 - args: - - --domain={{ .DNSDomain }}. - - --dns-port=10053 - - --config-dir=/kube-dns-config - - --v=2 - env: - - name: PROMETHEUS_PORT - value: "10055" - ports: - - containerPort: 10053 - name: dns-local - protocol: UDP - - containerPort: 10053 - name: dns-tcp-local - protocol: TCP - - containerPort: 10055 - name: metrics - protocol: TCP - volumeMounts: - - name: kube-dns-config - mountPath: /kube-dns-config - - name: dnsmasq - image: {{ .DNSMasqImage }} - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthcheck/dnsmasq - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - -v=2 - - -logtostderr - - -configDir=/etc/k8s/dns/dnsmasq-nanny - - -restartDnsmasq=true - - -- - - -k - - --cache-size=1000 - - --no-negcache - - --dns-loop-detect - - --log-facility=- - - --server=/{{ .DNSDomain }}/{{ .DNSBindAddr }}#10053 - - --server=/in-addr.arpa/{{ .DNSBindAddr }}#10053 - - --server=/ip6.arpa/{{ .DNSBindAddr }}#10053 - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - # see: https://github.com/kubernetes/kubernetes/issues/29055 for details - resources: - requests: - cpu: 150m - memory: 20Mi - volumeMounts: - - name: kube-dns-config - mountPath: /etc/k8s/dns/dnsmasq-nanny - - name: sidecar - image: {{ .SidecarImage }} - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /metrics - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - --v=2 - - --logtostderr - - --probe=kubedns,{{ .DNSProbeAddr }}:10053,kubernetes.default.svc.{{ .DNSDomain }},5,SRV - - --probe=dnsmasq,{{ .DNSProbeAddr }}:53,kubernetes.default.svc.{{ .DNSDomain }},5,SRV - ports: - - containerPort: 10054 - name: metrics - protocol: TCP - resources: - requests: - memory: 20Mi - cpu: 10m - dnsPolicy: Default # Don't use cluster DNS. - serviceAccountName: kube-dns - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: {{ .OldControlPlaneTaintKey }} - effect: NoSchedule - - key: {{ .ControlPlaneTaintKey }} - effect: NoSchedule -` - - // KubeDNSService is the kube-dns Service manifest - KubeDNSService = ` -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeDNS" - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - # Without this resourceVersion value, an update of the Service between versions will yield: - # Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update - resourceVersion: "0" -spec: - clusterIP: {{ .DNSIP }} - ports: - - name: dns - port: 53 - protocol: UDP - targetPort: 53 - - name: dns-tcp - port: 53 - protocol: TCP - targetPort: 53 - - name: metrics - port: 9153 - protocol: TCP - targetPort: 9153 - selector: - k8s-app: kube-dns -` - - // CoreDNSDeployment is the CoreDNS Deployment manifest - CoreDNSDeployment = ` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .DeploymentName }} - namespace: kube-system - labels: - k8s-app: kube-dns -spec: - replicas: {{ .Replicas }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - spec: - priorityClassName: system-cluster-critical - serviceAccountName: coredns - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: {{ .OldControlPlaneTaintKey }} - effect: NoSchedule - - key: {{ .ControlPlaneTaintKey }} - effect: NoSchedule - nodeSelector: - kubernetes.io/os: linux - containers: - - name: coredns - image: {{ .Image }} - imagePullPolicy: IfNotPresent - resources: - limits: - memory: 170Mi - requests: - cpu: 100m - memory: 70Mi - args: [ "-conf", "/etc/coredns/Corefile" ] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9153 - name: metrics - protocol: TCP - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - all - readOnlyRootFilesystem: true - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile -` - - // CoreDNSConfigMap is the CoreDNS ConfigMap manifest - CoreDNSConfigMap = ` -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system -data: - Corefile: | - .:53 { - errors - health { - lameduck 5s - } - ready - kubernetes {{ .DNSDomain }} in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 30 - } - prometheus :9153 - forward . {{ .UpstreamNameserver }} { - max_concurrent 1000 - } - cache 30 - loop - reload - loadbalance - }{{ .StubDomain }} -` - // CoreDNSClusterRole is the CoreDNS ClusterRole manifest - CoreDNSClusterRole = ` -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system:coredns -rules: -- apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -` - // CoreDNSClusterRoleBinding is the CoreDNS Clusterrolebinding manifest - CoreDNSClusterRoleBinding = ` -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system:coredns -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns -subjects: -- kind: ServiceAccount - name: coredns - namespace: kube-system -` - // CoreDNSServiceAccount is the CoreDNS ServiceAccount manifest - CoreDNSServiceAccount = ` -apiVersion: v1 -kind: ServiceAccount -metadata: - name: coredns - namespace: kube-system -` -) diff --git a/cmd/kubeadm/app/phases/addons/proxy/BUILD b/cmd/kubeadm/app/phases/addons/proxy/BUILD deleted file mode 100644 index 03a3dc7328bd5..0000000000000 --- a/cmd/kubeadm/app/phases/addons/proxy/BUILD +++ /dev/null @@ -1,64 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["proxy_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "manifests.go", - "proxy.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/addons/proxy/manifests.go b/cmd/kubeadm/app/phases/addons/proxy/manifests.go deleted file mode 100644 index 9f2a6a2fe927b..0000000000000 --- a/cmd/kubeadm/app/phases/addons/proxy/manifests.go +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proxy - -const ( - // KubeProxyConfigMap19 is the proxy ConfigMap manifest for Kubernetes 1.9 and above - KubeProxyConfigMap19 = ` -kind: ConfigMap -apiVersion: v1 -metadata: - name: {{ .ProxyConfigMap }} - namespace: kube-system - labels: - app: kube-proxy -data: - kubeconfig.conf: |- - apiVersion: v1 - kind: Config - clusters: - - cluster: - certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - server: {{ .ControlPlaneEndpoint }} - name: default - contexts: - - context: - cluster: default - namespace: default - user: default - name: default - current-context: default - users: - - name: default - user: - tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token - {{ .ProxyConfigMapKey }}: |- -{{ .ProxyConfig}} -` - - // KubeProxyDaemonSet19 is the proxy DaemonSet manifest for Kubernetes 1.9 and above - KubeProxyDaemonSet19 = ` -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: kube-proxy - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: kube-proxy - spec: - priorityClassName: system-node-critical - containers: - - name: kube-proxy - image: {{ .Image }} - imagePullPolicy: IfNotPresent - command: - - /usr/local/bin/kube-proxy - - --config=/var/lib/kube-proxy/{{ .ProxyConfigMapKey }} - - --hostname-override=$(NODE_NAME) - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: true - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - hostNetwork: true - serviceAccountName: kube-proxy - volumes: - - name: kube-proxy - configMap: - name: {{ .ProxyConfigMap }} - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: lib-modules - hostPath: - path: /lib/modules - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - nodeSelector: - kubernetes.io/os: linux -` -) diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy.go b/cmd/kubeadm/app/phases/addons/proxy/proxy.go deleted file mode 100644 index 024e43e0fc562..0000000000000 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy.go +++ /dev/null @@ -1,212 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proxy - -import ( - "bytes" - "fmt" - - "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kuberuntime "k8s.io/apimachinery/pkg/runtime" - clientset "k8s.io/client-go/kubernetes" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // KubeProxyClusterRoleName sets the name for the kube-proxy ClusterRole - // TODO: This k8s-generic, well-known constant should be fetchable from another source, not be in this package - KubeProxyClusterRoleName = "system:node-proxier" - - // KubeProxyServiceAccountName describes the name of the ServiceAccount for the kube-proxy addon - KubeProxyServiceAccountName = "kube-proxy" -) - -// EnsureProxyAddon creates the kube-proxy addons -func EnsureProxyAddon(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface) error { - if err := CreateServiceAccount(client); err != nil { - return errors.Wrap(err, "error when creating kube-proxy service account") - } - - if err := createKubeProxyConfigMap(cfg, localEndpoint, client); err != nil { - return err - } - - if err := createKubeProxyAddon(cfg, client); err != nil { - return err - } - - if err := CreateRBACRules(client); err != nil { - return errors.Wrap(err, "error when creating kube-proxy RBAC rules") - } - - fmt.Println("[addons] Applied essential addon: kube-proxy") - return nil -} - -// CreateServiceAccount creates the necessary serviceaccounts that kubeadm uses/might use, if they don't already exist. -func CreateServiceAccount(client clientset.Interface) error { - - return apiclient.CreateOrUpdateServiceAccount(client, &v1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: KubeProxyServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }) -} - -// CreateRBACRules creates the essential RBAC rules for a minimally set-up cluster -func CreateRBACRules(client clientset.Interface) error { - return createClusterRoleBindings(client) -} - -func createKubeProxyConfigMap(cfg *kubeadmapi.ClusterConfiguration, localEndpoint *kubeadmapi.APIEndpoint, client clientset.Interface) error { - // Generate ControlPlane Enpoint kubeconfig file - controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, localEndpoint) - if err != nil { - return err - } - - kubeProxyCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeProxyGroup] - if !ok { - return errors.New("no kube-proxy component config found in the active component config set") - } - - proxyBytes, err := kubeProxyCfg.Marshal() - if err != nil { - return errors.Wrap(err, "error when marshaling") - } - var prefixBytes bytes.Buffer - apiclient.PrintBytesWithLinePrefix(&prefixBytes, proxyBytes, " ") - configMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap19, - struct { - ControlPlaneEndpoint string - ProxyConfig string - ProxyConfigMap string - ProxyConfigMapKey string - }{ - ControlPlaneEndpoint: controlPlaneEndpoint, - ProxyConfig: prefixBytes.String(), - ProxyConfigMap: constants.KubeProxyConfigMap, - ProxyConfigMapKey: constants.KubeProxyConfigMapKey, - }) - if err != nil { - return errors.Wrap(err, "error when parsing kube-proxy configmap template") - } - - kubeproxyConfigMap := &v1.ConfigMap{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), configMapBytes, kubeproxyConfigMap); err != nil { - return errors.Wrap(err, "unable to decode kube-proxy configmap") - } - - if !kubeProxyCfg.IsUserSupplied() { - componentconfigs.SignConfigMap(kubeproxyConfigMap) - } - - // Create the ConfigMap for kube-proxy or update it in case it already exists - return apiclient.CreateOrUpdateConfigMap(client, kubeproxyConfigMap) -} - -func createKubeProxyAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error { - daemonSetbytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{ - Image: images.GetKubernetesImage(constants.KubeProxy, cfg), - ProxyConfigMap: constants.KubeProxyConfigMap, - ProxyConfigMapKey: constants.KubeProxyConfigMapKey, - }) - if err != nil { - return errors.Wrap(err, "error when parsing kube-proxy daemonset template") - } - - kubeproxyDaemonSet := &apps.DaemonSet{} - if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), daemonSetbytes, kubeproxyDaemonSet); err != nil { - return errors.Wrap(err, "unable to decode kube-proxy daemonset") - } - // Propagate the http/https proxy host environment variables to the container - env := &kubeproxyDaemonSet.Spec.Template.Spec.Containers[0].Env - *env = append(*env, kubeadmutil.GetProxyEnvVars()...) - - // Create the DaemonSet for kube-proxy or update it in case it already exists - return apiclient.CreateOrUpdateDaemonSet(client, kubeproxyDaemonSet) -} - -func createClusterRoleBindings(client clientset.Interface) error { - if err := apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubeadm:node-proxier", - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: KubeProxyClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.ServiceAccountKind, - Name: KubeProxyServiceAccountName, - Namespace: metav1.NamespaceSystem, - }, - }, - }); err != nil { - return err - } - - // Create a role for granting read only access to the kube-proxy component config ConfigMap - if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.KubeProxyConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{constants.KubeProxyConfigMap}, - }, - }, - }); err != nil { - return err - } - - // Bind the role to bootstrap tokens for allowing fetchConfiguration during join - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.KubeProxyConfigMap, - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: constants.KubeProxyConfigMap, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: constants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} diff --git a/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go b/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go deleted file mode 100644 index e37fa3b2823c0..0000000000000 --- a/cmd/kubeadm/app/phases/addons/proxy/proxy_test.go +++ /dev/null @@ -1,255 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package proxy - -import ( - "strings" - "testing" - - apps "k8s.io/api/apps/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - clientsetfake "k8s.io/client-go/kubernetes/fake" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" - core "k8s.io/client-go/testing" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -func TestCreateServiceAccount(t *testing.T) { - tests := []struct { - name string - createErr error - expectErr bool - }{ - { - "error-free case", - nil, - false, - }, - { - "duplication errors should be ignored", - apierrors.NewAlreadyExists(schema.GroupResource{}, ""), - false, - }, - { - "unexpected errors should be returned", - apierrors.NewUnauthorized(""), - true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - if tc.createErr != nil { - client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, tc.createErr - }) - } - - err := CreateServiceAccount(client) - if tc.expectErr { - if err == nil { - t.Errorf("CreateServiceAccounts(%s) wanted err, got nil", tc.name) - } - return - } else if !tc.expectErr && err != nil { - t.Errorf("CreateServiceAccounts(%s) returned unexpected err: %v", tc.name, err) - } - - wantResourcesCreated := 1 - if len(client.Actions()) != wantResourcesCreated { - t.Errorf("CreateServiceAccounts(%s) should have made %d actions, but made %d", tc.name, wantResourcesCreated, len(client.Actions())) - } - - for _, action := range client.Actions() { - if action.GetVerb() != "create" || action.GetResource().Resource != "serviceaccounts" { - t.Errorf("CreateServiceAccounts(%s) called [%v %v], but wanted [create serviceaccounts]", - tc.name, action.GetVerb(), action.GetResource().Resource) - } - } - }) - } -} - -func TestCompileManifests(t *testing.T) { - var tests = []struct { - name string - manifest string - data interface{} - }{ - { - name: "KubeProxyConfigMap19", - manifest: KubeProxyConfigMap19, - data: struct { - ControlPlaneEndpoint, ProxyConfig, ProxyConfigMap, ProxyConfigMapKey string - }{ - ControlPlaneEndpoint: "foo", - ProxyConfig: " bindAddress: 0.0.0.0\n clusterCIDR: 192.168.1.1\n enableProfiling: false", - ProxyConfigMap: "bar", - ProxyConfigMapKey: "baz", - }, - }, - { - name: "KubeProxyDaemonSet19", - manifest: KubeProxyDaemonSet19, - data: struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{ - Image: "foo", - ProxyConfigMap: "bar", - ProxyConfigMapKey: "baz", - }, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - _, err := kubeadmutil.ParseTemplate(rt.manifest, rt.data) - if err != nil { - t.Errorf("unexpected ParseTemplate failure: %+v", err) - } - }) - } -} - -func TestEnsureProxyAddon(t *testing.T) { - type SimulatedError int - const ( - NoError SimulatedError = iota - ServiceAccountError - InvalidControlPlaneEndpoint - IPv6SetBindAddress - ) - - var testCases = []struct { - name string - simError SimulatedError - expErrString string - expBindAddr string - expClusterCIDR string - }{ - { - name: "Successful proxy addon", - simError: NoError, - expErrString: "", - expBindAddr: "0.0.0.0", - expClusterCIDR: "5.6.7.8/24", - }, { - name: "Simulated service account error", - simError: ServiceAccountError, - expErrString: "error when creating kube-proxy service account", - expBindAddr: "0.0.0.0", - expClusterCIDR: "5.6.7.8/24", - }, { - name: "IPv6 AdvertiseAddress address", - simError: IPv6SetBindAddress, - expErrString: "", - expBindAddr: "::", - expClusterCIDR: "2001:101::/96", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Create a fake client and set up default test configuration - client := clientsetfake.NewSimpleClientset() - // TODO: Consider using a YAML file instead for this that makes it possible to specify YAML documents for the ComponentConfigs - controlPlaneConfig := &kubeadmapiv1beta2.InitConfiguration{ - LocalAPIEndpoint: kubeadmapiv1beta2.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - BindPort: 1234, - }, - } - controlPlaneClusterConfig := &kubeadmapiv1beta2.ClusterConfiguration{ - Networking: kubeadmapiv1beta2.Networking{ - PodSubnet: "5.6.7.8/24", - }, - ImageRepository: "someRepo", - KubernetesVersion: constants.MinimumControlPlaneVersion.String(), - } - - // Simulate an error if necessary - switch tc.simError { - case ServiceAccountError: - client.PrependReactor("create", "serviceaccounts", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, apierrors.NewUnauthorized("") - }) - case InvalidControlPlaneEndpoint: - controlPlaneConfig.LocalAPIEndpoint.AdvertiseAddress = "1.2.3" - case IPv6SetBindAddress: - controlPlaneConfig.LocalAPIEndpoint.AdvertiseAddress = "1:2::3:4" - controlPlaneClusterConfig.Networking.PodSubnet = "2001:101::/48" - } - - intControlPlane, err := configutil.DefaultedInitConfiguration(controlPlaneConfig, controlPlaneClusterConfig) - if err != nil { - t.Errorf("test failed to convert external to internal version: %v", err) - return - } - err = EnsureProxyAddon(&intControlPlane.ClusterConfiguration, &intControlPlane.LocalAPIEndpoint, client) - - // Compare actual to expected errors - actErr := "No error" - if err != nil { - actErr = err.Error() - } - expErr := "No error" - if tc.expErrString != "" { - expErr = tc.expErrString - } - if !strings.Contains(actErr, expErr) { - t.Errorf( - "%s test failed, expected: %s, got: %s", - tc.name, - expErr, - actErr) - } - }) - } -} - -func TestDaemonSetsHaveSystemNodeCriticalPriorityClassName(t *testing.T) { - testCases := []struct { - name string - manifest string - data interface{} - }{ - { - name: "KubeProxyDaemonSet19", - manifest: KubeProxyDaemonSet19, - data: struct{ Image, ProxyConfigMap, ProxyConfigMapKey string }{ - Image: "foo", - ProxyConfigMap: "foo", - ProxyConfigMapKey: "foo", - }, - }, - } - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - daemonSetBytes, _ := kubeadmutil.ParseTemplate(testCase.manifest, testCase.data) - daemonSet := &apps.DaemonSet{} - if err := runtime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), daemonSetBytes, daemonSet); err != nil { - t.Errorf("unexpected error: %v", err) - } - if daemonSet.Spec.Template.Spec.PriorityClassName != "system-node-critical" { - t.Errorf("expected to see system-node-critical priority class name. Got %q instead", daemonSet.Spec.Template.Spec.PriorityClassName) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD deleted file mode 100644 index be3d0c85d579d..0000000000000 --- a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["clusterinfo_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = ["clusterinfo.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo", - deps = [ - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go deleted file mode 100644 index 5f6c27e3d8ccb..0000000000000 --- a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterinfo - -import ( - "fmt" - - "github.com/pkg/errors" - "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apiserver/pkg/authentication/user" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // BootstrapSignerClusterRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-public ns - BootstrapSignerClusterRoleName = "kubeadm:bootstrap-signer-clusterinfo" -) - -// CreateBootstrapConfigMapIfNotExists creates the kube-public ConfigMap if it doesn't exist already -func CreateBootstrapConfigMapIfNotExists(client clientset.Interface, file string) error { - - fmt.Printf("[bootstrap-token] Creating the %q ConfigMap in the %q namespace\n", bootstrapapi.ConfigMapClusterInfo, metav1.NamespacePublic) - - klog.V(1).Infoln("[bootstrap-token] loading admin kubeconfig") - adminConfig, err := clientcmd.LoadFromFile(file) - if err != nil { - return errors.Wrap(err, "failed to load admin kubeconfig") - } - - adminCluster := adminConfig.Contexts[adminConfig.CurrentContext].Cluster - // Copy the cluster from admin.conf to the bootstrap kubeconfig, contains the CA cert and the server URL - klog.V(1).Infoln("[bootstrap-token] copying the cluster from admin.conf to the bootstrap kubeconfig") - bootstrapConfig := &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": adminConfig.Clusters[adminCluster], - }, - } - bootstrapBytes, err := clientcmd.Write(*bootstrapConfig) - if err != nil { - return err - } - - // Create or update the ConfigMap in the kube-public namespace - klog.V(1).Infoln("[bootstrap-token] creating/updating ConfigMap in kube-public namespace") - return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: bootstrapapi.ConfigMapClusterInfo, - Namespace: metav1.NamespacePublic, - }, - Data: map[string]string{ - bootstrapapi.KubeConfigKey: string(bootstrapBytes), - }, - }) -} - -// CreateClusterInfoRBACRules creates the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace to unauthenticated users -func CreateClusterInfoRBACRules(client clientset.Interface) error { - klog.V(1).Infoln("creating the RBAC rules for exposing the cluster-info ConfigMap in the kube-public namespace") - err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: BootstrapSignerClusterRoleName, - Namespace: metav1.NamespacePublic, - }, - Rules: []rbac.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{bootstrapapi.ConfigMapClusterInfo}, - }, - }, - }) - if err != nil { - return err - } - - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: BootstrapSignerClusterRoleName, - Namespace: metav1.NamespacePublic, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: BootstrapSignerClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.UserKind, - Name: user.Anonymous, - }, - }, - }) -} diff --git a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo_test.go b/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo_test.go deleted file mode 100644 index 2bc641d45ba81..0000000000000 --- a/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo/clusterinfo_test.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clusterinfo - -import ( - "io/ioutil" - "os" - "testing" - "text/template" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - clientsetfake "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" -) - -var testConfigTempl = template.Must(template.New("test").Parse(`apiVersion: v1 -clusters: -- cluster: - server: {{.Server}} - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: kubernetes-admin - name: kubernetes-admin@kubernetes -current-context: kubernetes-admin@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-admin`)) - -func TestCreateBootstrapConfigMapIfNotExists(t *testing.T) { - tests := []struct { - name string - createErr error - updateErr error - expectErr bool - }{ - { - "successful case should have no error", - nil, - nil, - false, - }, - { - "if both create and update errors, return error", - apierrors.NewAlreadyExists(schema.GroupResource{Resource: "configmaps"}, "test"), - apierrors.NewUnauthorized("go away!"), - true, - }, - { - "unexpected error should be returned", - apierrors.NewUnauthorized("go away!"), - nil, - true, - }, - } - - servers := []struct { - Server string - }{ - {Server: "https://10.128.0.6:6443"}, - {Server: "https://[2001:db8::6]:3446"}, - } - - for _, server := range servers { - file, err := ioutil.TempFile("", "") - if err != nil { - t.Fatalf("could not create tempfile: %v", err) - } - defer os.Remove(file.Name()) - - if err := testConfigTempl.Execute(file, server); err != nil { - t.Fatalf("could not write to tempfile: %v", err) - } - - if err := file.Close(); err != nil { - t.Fatalf("could not close tempfile: %v", err) - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - if tc.createErr != nil { - client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, tc.createErr - }) - } - - err := CreateBootstrapConfigMapIfNotExists(client, file.Name()) - if tc.expectErr && err == nil { - t.Errorf("CreateBootstrapConfigMapIfNotExists(%s) wanted error, got nil", tc.name) - } else if !tc.expectErr && err != nil { - t.Errorf("CreateBootstrapConfigMapIfNotExists(%s) returned unexpected error: %v", tc.name, err) - } - }) - } - } -} diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/BUILD b/cmd/kubeadm/app/phases/bootstraptoken/node/BUILD deleted file mode 100644 index 6ee4c7c89a799..0000000000000 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = [ - "tlsbootstrap.go", - "token.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go b/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go deleted file mode 100644 index e190079a947fe..0000000000000 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/tlsbootstrap.go +++ /dev/null @@ -1,154 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "fmt" - - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // NodeBootstrapperClusterRoleName defines the name of the auto-bootstrapped ClusterRole for letting someone post a CSR - // TODO: This value should be defined in an other, generic authz package instead of here - NodeBootstrapperClusterRoleName = "system:node-bootstrapper" - // NodeKubeletBootstrap defines the name of the ClusterRoleBinding that lets kubelets post CSRs - NodeKubeletBootstrap = "kubeadm:kubelet-bootstrap" - // GetNodesClusterRoleName defines the name of the ClusterRole and ClusterRoleBinding to get nodes - GetNodesClusterRoleName = "kubeadm:get-nodes" - - // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR - // TODO: This value should be defined in an other, generic authz package instead of here - // Starting from v1.8, CSRAutoApprovalClusterRoleName is automatically created by the API server on startup - CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" - // NodeSelfCSRAutoApprovalClusterRoleName is a role defined in default 1.8 RBAC policies for automatic CSR approvals for automatically rotated node certificates - NodeSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient" - // NodeAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve node CSRs - NodeAutoApproveBootstrapClusterRoleBinding = "kubeadm:node-autoapprove-bootstrap" - // NodeAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve node auto rotated CSRs - NodeAutoApproveCertificateRotationClusterRoleBinding = "kubeadm:node-autoapprove-certificate-rotation" -) - -// AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Node Bootstrap Tokens able to post CSRs -func AllowBootstrapTokensToPostCSRs(client clientset.Interface) error { - fmt.Println("[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials") - - return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: NodeKubeletBootstrap, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: NodeBootstrapperClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: constants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} - -// AllowBoostrapTokensToGetNodes creates RBAC rules to allow Node Bootstrap Tokens to list nodes -func AllowBoostrapTokensToGetNodes(client clientset.Interface) error { - fmt.Println("[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes") - - if err := apiclient.CreateOrUpdateClusterRole(client, &rbac.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: GetNodesClusterRoleName, - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"nodes"}, - }, - }, - }); err != nil { - return err - } - - return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: GetNodesClusterRoleName, - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: GetNodesClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: constants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} - -// AutoApproveNodeBootstrapTokens creates RBAC rules in a way that makes Node Bootstrap Tokens' CSR auto-approved by the csrapprover controller -func AutoApproveNodeBootstrapTokens(client clientset.Interface) error { - fmt.Println("[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token") - - // Always create this kubeadm-specific binding though - return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: NodeAutoApproveBootstrapClusterRoleBinding, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: CSRAutoApprovalClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: "Group", - Name: constants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} - -// AutoApproveNodeCertificateRotation creates RBAC rules in a way that makes Node certificate rotation CSR auto-approved by the csrapprover controller -func AutoApproveNodeCertificateRotation(client clientset.Interface) error { - fmt.Println("[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster") - - return apiclient.CreateOrUpdateClusterRoleBinding(client, &rbac.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: NodeAutoApproveCertificateRotationClusterRoleBinding, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "ClusterRole", - Name: NodeSelfCSRAutoApprovalClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: "Group", - Name: constants.NodesGroup, - }, - }, - }) -} diff --git a/cmd/kubeadm/app/phases/bootstraptoken/node/token.go b/cmd/kubeadm/app/phases/bootstraptoken/node/token.go deleted file mode 100644 index 9a2dce93bd29b..0000000000000 --- a/cmd/kubeadm/app/phases/bootstraptoken/node/token.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "context" - "github.com/pkg/errors" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// TODO(mattmoyer): Move CreateNewTokens, UpdateOrCreateTokens out of this package to client-go for a generic abstraction and client for a Bootstrap Token - -// CreateNewTokens tries to create a token and fails if one with the same ID already exists -func CreateNewTokens(client clientset.Interface, tokens []kubeadmapi.BootstrapToken) error { - return UpdateOrCreateTokens(client, true, tokens) -} - -// UpdateOrCreateTokens attempts to update a token with the given ID, or create if it does not already exist. -func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens []kubeadmapi.BootstrapToken) error { - - for _, token := range tokens { - - secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID) - secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{}) - if secret != nil && err == nil && failIfExists { - return errors.Errorf("a token with id %q already exists", token.Token.ID) - } - - updatedOrNewSecret := token.ToSecret() - // Try to create or update the token with an exponential backoff - err = apiclient.TryRunCommand(func() error { - if err := apiclient.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil { - return errors.Wrapf(err, "failed to create or update bootstrap token with name %s", secretName) - } - return nil - }, 5) - if err != nil { - return err - } - } - return nil -} diff --git a/cmd/kubeadm/app/phases/certs/BUILD b/cmd/kubeadm/app/phases/certs/BUILD deleted file mode 100644 index c5830367b4ab2..0000000000000 --- a/cmd/kubeadm/app/phases/certs/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = [ - "certlist_test.go", - "certs_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/certs:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/stretchr/testify/assert:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "certlist.go", - "certs.go", - "doc.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/phases/certs/renewal:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/certs/certlist.go b/cmd/kubeadm/app/phases/certs/certlist.go deleted file mode 100644 index 18b05672cabb5..0000000000000 --- a/cmd/kubeadm/app/phases/certs/certlist.go +++ /dev/null @@ -1,499 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "crypto" - "crypto/x509" - "fmt" - "io" - - "github.com/pkg/errors" - - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -const ( - errInvalid = "invalid argument" - errExist = "file already exists" -) - -type configMutatorsFunc func(*kubeadmapi.InitConfiguration, *pkiutil.CertConfig) error - -// KubeadmCert represents a certificate that Kubeadm will create to function properly. -type KubeadmCert struct { - Name string - LongName string - BaseName string - CAName string - // Some attributes will depend on the InitConfiguration, only known at runtime. - // These functions will be run in series, passed both the InitConfiguration and a cert Config. - configMutators []configMutatorsFunc - config pkiutil.CertConfig -} - -// GetConfig returns the definition for the given cert given the provided InitConfiguration -func (k *KubeadmCert) GetConfig(ic *kubeadmapi.InitConfiguration) (*pkiutil.CertConfig, error) { - for _, f := range k.configMutators { - if err := f(ic, &k.config); err != nil { - return nil, err - } - } - - k.config.PublicKeyAlgorithm = ic.ClusterConfiguration.PublicKeyAlgorithm() - return &k.config, nil -} - -// CreateFromCA makes and writes a certificate using the given CA cert and key. -func (k *KubeadmCert) CreateFromCA(ic *kubeadmapi.InitConfiguration, caCert *x509.Certificate, caKey crypto.Signer) error { - cfg, err := k.GetConfig(ic) - if err != nil { - return errors.Wrapf(err, "couldn't create %q certificate", k.Name) - } - cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg) - if err != nil { - return err - } - err = writeCertificateFilesIfNotExist( - ic.CertificatesDir, - k.BaseName, - caCert, - cert, - key, - cfg, - ) - - if err != nil { - return errors.Wrapf(err, "failed to write or validate certificate %q", k.Name) - } - - return nil -} - -// CreateAsCA creates a certificate authority, writing the files to disk and also returning the created CA so it can be used to sign child certs. -func (k *KubeadmCert) CreateAsCA(ic *kubeadmapi.InitConfiguration) (*x509.Certificate, crypto.Signer, error) { - cfg, err := k.GetConfig(ic) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't get configuration for %q CA certificate", k.Name) - } - caCert, caKey, err := pkiutil.NewCertificateAuthority(cfg) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't generate %q CA certificate", k.Name) - } - - err = writeCertificateAuthorityFilesIfNotExist( - ic.CertificatesDir, - k.BaseName, - caCert, - caKey, - ) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't write out %q CA certificate", k.Name) - } - - return caCert, caKey, nil -} - -// CertificateTree is represents a one-level-deep tree, mapping a CA to the certs that depend on it. -type CertificateTree map[*KubeadmCert]Certificates - -// CreateTree creates the CAs, certs signed by the CAs, and writes them all to disk. -func (t CertificateTree) CreateTree(ic *kubeadmapi.InitConfiguration) error { - for ca, leaves := range t { - cfg, err := ca.GetConfig(ic) - if err != nil { - return err - } - - var caKey crypto.Signer - - caCert, err := pkiutil.TryLoadCertFromDisk(ic.CertificatesDir, ca.BaseName) - if err == nil { - // Validate period - CheckCertificatePeriodValidity(ca.BaseName, caCert) - - // Cert exists already, make sure it's valid - if !caCert.IsCA { - return errors.Errorf("certificate %q is not a CA", ca.Name) - } - // Try and load a CA Key - caKey, err = pkiutil.TryLoadKeyFromDisk(ic.CertificatesDir, ca.BaseName) - if err != nil { - // If there's no CA key, make sure every certificate exists. - for _, leaf := range leaves { - cl := certKeyLocation{ - pkiDir: ic.CertificatesDir, - baseName: leaf.BaseName, - uxName: leaf.Name, - } - if err := validateSignedCertWithCA(cl, caCert); err != nil { - return errors.Wrapf(err, "could not load expected certificate %q or validate the existence of key %q for it", leaf.Name, ca.Name) - } - } - continue - } - // CA key exists; just use that to create new certificates. - } else { - // CACert doesn't already exist, create a new cert and key. - caCert, caKey, err = pkiutil.NewCertificateAuthority(cfg) - if err != nil { - return err - } - - err = writeCertificateAuthorityFilesIfNotExist( - ic.CertificatesDir, - ca.BaseName, - caCert, - caKey, - ) - if err != nil { - return err - } - } - - for _, leaf := range leaves { - if err := leaf.CreateFromCA(ic, caCert, caKey); err != nil { - return err - } - } - } - return nil -} - -// CertificateMap is a flat map of certificates, keyed by Name. -type CertificateMap map[string]*KubeadmCert - -// CertTree returns a one-level-deep tree, mapping a CA cert to an array of certificates that should be signed by it. -func (m CertificateMap) CertTree() (CertificateTree, error) { - caMap := make(CertificateTree) - - for _, cert := range m { - if cert.CAName == "" { - if _, ok := caMap[cert]; !ok { - caMap[cert] = []*KubeadmCert{} - } - } else { - ca, ok := m[cert.CAName] - if !ok { - return nil, errors.Errorf("certificate %q references unknown CA %q", cert.Name, cert.CAName) - } - caMap[ca] = append(caMap[ca], cert) - } - } - - return caMap, nil -} - -// Certificates is a list of Certificates that Kubeadm should create. -type Certificates []*KubeadmCert - -// AsMap returns the list of certificates as a map, keyed by name. -func (c Certificates) AsMap() CertificateMap { - certMap := make(map[string]*KubeadmCert) - for _, cert := range c { - certMap[cert.Name] = cert - } - - return certMap -} - -// GetDefaultCertList returns all of the certificates kubeadm requires to function. -func GetDefaultCertList() Certificates { - return Certificates{ - KubeadmCertRootCA(), - KubeadmCertAPIServer(), - KubeadmCertKubeletClient(), - // Front Proxy certs - KubeadmCertFrontProxyCA(), - KubeadmCertFrontProxyClient(), - // etcd certs - KubeadmCertEtcdCA(), - KubeadmCertEtcdServer(), - KubeadmCertEtcdPeer(), - KubeadmCertEtcdHealthcheck(), - KubeadmCertEtcdAPIClient(), - } -} - -// GetCertsWithoutEtcd returns all of the certificates kubeadm needs when etcd is hosted externally. -func GetCertsWithoutEtcd() Certificates { - return Certificates{ - KubeadmCertRootCA(), - KubeadmCertAPIServer(), - KubeadmCertKubeletClient(), - // Front Proxy certs - KubeadmCertFrontProxyCA(), - KubeadmCertFrontProxyClient(), - } -} - -// KubeadmCertRootCA is the definition of the Kubernetes Root CA for the API Server and kubelet. -func KubeadmCertRootCA() *KubeadmCert { - return &KubeadmCert{ - Name: "ca", - LongName: "self-signed Kubernetes CA to provision identities for other Kubernetes components", - BaseName: kubeadmconstants.CACertAndKeyBaseName, - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "kubernetes", - }, - }, - } -} - -// KubeadmCertAPIServer is the definition of the cert used to serve the Kubernetes API. -func KubeadmCertAPIServer() *KubeadmCert { - return &KubeadmCert{ - Name: "apiserver", - LongName: "certificate for serving the Kubernetes API", - BaseName: kubeadmconstants.APIServerCertAndKeyBaseName, - CAName: "ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: kubeadmconstants.APIServerCertCommonName, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - }, - configMutators: []configMutatorsFunc{ - makeAltNamesMutator(pkiutil.GetAPIServerAltNames), - }, - } -} - -// KubeadmCertKubeletClient is the definition of the cert used by the API server to access the kubelet. -func KubeadmCertKubeletClient() *KubeadmCert { - return &KubeadmCert{ - Name: "apiserver-kubelet-client", - LongName: "certificate for the API server to connect to kubelet", - BaseName: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, - CAName: "ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: kubeadmconstants.APIServerKubeletClientCertCommonName, - Organization: []string{kubeadmconstants.SystemPrivilegedGroup}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - }, - } -} - -// KubeadmCertFrontProxyCA is the definition of the CA used for the front end proxy. -func KubeadmCertFrontProxyCA() *KubeadmCert { - return &KubeadmCert{ - Name: "front-proxy-ca", - LongName: "self-signed CA to provision identities for front proxy", - BaseName: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "front-proxy-ca", - }, - }, - } -} - -// KubeadmCertFrontProxyClient is the definition of the cert used by the API server to access the front proxy. -func KubeadmCertFrontProxyClient() *KubeadmCert { - return &KubeadmCert{ - Name: "front-proxy-client", - BaseName: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - LongName: "certificate for the front proxy client", - CAName: "front-proxy-ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: kubeadmconstants.FrontProxyClientCertCommonName, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - }, - } -} - -// KubeadmCertEtcdCA is the definition of the root CA used by the hosted etcd server. -func KubeadmCertEtcdCA() *KubeadmCert { - return &KubeadmCert{ - Name: "etcd-ca", - LongName: "self-signed CA to provision identities for etcd", - BaseName: kubeadmconstants.EtcdCACertAndKeyBaseName, - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "etcd-ca", - }, - }, - } -} - -// KubeadmCertEtcdServer is the definition of the cert used to serve etcd to clients. -func KubeadmCertEtcdServer() *KubeadmCert { - return &KubeadmCert{ - Name: "etcd-server", - LongName: "certificate for serving etcd", - BaseName: kubeadmconstants.EtcdServerCertAndKeyBaseName, - CAName: "etcd-ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - // TODO: etcd 3.2 introduced an undocumented requirement for ClientAuth usage on the - // server cert: https://github.com/coreos/etcd/issues/9785#issuecomment-396715692 - // Once the upstream issue is resolved, this should be returned to only allowing - // ServerAuth usage. - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - }, - }, - configMutators: []configMutatorsFunc{ - makeAltNamesMutator(pkiutil.GetEtcdAltNames), - setCommonNameToNodeName(), - }, - } -} - -// KubeadmCertEtcdPeer is the definition of the cert used by etcd peers to access each other. -func KubeadmCertEtcdPeer() *KubeadmCert { - return &KubeadmCert{ - Name: "etcd-peer", - LongName: "certificate for etcd nodes to communicate with each other", - BaseName: kubeadmconstants.EtcdPeerCertAndKeyBaseName, - CAName: "etcd-ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - }, - }, - configMutators: []configMutatorsFunc{ - makeAltNamesMutator(pkiutil.GetEtcdPeerAltNames), - setCommonNameToNodeName(), - }, - } -} - -// KubeadmCertEtcdHealthcheck is the definition of the cert used by Kubernetes to check the health of the etcd server. -func KubeadmCertEtcdHealthcheck() *KubeadmCert { - return &KubeadmCert{ - Name: "etcd-healthcheck-client", - LongName: "certificate for liveness probes to healthcheck etcd", - BaseName: kubeadmconstants.EtcdHealthcheckClientCertAndKeyBaseName, - CAName: "etcd-ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: kubeadmconstants.EtcdHealthcheckClientCertCommonName, - Organization: []string{kubeadmconstants.SystemPrivilegedGroup}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - }, - } -} - -// KubeadmCertEtcdAPIClient is the definition of the cert used by the API server to access etcd. -func KubeadmCertEtcdAPIClient() *KubeadmCert { - return &KubeadmCert{ - Name: "apiserver-etcd-client", - LongName: "certificate the apiserver uses to access etcd", - BaseName: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, - CAName: "etcd-ca", - config: pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: kubeadmconstants.APIServerEtcdClientCertCommonName, - Organization: []string{kubeadmconstants.SystemPrivilegedGroup}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - }, - } -} - -func makeAltNamesMutator(f func(*kubeadmapi.InitConfiguration) (*certutil.AltNames, error)) configMutatorsFunc { - return func(mc *kubeadmapi.InitConfiguration, cc *pkiutil.CertConfig) error { - altNames, err := f(mc) - if err != nil { - return err - } - cc.AltNames = *altNames - return nil - } -} - -func setCommonNameToNodeName() configMutatorsFunc { - return func(mc *kubeadmapi.InitConfiguration, cc *pkiutil.CertConfig) error { - cc.CommonName = mc.NodeRegistration.Name - return nil - } -} - -// leafCertificates returns non-CA certificates from the supplied Certificates. -func leafCertificates(c Certificates) (Certificates, error) { - certTree, err := c.AsMap().CertTree() - if err != nil { - return nil, err - } - - var out Certificates - for _, leafCertificates := range certTree { - out = append(out, leafCertificates...) - } - return out, nil -} - -func createKeyAndCSR(kubeadmConfig *kubeadmapi.InitConfiguration, cert *KubeadmCert) error { - if kubeadmConfig == nil { - return errors.Errorf("%s: kubeadmConfig was nil", errInvalid) - } - if cert == nil { - return errors.Errorf("%s: cert was nil", errInvalid) - } - certDir := kubeadmConfig.CertificatesDir - name := cert.BaseName - if pkiutil.CSROrKeyExist(certDir, name) { - return errors.Errorf("%s: key or CSR %s/%s", errExist, certDir, name) - } - cfg, err := cert.GetConfig(kubeadmConfig) - if err != nil { - return err - } - csr, key, err := pkiutil.NewCSRAndKey(cfg) - if err != nil { - return err - } - err = pkiutil.WriteKey(certDir, name, key) - if err != nil { - return err - } - err = pkiutil.WriteCSR(certDir, name, csr) - if err != nil { - return err - } - return nil -} - -// CreateDefaultKeysAndCSRFiles is used in ExternalCA mode to create key files -// and adjacent CSR files. -func CreateDefaultKeysAndCSRFiles(out io.Writer, config *kubeadmapi.InitConfiguration) error { - certificates, err := leafCertificates(GetDefaultCertList()) - if err != nil { - return err - } - if out != nil { - fmt.Fprintf(out, "generating keys and CSRs in %s\n", config.CertificatesDir) - } - for _, cert := range certificates { - if err := createKeyAndCSR(config, cert); err != nil { - return err - } - if out != nil { - fmt.Fprintf(out, " %s\n", cert.BaseName) - } - } - return nil -} diff --git a/cmd/kubeadm/app/phases/certs/certlist_test.go b/cmd/kubeadm/app/phases/certs/certlist_test.go deleted file mode 100644 index c1d99bc85a4b2..0000000000000 --- a/cmd/kubeadm/app/phases/certs/certlist_test.go +++ /dev/null @@ -1,224 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "crypto" - "crypto/tls" - "crypto/x509" - "io/ioutil" - "os" - "path" - "testing" - - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -func TestCertListOrder(t *testing.T) { - tests := []struct { - certs Certificates - name string - }{ - { - name: "Default Certificate List", - certs: GetDefaultCertList(), - }, - { - name: "Cert list less etcd", - certs: GetCertsWithoutEtcd(), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var lastCA *KubeadmCert - for i, cert := range test.certs { - if i > 0 && lastCA == nil { - t.Fatalf("CA not present in list before certificate %q", cert.Name) - } - if cert.CAName == "" { - lastCA = cert - } else { - if cert.CAName != lastCA.Name { - t.Fatalf("expected CA name %q, got %q, for certificate %q", lastCA.Name, cert.CAName, cert.Name) - } - } - } - }) - } -} - -func TestCAPointersValid(t *testing.T) { - tests := []struct { - certs Certificates - name string - }{ - { - name: "Default Certificate List", - certs: GetDefaultCertList(), - }, - { - name: "Cert list less etcd", - certs: GetCertsWithoutEtcd(), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - - certMap := test.certs.AsMap() - - for _, cert := range test.certs { - if cert.CAName != "" && certMap[cert.CAName] == nil { - t.Errorf("Certificate %q references nonexistent CA %q", cert.Name, cert.CAName) - } - } - }) - } -} - -func TestMakeCertTree(t *testing.T) { - rootCert := &KubeadmCert{ - Name: "root", - } - leaf0 := &KubeadmCert{ - Name: "leaf0", - CAName: "root", - } - leaf1 := &KubeadmCert{ - Name: "leaf1", - CAName: "root", - } - selfSigned := &KubeadmCert{ - Name: "self-signed", - } - - certMap := CertificateMap{ - "root": rootCert, - "leaf0": leaf0, - "leaf1": leaf1, - "self-signed": selfSigned, - } - - orphanCertMap := CertificateMap{ - "leaf0": leaf0, - } - - if _, err := orphanCertMap.CertTree(); err == nil { - t.Error("expected orphan cert map to error, but got nil") - } - - certTree, err := certMap.CertTree() - t.Logf("cert tree: %v", certTree) - if err != nil { - t.Errorf("expected no error, but got %v", err) - } - - if len(certTree) != 2 { - t.Errorf("Expected tree to have 2 roots, got %d", len(certTree)) - } - - if len(certTree[rootCert]) != 2 { - t.Errorf("Expected root to have 2 leaves, got %d", len(certTree[rootCert])) - } - - if _, ok := certTree[selfSigned]; !ok { - t.Error("Expected selfSigned to be present in tree, but missing") - } -} - -func TestCreateCertificateChain(t *testing.T) { - dir, err := ioutil.TempDir("", t.Name()) - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - ic := &kubeadmapi.InitConfiguration{ - NodeRegistration: kubeadmapi.NodeRegistrationOptions{ - Name: "test-node", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: dir, - }, - } - - caCfg := Certificates{ - { - config: pkiutil.CertConfig{}, - Name: "test-ca", - BaseName: "test-ca", - }, - { - config: pkiutil.CertConfig{ - Config: certutil.Config{ - AltNames: certutil.AltNames{ - DNSNames: []string{"test-domain.space"}, - }, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - }, - configMutators: []configMutatorsFunc{ - setCommonNameToNodeName(), - }, - CAName: "test-ca", - Name: "test-daughter", - BaseName: "test-daughter", - }, - } - - certTree, err := caCfg.AsMap().CertTree() - if err != nil { - t.Fatalf("unexpected error getting tree: %v", err) - } - - if certTree.CreateTree(ic); err != nil { - t.Fatal(err) - } - - caCert, _ := parseCertAndKey(path.Join(dir, "test-ca"), t) - daughterCert, _ := parseCertAndKey(path.Join(dir, "test-daughter"), t) - - pool := x509.NewCertPool() - pool.AddCert(caCert) - - _, err = daughterCert.Verify(x509.VerifyOptions{ - DNSName: "test-domain.space", - Roots: pool, - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }) - if err != nil { - t.Errorf("couldn't verify daughter cert: %v", err) - } - -} - -func parseCertAndKey(basePath string, t *testing.T) (*x509.Certificate, crypto.PrivateKey) { - certPair, err := tls.LoadX509KeyPair(basePath+".crt", basePath+".key") - if err != nil { - t.Fatalf("couldn't parse certificate and key: %v", err) - } - - parsedCert, err := x509.ParseCertificate(certPair.Certificate[0]) - if err != nil { - t.Fatalf("couldn't parse certificate: %v", err) - } - - return parsedCert, certPair.PrivateKey -} diff --git a/cmd/kubeadm/app/phases/certs/certs.go b/cmd/kubeadm/app/phases/certs/certs.go deleted file mode 100644 index 4e15dc9c95760..0000000000000 --- a/cmd/kubeadm/app/phases/certs/certs.go +++ /dev/null @@ -1,478 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "crypto" - "crypto/x509" - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/pkg/errors" - "k8s.io/client-go/util/keyutil" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -var ( - // certPeriodValidation is used to store if period validation was done for a certificate - certPeriodValidationMutex sync.Mutex - certPeriodValidation = map[string]struct{}{} -) - -// CreatePKIAssets will create and write to disk all PKI assets necessary to establish the control plane. -// If the PKI assets already exists in the target folder, they are used only if evaluated equal; otherwise an error is returned. -func CreatePKIAssets(cfg *kubeadmapi.InitConfiguration) error { - klog.V(1).Infoln("creating PKI assets") - - // This structure cannot handle multilevel CA hierarchies. - // This isn't a problem right now, but may become one in the future. - - var certList Certificates - - if cfg.Etcd.Local == nil { - certList = GetCertsWithoutEtcd() - } else { - certList = GetDefaultCertList() - } - - certTree, err := certList.AsMap().CertTree() - if err != nil { - return err - } - - if err := certTree.CreateTree(cfg); err != nil { - return errors.Wrap(err, "error creating PKI assets") - } - - fmt.Printf("[certs] Valid certificates and keys now exist in %q\n", cfg.CertificatesDir) - - // Service accounts are not x509 certs, so handled separately - return CreateServiceAccountKeyAndPublicKeyFiles(cfg.CertificatesDir, cfg.ClusterConfiguration.PublicKeyAlgorithm()) -} - -// CreateServiceAccountKeyAndPublicKeyFiles creates new public/private key files for signing service account users. -// If the sa public/private key files already exist in the target folder, they are used only if evaluated equals; otherwise an error is returned. -func CreateServiceAccountKeyAndPublicKeyFiles(certsDir string, keyType x509.PublicKeyAlgorithm) error { - klog.V(1).Infoln("creating new public/private key files for signing service account users") - _, err := keyutil.PrivateKeyFromFile(filepath.Join(certsDir, kubeadmconstants.ServiceAccountPrivateKeyName)) - if err == nil { - // kubeadm doesn't validate the existing certificate key more than this; - // Basically, if we find a key file with the same path kubeadm thinks those files - // are equal and doesn't bother writing a new file - fmt.Printf("[certs] Using the existing %q key\n", kubeadmconstants.ServiceAccountKeyBaseName) - return nil - } else if !os.IsNotExist(err) { - return errors.Wrapf(err, "file %s existed but it could not be loaded properly", kubeadmconstants.ServiceAccountPrivateKeyName) - } - - // The key does NOT exist, let's generate it now - key, err := pkiutil.NewPrivateKey(keyType) - if err != nil { - return err - } - - // Write .key and .pub files to disk - fmt.Printf("[certs] Generating %q key and public key\n", kubeadmconstants.ServiceAccountKeyBaseName) - - if err := pkiutil.WriteKey(certsDir, kubeadmconstants.ServiceAccountKeyBaseName, key); err != nil { - return err - } - - return pkiutil.WritePublicKey(certsDir, kubeadmconstants.ServiceAccountKeyBaseName, key.Public()) -} - -// CreateCACertAndKeyFiles generates and writes out a given certificate authority. -// The certSpec should be one of the variables from this package. -func CreateCACertAndKeyFiles(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) error { - if certSpec.CAName != "" { - return errors.Errorf("this function should only be used for CAs, but cert %s has CA %s", certSpec.Name, certSpec.CAName) - } - klog.V(1).Infof("creating a new certificate authority for %s", certSpec.Name) - - certConfig, err := certSpec.GetConfig(cfg) - if err != nil { - return err - } - - caCert, caKey, err := pkiutil.NewCertificateAuthority(certConfig) - if err != nil { - return err - } - - return writeCertificateAuthorityFilesIfNotExist( - cfg.CertificatesDir, - certSpec.BaseName, - caCert, - caKey, - ) -} - -// NewCSR will generate a new CSR and accompanying key -func NewCSR(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) (*x509.CertificateRequest, crypto.Signer, error) { - certConfig, err := certSpec.GetConfig(cfg) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to retrieve cert configuration") - } - - return pkiutil.NewCSRAndKey(certConfig) -} - -// CreateCSR creates a certificate signing request -func CreateCSR(certSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration, path string) error { - csr, key, err := NewCSR(certSpec, cfg) - if err != nil { - return err - } - return writeCSRFilesIfNotExist(path, certSpec.BaseName, csr, key) -} - -// CreateCertAndKeyFilesWithCA loads the given certificate authority from disk, then generates and writes out the given certificate and key. -// The certSpec and caCertSpec should both be one of the variables from this package. -func CreateCertAndKeyFilesWithCA(certSpec *KubeadmCert, caCertSpec *KubeadmCert, cfg *kubeadmapi.InitConfiguration) error { - if certSpec.CAName != caCertSpec.Name { - return errors.Errorf("expected CAname for %s to be %q, but was %s", certSpec.Name, certSpec.CAName, caCertSpec.Name) - } - - caCert, caKey, err := LoadCertificateAuthority(cfg.CertificatesDir, caCertSpec.BaseName) - if err != nil { - return errors.Wrapf(err, "couldn't load CA certificate %s", caCertSpec.Name) - } - - return certSpec.CreateFromCA(cfg, caCert, caKey) -} - -// LoadCertificateAuthority tries to load a CA in the given directory with the given name. -func LoadCertificateAuthority(pkiDir string, baseName string) (*x509.Certificate, crypto.Signer, error) { - // Checks if certificate authority exists in the PKI directory - if !pkiutil.CertOrKeyExist(pkiDir, baseName) { - return nil, nil, errors.Errorf("couldn't load %s certificate authority from %s", baseName, pkiDir) - } - - // Try to load certificate authority .crt and .key from the PKI directory - caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return nil, nil, errors.Wrapf(err, "failure loading %s certificate authority", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, caCert) - - // Make sure the loaded CA cert actually is a CA - if !caCert.IsCA { - return nil, nil, errors.Errorf("%s certificate is not a certificate authority", baseName) - } - - return caCert, caKey, nil -} - -// writeCertificateAuthorityFilesIfNotExist write a new certificate Authority to the given path. -// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// otherwise this function returns an error. -func writeCertificateAuthorityFilesIfNotExist(pkiDir string, baseName string, caCert *x509.Certificate, caKey crypto.Signer) error { - - // If cert or key exists, we should try to load them - if pkiutil.CertOrKeyExist(pkiDir, baseName) { - - // Try to load .crt and .key from the PKI directory - caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s certificate", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, caCert) - - // Check if the existing cert is a CA - if !caCert.IsCA { - return errors.Errorf("certificate %s is not a CA", baseName) - } - - // kubeadm doesn't validate the existing certificate Authority more than this; - // Basically, if we find a certificate file with the same path; and it is a CA - // kubeadm thinks those files are equal and doesn't bother writing a new file - fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName) - } else { - // Write .crt and .key files to disk - fmt.Printf("[certs] Generating %q certificate and key\n", baseName) - - if err := pkiutil.WriteCertAndKey(pkiDir, baseName, caCert, caKey); err != nil { - return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) - } - } - return nil -} - -// writeCertificateFilesIfNotExist write a new certificate to the given path. -// If there already is a certificate file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected certificate equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// otherwise this function returns an error. -func writeCertificateFilesIfNotExist(pkiDir string, baseName string, signingCert *x509.Certificate, cert *x509.Certificate, key crypto.Signer, cfg *pkiutil.CertConfig) error { - - // Checks if the signed certificate exists in the PKI directory - if pkiutil.CertOrKeyExist(pkiDir, baseName) { - // Try to load signed certificate .crt and .key from the PKI directory - signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkiDir, baseName) - if err != nil { - return errors.Wrapf(err, "failure loading %s certificate", baseName) - } - // Validate period - CheckCertificatePeriodValidity(baseName, signedCert) - - // Check if the existing cert is signed by the given CA - if err := signedCert.CheckSignatureFrom(signingCert); err != nil { - return errors.Errorf("certificate %s is not signed by corresponding CA", baseName) - } - - // Check if the certificate has the correct attributes - if err := validateCertificateWithConfig(signedCert, baseName, cfg); err != nil { - return err - } - - fmt.Printf("[certs] Using the existing %q certificate and key\n", baseName) - } else { - // Write .crt and .key files to disk - fmt.Printf("[certs] Generating %q certificate and key\n", baseName) - - if err := pkiutil.WriteCertAndKey(pkiDir, baseName, cert, key); err != nil { - return errors.Wrapf(err, "failure while saving %s certificate and key", baseName) - } - if pkiutil.HasServerAuth(cert) { - fmt.Printf("[certs] %s serving cert is signed for DNS names %v and IPs %v\n", baseName, cert.DNSNames, cert.IPAddresses) - } - } - - return nil -} - -// writeCSRFilesIfNotExist writes a new CSR to the given path. -// If there already is a CSR file at the given path; kubeadm tries to load it and check if it's a valid certificate. -// otherwise this function returns an error. -func writeCSRFilesIfNotExist(csrDir string, baseName string, csr *x509.CertificateRequest, key crypto.Signer) error { - if pkiutil.CSROrKeyExist(csrDir, baseName) { - _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(csrDir, baseName) - if err != nil { - return errors.Wrapf(err, "%s CSR existed but it could not be loaded properly", baseName) - } - - fmt.Printf("[certs] Using the existing %q CSR\n", baseName) - } else { - // Write .key and .csr files to disk - fmt.Printf("[certs] Generating %q key and CSR\n", baseName) - - if err := pkiutil.WriteKey(csrDir, baseName, key); err != nil { - return errors.Wrapf(err, "failure while saving %s key", baseName) - } - - if err := pkiutil.WriteCSR(csrDir, baseName, csr); err != nil { - return errors.Wrapf(err, "failure while saving %s CSR", baseName) - } - } - - return nil -} - -type certKeyLocation struct { - pkiDir string - caBaseName string - baseName string - uxName string -} - -// SharedCertificateExists verifies if the shared certificates - the certificates that must be -// equal across control-plane nodes: ca.key, ca.crt, sa.key, sa.pub + etcd/ca.key, etcd/ca.crt if local/stacked etcd -// Missing keys are non-fatal and produce warnings. -func SharedCertificateExists(cfg *kubeadmapi.ClusterConfiguration) (bool, error) { - - if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil { - return false, err - } - - if err := validatePrivatePublicKey(certKeyLocation{cfg.CertificatesDir, "", kubeadmconstants.ServiceAccountKeyBaseName, "service account"}); err != nil { - return false, err - } - - if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil { - return false, err - } - - // in case of local/stacked etcd - if cfg.Etcd.External == nil { - if err := validateCACertAndKey(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.EtcdCACertAndKeyBaseName, "", "etcd CA"}); err != nil { - return false, err - } - } - - return true, nil -} - -// UsingExternalCA determines whether the user is relying on an external CA. We currently implicitly determine this is the case -// when the CA Cert is present but the CA Key is not. -// This allows us to, e.g., skip generating certs or not start the csr signing controller. -// In case we are using an external front-proxy CA, the function validates the certificates signed by front-proxy CA that should be provided by the user. -func UsingExternalCA(cfg *kubeadmapi.ClusterConfiguration) (bool, error) { - - if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, "", "CA"}); err != nil { - return false, err - } - - caKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName) - if _, err := os.Stat(caKeyPath); !os.IsNotExist(err) { - return false, nil - } - - if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerCertAndKeyBaseName, "API server"}); err != nil { - return true, err - } - - if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName, kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, "API server kubelet client"}); err != nil { - return true, err - } - - return true, nil -} - -// UsingExternalFrontProxyCA determines whether the user is relying on an external front-proxy CA. We currently implicitly determine this is the case -// when the front proxy CA Cert is present but the front proxy CA Key is not. -// In case we are using an external front-proxy CA, the function validates the certificates signed by front-proxy CA that should be provided by the user. -func UsingExternalFrontProxyCA(cfg *kubeadmapi.ClusterConfiguration) (bool, error) { - - if err := validateCACert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, "", "front-proxy CA"}); err != nil { - return false, err - } - - frontProxyCAKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName) - if _, err := os.Stat(frontProxyCAKeyPath); !os.IsNotExist(err) { - return false, nil - } - - if err := validateSignedCert(certKeyLocation{cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertAndKeyBaseName, kubeadmconstants.FrontProxyClientCertAndKeyBaseName, "front-proxy client"}); err != nil { - return true, err - } - - return true, nil -} - -// validateCACert tries to load a x509 certificate from pkiDir and validates that it is a CA -func validateCACert(l certKeyLocation) error { - // Check CA Cert - caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName) - if err != nil { - return errors.Wrapf(err, "failure loading certificate for %s", l.uxName) - } - // Validate period - CheckCertificatePeriodValidity(l.uxName, caCert) - - // Check if cert is a CA - if !caCert.IsCA { - return errors.Errorf("certificate %s is not a CA", l.uxName) - } - return nil -} - -// validateCACertAndKey tries to load a x509 certificate and private key from pkiDir, -// and validates that the cert is a CA. Failure to load the key produces a warning. -func validateCACertAndKey(l certKeyLocation) error { - if err := validateCACert(l); err != nil { - return err - } - - _, err := pkiutil.TryLoadKeyFromDisk(l.pkiDir, l.caBaseName) - if err != nil { - klog.Warningf("assuming external key for %s: %v", l.uxName, err) - } - return nil -} - -// validateSignedCert tries to load a x509 certificate and private key from pkiDir and validates -// that the cert is signed by a given CA -func validateSignedCert(l certKeyLocation) error { - // Try to load CA - caCert, err := pkiutil.TryLoadCertFromDisk(l.pkiDir, l.caBaseName) - if err != nil { - return errors.Wrapf(err, "failure loading certificate authority for %s", l.uxName) - } - // Validate period - CheckCertificatePeriodValidity(l.uxName, caCert) - - return validateSignedCertWithCA(l, caCert) -} - -// validateSignedCertWithCA tries to load a certificate and validate it with the given caCert -func validateSignedCertWithCA(l certKeyLocation, caCert *x509.Certificate) error { - // Try to load key and signed certificate - signedCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(l.pkiDir, l.baseName) - if err != nil { - return errors.Wrapf(err, "failure loading certificate for %s", l.uxName) - } - // Validate period - CheckCertificatePeriodValidity(l.uxName, signedCert) - - // Check if the cert is signed by the CA - if err := signedCert.CheckSignatureFrom(caCert); err != nil { - return errors.Wrapf(err, "certificate %s is not signed by corresponding CA", l.uxName) - } - return nil -} - -// validatePrivatePublicKey tries to load a private key from pkiDir -func validatePrivatePublicKey(l certKeyLocation) error { - // Try to load key - _, _, err := pkiutil.TryLoadPrivatePublicKeyFromDisk(l.pkiDir, l.baseName) - if err != nil { - return errors.Wrapf(err, "failure loading key for %s", l.uxName) - } - return nil -} - -// validateCertificateWithConfig makes sure that a given certificate is valid at -// least for the SANs defined in the configuration. -func validateCertificateWithConfig(cert *x509.Certificate, baseName string, cfg *pkiutil.CertConfig) error { - for _, dnsName := range cfg.AltNames.DNSNames { - if err := cert.VerifyHostname(dnsName); err != nil { - return errors.Wrapf(err, "certificate %s is invalid", baseName) - } - } - for _, ipAddress := range cfg.AltNames.IPs { - if err := cert.VerifyHostname(ipAddress.String()); err != nil { - return errors.Wrapf(err, "certificate %s is invalid", baseName) - } - } - return nil -} - -// CheckCertificatePeriodValidity takes a certificate and prints a warning if its period -// is not valid related to the current time. It does so only if the certificate was not validated already -// by keeping track with a cache. -func CheckCertificatePeriodValidity(baseName string, cert *x509.Certificate) { - certPeriodValidationMutex.Lock() - if _, exists := certPeriodValidation[baseName]; exists { - certPeriodValidationMutex.Unlock() - return - } - certPeriodValidation[baseName] = struct{}{} - certPeriodValidationMutex.Unlock() - - klog.V(5).Infof("validating certificate period for %s certificate", baseName) - if err := pkiutil.ValidateCertPeriod(cert, 0); err != nil { - klog.Warningf("WARNING: could not validate bounds for certificate %s: %v", baseName, err) - } -} diff --git a/cmd/kubeadm/app/phases/certs/certs_test.go b/cmd/kubeadm/app/phases/certs/certs_test.go deleted file mode 100644 index e0e9436a4e4c8..0000000000000 --- a/cmd/kubeadm/app/phases/certs/certs_test.go +++ /dev/null @@ -1,840 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "bytes" - "crypto" - "crypto/sha256" - "crypto/x509" - "io/ioutil" - "net" - "os" - "path" - "path/filepath" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func createTestCSR(t *testing.T) (*x509.CertificateRequest, crypto.Signer) { - csr, key, err := pkiutil.NewCSRAndKey( - &pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "testCert", - }, - }) - if err != nil { - t.Fatalf("couldn't create test cert: %v", err) - } - - return csr, key -} - -func TestWriteCertificateAuthorityFilesIfNotExist(t *testing.T) { - setupCert, setupKey := certstestutil.CreateCACert(t) - caCert, caKey := certstestutil.CreateCACert(t) - - var tests = []struct { - setupFunc func(pkiDir string) error - expectedError bool - expectedCa *x509.Certificate - }{ - { // ca cert does not exists > ca written - expectedCa: caCert, - }, - { // ca cert exists, is ca > existing ca used - setupFunc: func(pkiDir string) error { - return writeCertificateAuthorityFilesIfNotExist(pkiDir, "dummy", setupCert, setupKey) - }, - expectedCa: setupCert, - }, - { // some file exists, but it is not a valid ca cert > err - setupFunc: func(pkiDir string) error { - testutil.SetupEmptyFiles(t, pkiDir, "dummy.crt") - return nil - }, - expectedError: true, - }, - { // cert exists, but it is not a ca > err - setupFunc: func(pkiDir string) error { - cert, key, config := certstestutil.CreateTestCert(t, setupCert, setupKey, certutil.AltNames{}) - return writeCertificateFilesIfNotExist(pkiDir, "dummy", setupCert, cert, key, config) - }, - expectedError: true, - }, - } - - for _, test := range tests { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // executes setup func (if necessary) - if test.setupFunc != nil { - if err := test.setupFunc(tmpdir); err != nil { - t.Errorf("error executing setupFunc: %v", err) - continue - } - } - - // executes create func - err := writeCertificateAuthorityFilesIfNotExist(tmpdir, "dummy", caCert, caKey) - - if !test.expectedError && err != nil { - t.Errorf("error writeCertificateAuthorityFilesIfNotExist failed when not expected to fail: %v", err) - continue - } else if test.expectedError && err == nil { - t.Error("error writeCertificateAuthorityFilesIfNotExist didn't failed when expected") - continue - } else if test.expectedError { - continue - } - - // asserts expected files are there - testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.crt") - - // check created cert - resultingCaCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpdir, "dummy") - if err != nil { - t.Errorf("failure reading created cert: %v", err) - continue - } - if !resultingCaCert.Equal(test.expectedCa) { - t.Error("created ca cert does not match expected ca cert") - } - } -} - -func TestWriteCertificateFilesIfNotExist(t *testing.T) { - altNames := certutil.AltNames{ - DNSNames: []string{"example.com"}, - IPs: []net.IP{ - net.IPv4(0, 0, 0, 0), - }, - } - - caCert, caKey := certstestutil.CreateCACert(t) - setupCert, setupKey, _ := certstestutil.CreateTestCert(t, caCert, caKey, altNames) - cert, key, config := certstestutil.CreateTestCert(t, caCert, caKey, altNames) - - var tests = []struct { - setupFunc func(pkiDir string) error - expectedError bool - expectedCert *x509.Certificate - }{ - { // cert does not exists > cert written - expectedCert: cert, - }, - { // cert exists, is signed by the same ca, missing SANs (dns name) > err - setupFunc: func(pkiDir string) error { - setupCert, setupKey, setupConfig := certstestutil.CreateTestCert(t, caCert, caKey, certutil.AltNames{ - IPs: []net.IP{ - net.IPv4(0, 0, 0, 0), - }, - }) - return writeCertificateFilesIfNotExist(pkiDir, "dummy", caCert, setupCert, setupKey, setupConfig) - }, - expectedError: true, - }, - { // cert exists, is signed by the same ca, missing SANs (IP address) > err - setupFunc: func(pkiDir string) error { - setupCert, setupKey, setupConfig := certstestutil.CreateTestCert(t, caCert, caKey, certutil.AltNames{ - DNSNames: []string{"example.com"}, - }) - return writeCertificateFilesIfNotExist(pkiDir, "dummy", caCert, setupCert, setupKey, setupConfig) - }, - expectedError: true, - }, - { // cert exists, is signed by the same ca, all SANs present > existing cert used - setupFunc: func(pkiDir string) error { - return writeCertificateFilesIfNotExist(pkiDir, "dummy", caCert, setupCert, setupKey, config) - }, - expectedCert: setupCert, - }, - { // some file exists, but it is not a valid cert > err - setupFunc: func(pkiDir string) error { - testutil.SetupEmptyFiles(t, pkiDir, "dummy.crt") - return nil - }, - expectedError: true, - }, - { // cert exists, is signed by another ca > err - setupFunc: func(pkiDir string) error { - anotherCaCert, anotherCaKey := certstestutil.CreateCACert(t) - anotherCert, anotherKey, config := certstestutil.CreateTestCert(t, anotherCaCert, anotherCaKey, certutil.AltNames{}) - - return writeCertificateFilesIfNotExist(pkiDir, "dummy", anotherCaCert, anotherCert, anotherKey, config) - }, - expectedError: true, - }, - } - - for _, test := range tests { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // executes setup func (if necessary) - if test.setupFunc != nil { - if err := test.setupFunc(tmpdir); err != nil { - t.Errorf("error executing setupFunc: %v", err) - continue - } - } - - // executes create func - err := writeCertificateFilesIfNotExist(tmpdir, "dummy", caCert, cert, key, config) - - if !test.expectedError && err != nil { - t.Errorf("error writeCertificateFilesIfNotExist failed when not expected to fail: %v", err) - continue - } else if test.expectedError && err == nil { - t.Error("error writeCertificateFilesIfNotExist didn't fail when expected") - continue - } else if test.expectedError { - continue - } - - // asserts expected files are there - testutil.AssertFileExists(t, tmpdir, "dummy.key", "dummy.crt") - - // check created cert - resultingCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(tmpdir, "dummy") - if err != nil { - t.Errorf("failure reading created cert: %v", err) - continue - } - if !resultingCert.Equal(test.expectedCert) { - t.Error("created cert does not match expected cert") - } - } -} - -func TestWriteCSRFilesIfNotExist(t *testing.T) { - csr, key := createTestCSR(t) - csr2, key2 := createTestCSR(t) - - var tests = []struct { - name string - setupFunc func(csrPath string) error - expectedError bool - expectedCSR *x509.CertificateRequest - }{ - { - name: "no files exist", - expectedCSR: csr, - }, - { - name: "other key exists", - setupFunc: func(csrPath string) error { - if err := pkiutil.WriteCSR(csrPath, "dummy", csr2); err != nil { - return err - } - return pkiutil.WriteKey(csrPath, "dummy", key2) - }, - expectedCSR: csr2, - }, - { - name: "existing CSR is garbage", - setupFunc: func(csrPath string) error { - return ioutil.WriteFile(path.Join(csrPath, "dummy.csr"), []byte("a--bunch--of-garbage"), os.ModePerm) - }, - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - if test.setupFunc != nil { - if err := test.setupFunc(tmpdir); err != nil { - t.Fatalf("couldn't set up test: %v", err) - } - } - - if err := writeCSRFilesIfNotExist(tmpdir, "dummy", csr, key); err != nil { - if test.expectedError { - return - } - t.Fatalf("unexpected error %v: ", err) - } - - if test.expectedError { - t.Fatal("Expected error, but got none") - } - - parsedCSR, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(tmpdir, "dummy") - if err != nil { - t.Fatalf("couldn't load csr and key: %v", err) - } - - if sha256.Sum256(test.expectedCSR.Raw) != sha256.Sum256(parsedCSR.Raw) { - t.Error("expected csr's fingerprint does not match ") - } - - }) - } - -} - -func TestCreateServiceAccountKeyAndPublicKeyFiles(t *testing.T) { - setupKey, err := keyutil.MakeEllipticPrivateKeyPEM() - if err != nil { - t.Fatalf("Can't setup test: %v", err) - } - - tcases := []struct { - name string - setupFunc func(pkiDir string) error - expectedErr bool - expectedKey []byte - }{ - { // key does not exists > key written - name: "generate successfully", - }, - { // key exists > existing key used - name: "use existing key", - setupFunc: func(pkiDir string) error { - err := keyutil.WriteKey(filepath.Join(pkiDir, kubeadmconstants.ServiceAccountPrivateKeyName), setupKey) - return err - }, - expectedKey: setupKey, - }, - { // some file exists, but it is not a valid key > err - name: "empty key", - setupFunc: func(pkiDir string) error { - testutil.SetupEmptyFiles(t, pkiDir, kubeadmconstants.ServiceAccountPrivateKeyName) - return nil - }, - expectedErr: true, - }, - } - for _, tt := range tcases { - t.Run(tt.name, func(t *testing.T) { - dir := testutil.SetupTempDir(t) - defer os.RemoveAll(dir) - - if tt.setupFunc != nil { - if err := tt.setupFunc(dir); err != nil { - t.Fatalf("error executing setupFunc: %v", err) - } - } - - err := CreateServiceAccountKeyAndPublicKeyFiles(dir, x509.RSA) - if (err != nil) != tt.expectedErr { - t.Fatalf("expected error: %v, got: %v, error: %v", tt.expectedErr, err != nil, err) - } else if tt.expectedErr { - return - } - - resultingKeyPEM, wasGenerated, err := keyutil.LoadOrGenerateKeyFile(filepath.Join(dir, kubeadmconstants.ServiceAccountPrivateKeyName)) - if err != nil { - t.Errorf("Can't load created key: %v", err) - } else if wasGenerated { - t.Error("The key was not created") - } else if tt.expectedKey != nil && !bytes.Equal(resultingKeyPEM, tt.expectedKey) { - t.Error("Non-existing key is used") - } - }) - } -} - -func TestSharedCertificateExists(t *testing.T) { - caCert, caKey := certstestutil.CreateCACert(t) - _, key, _ := certstestutil.CreateTestCert(t, caCert, caKey, certutil.AltNames{}) - publicKey := key.Public() - - var tests = []struct { - name string - files certstestutil.PKIFiles - expectedError bool - }{ - { - name: "success", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "front-proxy-ca.crt": caCert, - "front-proxy-ca.key": caKey, - "sa.pub": publicKey, - "sa.key": key, - "etcd/ca.crt": caCert, - "etcd/ca.key": caKey, - }, - }, - { - name: "missing ca.crt", - files: certstestutil.PKIFiles{ - "ca.key": caKey, - "front-proxy-ca.crt": caCert, - "front-proxy-ca.key": caKey, - "sa.pub": publicKey, - "sa.key": key, - "etcd/ca.crt": caCert, - "etcd/ca.key": caKey, - }, - expectedError: true, - }, - { - name: "missing ca.key", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "front-proxy-ca.crt": caCert, - "front-proxy-ca.key": caKey, - "sa.pub": publicKey, - "sa.key": key, - "etcd/ca.crt": caCert, - "etcd/ca.key": caKey, - }, - expectedError: false, - }, - { - name: "missing sa.key", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "front-proxy-ca.crt": caCert, - "front-proxy-ca.key": caKey, - "sa.pub": publicKey, - "etcd/ca.crt": caCert, - "etcd/ca.key": caKey, - }, - expectedError: true, - }, - { - name: "missing front-proxy.crt", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "front-proxy-ca.key": caKey, - "sa.pub": publicKey, - "sa.key": key, - "etcd/ca.crt": caCert, - "etcd/ca.key": caKey, - }, - expectedError: true, - }, - { - name: "missing etcd/ca.crt", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "front-proxy-ca.key": caKey, - "sa.pub": publicKey, - "sa.key": key, - "etcd/ca.crt": caCert, - "etcd/ca.key": caKey, - }, - expectedError: true, - }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - os.MkdirAll(tmpdir+"/etcd", os.ModePerm) - defer os.RemoveAll(tmpdir) - - cfg := &kubeadmapi.ClusterConfiguration{ - CertificatesDir: tmpdir, - } - - // created expected keys - certstestutil.WritePKIFiles(t, tmpdir, test.files) - - // executes create func - ret, err := SharedCertificateExists(cfg) - - switch { - case !test.expectedError && err != nil: - t.Errorf("error SharedCertificateExists failed when not expected to fail: %v", err) - case test.expectedError && err == nil: - t.Errorf("error SharedCertificateExists didn't failed when expected") - case ret != (err == nil): - t.Errorf("error SharedCertificateExists returned %v when expected to return %v", ret, err == nil) - } - }) - } -} - -func TestCreatePKIAssetsWithSparseCerts(t *testing.T) { - for _, test := range certstestutil.GetSparseCertTestCases(t) { - t.Run(test.Name, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - cfg := testutil.GetDefaultInternalConfig(t) - cfg.ClusterConfiguration.CertificatesDir = tmpdir - - certstestutil.WritePKIFiles(t, tmpdir, test.Files) - - err := CreatePKIAssets(cfg) - if err != nil { - if test.ExpectError { - return - } - t.Fatalf("Unexpected error: %v", err) - } - if test.ExpectError { - t.Fatal("Expected error from CreatePKIAssets, got none") - } - assertCertsExist(t, tmpdir) - }) - } - -} - -func TestUsingExternalCA(t *testing.T) { - tests := []struct { - name string - setupFuncs []func(cfg *kubeadmapi.InitConfiguration) error - externalCAFunc func(*kubeadmapi.ClusterConfiguration) (bool, error) - expected bool - expectedErr bool - }{ - { - name: "Test External CA, when complete PKI exists", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - }, - externalCAFunc: UsingExternalCA, - expected: false, - }, - { - name: "Test External CA, when ca.key missing", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - deleteCertOrKey(kubeadmconstants.CAKeyName), - }, - externalCAFunc: UsingExternalCA, - expected: true, - }, - { - name: "Test External CA, when ca.key missing and signed certs are missing", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - deleteCertOrKey(kubeadmconstants.CAKeyName), - deleteCertOrKey(kubeadmconstants.APIServerCertName), - }, - externalCAFunc: UsingExternalCA, - expected: true, - expectedErr: true, - }, - { - name: "Test External CA, when ca.key missing", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - deleteCertOrKey(kubeadmconstants.CAKeyName), - }, - externalCAFunc: UsingExternalCA, - expected: true, - }, - { - name: "Test External Front Proxy CA, when complete PKI exists", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - }, - externalCAFunc: UsingExternalFrontProxyCA, - expected: false, - }, - { - name: "Test External Front Proxy CA, when front-proxy-ca.key missing", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - deleteCertOrKey(kubeadmconstants.FrontProxyCAKeyName), - }, - externalCAFunc: UsingExternalFrontProxyCA, - expected: true, - }, - { - name: "Test External Front Proxy CA, when front-proxy-.key missing and signed certs are missing", - setupFuncs: []func(cfg *kubeadmapi.InitConfiguration) error{ - CreatePKIAssets, - deleteCertOrKey(kubeadmconstants.FrontProxyCAKeyName), - deleteCertOrKey(kubeadmconstants.FrontProxyClientCertName), - }, - externalCAFunc: UsingExternalFrontProxyCA, - expected: true, - expectedErr: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - dir := testutil.SetupTempDir(t) - defer os.RemoveAll(dir) - - cfg := &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - CertificatesDir: dir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"}, - } - - for _, f := range test.setupFuncs { - if err := f(cfg); err != nil { - t.Errorf("error executing setup function: %v", err) - } - } - - val, err := test.externalCAFunc(&cfg.ClusterConfiguration) - if val != test.expected { - t.Errorf("UsingExternalCA did not match expected: %v", test.expected) - } - - if (err != nil) != test.expectedErr { - t.Errorf("UsingExternalCA returned un expected err: %v", err) - } - }) - } -} - -func TestValidateMethods(t *testing.T) { - - caCert, caKey := certstestutil.CreateCACert(t) - cert, key, _ := certstestutil.CreateTestCert(t, caCert, caKey, certutil.AltNames{}) - - tests := []struct { - name string - files certstestutil.PKIFiles - validateFunc func(l certKeyLocation) error - loc certKeyLocation - expectedSuccess bool - }{ - { - name: "validateCACert", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - }, - validateFunc: validateCACert, - loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"}, - expectedSuccess: true, - }, - { - name: "validateCACertAndKey (files present)", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - }, - validateFunc: validateCACertAndKey, - loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"}, - expectedSuccess: true, - }, - { - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - }, - name: "validateCACertAndKey (key missing)", - validateFunc: validateCACertAndKey, - loc: certKeyLocation{caBaseName: "ca", baseName: "", uxName: "CA"}, - expectedSuccess: true, - }, - { - name: "validateSignedCert", - files: certstestutil.PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "apiserver.crt": cert, - "apiserver.key": key, - }, - validateFunc: validateSignedCert, - loc: certKeyLocation{caBaseName: "ca", baseName: "apiserver", uxName: "apiserver"}, - expectedSuccess: true, - }, - { - name: "validatePrivatePublicKey", - files: certstestutil.PKIFiles{ - "sa.pub": key.Public(), - "sa.key": key, - }, - validateFunc: validatePrivatePublicKey, - loc: certKeyLocation{baseName: "sa", uxName: "service account"}, - expectedSuccess: true, - }, - { - name: "validatePrivatePublicKey (missing key)", - files: certstestutil.PKIFiles{ - "sa.pub": key.Public(), - }, - validateFunc: validatePrivatePublicKey, - loc: certKeyLocation{baseName: "sa", uxName: "service account"}, - expectedSuccess: false, - }, - } - - for _, test := range tests { - dir := testutil.SetupTempDir(t) - defer os.RemoveAll(dir) - test.loc.pkiDir = dir - - certstestutil.WritePKIFiles(t, dir, test.files) - - err := test.validateFunc(test.loc) - if test.expectedSuccess && err != nil { - t.Errorf("expected success, error executing validateFunc: %v, %v", test.name, err) - } else if !test.expectedSuccess && err == nil { - t.Errorf("expected failure, no error executing validateFunc: %v", test.name) - } - } -} - -func TestNewCSR(t *testing.T) { - kubeadmCert := KubeadmCertAPIServer() - cfg := testutil.GetDefaultInternalConfig(t) - - certConfig, err := kubeadmCert.GetConfig(cfg) - if err != nil { - t.Fatalf("couldn't get cert config: %v", err) - } - - csr, _, err := NewCSR(kubeadmCert, cfg) - - if err != nil { - t.Errorf("invalid signature on CSR: %v", err) - } - - assert.ElementsMatch(t, certConfig.Organization, csr.Subject.Organization, "organizations not equal") - - if csr.Subject.CommonName != certConfig.CommonName { - t.Errorf("expected common name %q, got %q", certConfig.CommonName, csr.Subject.CommonName) - } - - assert.ElementsMatch(t, certConfig.AltNames.DNSNames, csr.DNSNames, "dns names not equal") - - assert.Len(t, csr.IPAddresses, len(certConfig.AltNames.IPs)) - - for i, ip := range csr.IPAddresses { - if !ip.Equal(certConfig.AltNames.IPs[i]) { - t.Errorf("[%d]: %v != %v", i, ip, certConfig.AltNames.IPs[i]) - } - } -} - -func TestCreateCertificateFilesMethods(t *testing.T) { - - var tests = []struct { - createFunc func(cfg *kubeadmapi.InitConfiguration) error - expectedFiles []string - externalEtcd bool - }{ - { - createFunc: CreatePKIAssets, - expectedFiles: []string{ - kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, - kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, - kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, - kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName, - kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName, - kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName, - kubeadmconstants.EtcdHealthcheckClientCertName, kubeadmconstants.EtcdHealthcheckClientKeyName, - kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName, - kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName, - kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, - kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName, - }, - }, - { - createFunc: CreatePKIAssets, - externalEtcd: true, - expectedFiles: []string{ - kubeadmconstants.CACertName, kubeadmconstants.CAKeyName, - kubeadmconstants.APIServerCertName, kubeadmconstants.APIServerKeyName, - kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName, - kubeadmconstants.ServiceAccountPrivateKeyName, kubeadmconstants.ServiceAccountPublicKeyName, - kubeadmconstants.FrontProxyCACertName, kubeadmconstants.FrontProxyCAKeyName, - kubeadmconstants.FrontProxyClientCertName, kubeadmconstants.FrontProxyClientKeyName, - }, - }, - } - - for _, test := range tests { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - cfg := &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{Local: &kubeadmapi.LocalEtcd{}}, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - CertificatesDir: tmpdir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"}, - } - - if test.externalEtcd { - if cfg.Etcd.External == nil { - cfg.Etcd.External = &kubeadmapi.ExternalEtcd{} - } - cfg.Etcd.Local = nil - cfg.Etcd.External.Endpoints = []string{"192.168.1.1:2379"} - } - - // executes create func - if err := test.createFunc(cfg); err != nil { - t.Errorf("error executing createFunc: %v", err) - continue - } - - // asserts expected files are there - testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) - } -} - -func deleteCertOrKey(name string) func(*kubeadmapi.InitConfiguration) error { - return func(cfg *kubeadmapi.InitConfiguration) error { - if err := os.Remove(filepath.Join(cfg.CertificatesDir, name)); err != nil { - return errors.Wrapf(err, "failed removing %s", name) - } - return nil - } -} - -func assertCertsExist(t *testing.T, dir string) { - tree, err := GetDefaultCertList().AsMap().CertTree() - if err != nil { - t.Fatalf("unexpected error getting certificates: %v", err) - } - - for caCert, certs := range tree { - if err := validateCACert(certKeyLocation{dir, caCert.BaseName, "", caCert.Name}); err != nil { - t.Errorf("couldn't validate CA certificate %v: %v", caCert.Name, err) - // Don't bother validating child certs, but do try the other CAs - continue - } - - for _, cert := range certs { - if err := validateSignedCert(certKeyLocation{dir, caCert.BaseName, cert.BaseName, cert.Name}); err != nil { - t.Errorf("couldn't validate certificate %v: %v", cert.Name, err) - } - } - } -} diff --git a/cmd/kubeadm/app/phases/certs/doc.go b/cmd/kubeadm/app/phases/certs/doc.go deleted file mode 100644 index 1ab96d110b7ee..0000000000000 --- a/cmd/kubeadm/app/phases/certs/doc.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -/* - - PHASE: CERTIFICATES - - INPUTS: - From InitConfiguration - .API.AdvertiseAddress is an optional parameter that can be passed for an extra addition to the SAN IPs - .APIServer.CertSANs is an optional parameter for adding DNS names and IPs to the API Server serving cert SAN - .Etcd.Local.ServerCertSANs is an optional parameter for adding DNS names and IPs to the etcd serving cert SAN - .Etcd.Local.PeerCertSANs is an optional parameter for adding DNS names and IPs to the etcd peer cert SAN - .Networking.DNSDomain is needed for knowing which DNS name the internal Kubernetes service has - .Networking.ServiceSubnet is needed for knowing which IP the internal Kubernetes service is going to point to - .CertificatesDir is required for knowing where all certificates should be stored - - OUTPUTS: - Files to .CertificatesDir (default /etc/kubernetes/pki): - - ca.crt - - ca.key - - apiserver.crt - - apiserver.key - - apiserver-kubelet-client.crt - - apiserver-kubelet-client.key - - apiserver-etcd-client.crt - - apiserver-etcd-client.key - - etcd/ca.crt - - etcd/ca.key - - etcd/server.crt - - etcd/server.key - - etcd/peer.crt - - etcd/peer.key - - etcd/healthcheck-client.crt - - etcd/healthcheck-client.key - - sa.pub - - sa.key - - front-proxy-ca.crt - - front-proxy-ca.key - - front-proxy-client.crt - - front-proxy-client.key - -*/ diff --git a/cmd/kubeadm/app/phases/certs/renewal/BUILD b/cmd/kubeadm/app/phases/certs/renewal/BUILD deleted file mode 100644 index f5c2e79d646ad..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "expiration.go", - "filerenewer.go", - "manager.go", - "readwriter.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "expiration_test.go", - "filerenewer_test.go", - "manager_test.go", - "readwriter_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/certs:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/phases/certs/renewal/expiration.go b/cmd/kubeadm/app/phases/certs/renewal/expiration.go deleted file mode 100644 index fe96ab2e02520..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/expiration.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto/x509" - "time" -) - -// ExpirationInfo defines expiration info for a certificate -type ExpirationInfo struct { - // Name of the certificate - // For PKI certificates, it is the name defined in the certsphase package, while for certificates - // embedded in the kubeConfig files, it is the kubeConfig file name defined in the kubeadm constants package. - // If you use the CertificateRenewHandler returned by Certificates func, handler.Name already contains the right value. - Name string - - // ExpirationDate defines certificate expiration date - ExpirationDate time.Time - - // ExternallyManaged defines if the certificate is externally managed, that is when - // the signing CA certificate is provided without the certificate key (In this case kubeadm can't renew the certificate) - ExternallyManaged bool -} - -// newExpirationInfo returns a new ExpirationInfo -func newExpirationInfo(name string, cert *x509.Certificate, externallyManaged bool) *ExpirationInfo { - return &ExpirationInfo{ - Name: name, - ExpirationDate: cert.NotAfter, - ExternallyManaged: externallyManaged, - } -} - -// ResidualTime returns the time missing to expiration -func (e *ExpirationInfo) ResidualTime() time.Duration { - return time.Until(e.ExpirationDate) -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/expiration_test.go b/cmd/kubeadm/app/phases/certs/renewal/expiration_test.go deleted file mode 100644 index afa13c53fe27b..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/expiration_test.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto/x509" - "math" - "testing" - "time" -) - -func TestExpirationInfo(t *testing.T) { - validity := 365 * 24 * time.Hour - cert := &x509.Certificate{ - NotAfter: time.Now().Add(validity), - } - - e := newExpirationInfo("x", cert, false) - - if math.Abs(float64(validity-e.ResidualTime())) > float64(5*time.Second) { // using 5s of tolerance because the function is not deterministic (it uses time.Now()) and we want to avoid flakes - t.Errorf("expected IsInRenewalWindow equal to %v, saw %v", validity, e.ResidualTime()) - } -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/filerenewer.go b/cmd/kubeadm/app/phases/certs/renewal/filerenewer.go deleted file mode 100644 index 6299ffe055608..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/filerenewer.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto" - "crypto/x509" - - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -// FileRenewer define a certificate renewer implementation that uses given CA cert and key for generating new certificates -type FileRenewer struct { - caCert *x509.Certificate - caKey crypto.Signer -} - -// NewFileRenewer returns a new certificate renewer that uses given CA cert and key for generating new certificates -func NewFileRenewer(caCert *x509.Certificate, caKey crypto.Signer) *FileRenewer { - return &FileRenewer{ - caCert: caCert, - caKey: caKey, - } -} - -// Renew a certificate using a given CA cert and key -func (r *FileRenewer) Renew(cfg *pkiutil.CertConfig) (*x509.Certificate, crypto.Signer, error) { - return pkiutil.NewCertAndKey(r.caCert, r.caKey, cfg) -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/filerenewer_test.go b/cmd/kubeadm/app/phases/certs/renewal/filerenewer_test.go deleted file mode 100644 index 371ffbf64c28b..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/filerenewer_test.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto/x509" - "testing" - - certutil "k8s.io/client-go/util/cert" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -func TestFileRenewer(t *testing.T) { - // creates a File renewer using a test Certificate authority - fr := NewFileRenewer(testCACert, testCAKey) - - // renews a certificate - certCfg := &pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "test-certs", - AltNames: certutil.AltNames{ - DNSNames: []string{"test-domain.space"}, - }, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - } - - cert, _, err := fr.Renew(certCfg) - if err != nil { - t.Fatalf("unexpected error renewing cert: %v", err) - } - - // verify the renewed certificate - pool := x509.NewCertPool() - pool.AddCert(testCACert) - - _, err = cert.Verify(x509.VerifyOptions{ - DNSName: "test-domain.space", - Roots: pool, - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }) - if err != nil { - t.Errorf("couldn't verify new cert: %v", err) - } - -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/manager.go b/cmd/kubeadm/app/phases/certs/renewal/manager.go deleted file mode 100644 index 0b3f93a31f964..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/manager.go +++ /dev/null @@ -1,391 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto/x509" - "sort" - - "github.com/pkg/errors" - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -// Manager can be used to coordinate certificate renewal and related processes, -// like CSR generation or checking certificate expiration -type Manager struct { - // cfg holds the kubeadm ClusterConfiguration - cfg *kubeadmapi.ClusterConfiguration - - // kubernetesDir holds the directory where kubeConfig files are stored - kubernetesDir string - - // certificates contains the certificateRenewHandler controlled by this manager - certificates map[string]*CertificateRenewHandler - - // cas contains the CAExpirationHandler related to the certificates that are controlled by this manager - cas map[string]*CAExpirationHandler -} - -// CertificateRenewHandler defines required info for renewing a certificate -type CertificateRenewHandler struct { - // Name of the certificate to be used for UX. - // This value can be used to trigger operations on this certificate - Name string - - // LongName of the certificate to be used for UX - LongName string - - // FileName defines the name (or the BaseName) of the certificate file - FileName string - - // CAName defines the name for the CA on which this certificate depends - CAName string - - // CABaseName defines the base name for the CA that should be used for certificate renewal - CABaseName string - - // readwriter defines a CertificateReadWriter to be used for certificate renewal - readwriter certificateReadWriter -} - -// CAExpirationHandler defines required info for CA expiration check -type CAExpirationHandler struct { - // Name of the CA to be used for UX. - // This value can be used to trigger operations on this CA - Name string - - // LongName of the CA to be used for UX - LongName string - - // FileName defines the name (or the BaseName) of the CA file - FileName string - - // readwriter defines a CertificateReadWriter to be used for CA expiration check - readwriter certificateReadWriter -} - -// NewManager return a new certificate renewal manager ready for handling certificates in the cluster -func NewManager(cfg *kubeadmapi.ClusterConfiguration, kubernetesDir string) (*Manager, error) { - rm := &Manager{ - cfg: cfg, - kubernetesDir: kubernetesDir, - certificates: map[string]*CertificateRenewHandler{}, - cas: map[string]*CAExpirationHandler{}, - } - - // gets the list of certificates that are expected according to the current cluster configuration - certListFunc := certsphase.GetDefaultCertList - if cfg.Etcd.External != nil { - certListFunc = certsphase.GetCertsWithoutEtcd - } - certTree, err := certListFunc().AsMap().CertTree() - if err != nil { - return nil, err - } - - // create a CertificateRenewHandler for each signed certificate in the certificate tree; - // NB. we are not offering support for renewing CAs; this would cause serious consequences - for ca, certs := range certTree { - for _, cert := range certs { - // create a ReadWriter for certificates stored in the K8s local PKI - pkiReadWriter := newPKICertificateReadWriter(rm.cfg.CertificatesDir, cert.BaseName) - - // adds the certificateRenewHandler. - // PKI certificates are indexed by name, that is a well know constant defined - // in the certsphase package and that can be reused across all the kubeadm codebase - rm.certificates[cert.Name] = &CertificateRenewHandler{ - Name: cert.Name, - LongName: cert.LongName, - FileName: cert.BaseName, - CAName: ca.Name, - CABaseName: ca.BaseName, //Nb. this is a path for etcd certs (they are stored in a subfolder) - readwriter: pkiReadWriter, - } - } - - pkiReadWriter := newPKICertificateReadWriter(rm.cfg.CertificatesDir, ca.BaseName) - rm.cas[ca.Name] = &CAExpirationHandler{ - Name: ca.Name, - LongName: ca.LongName, - FileName: ca.BaseName, - readwriter: pkiReadWriter, - } - } - - // gets the list of certificates that should be considered for renewal - kubeConfigs := []struct { - longName string - fileName string - }{ - { - longName: "certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself", - fileName: kubeadmconstants.AdminKubeConfigFileName, - }, - { - longName: "certificate embedded in the kubeconfig file for the controller manager to use", - fileName: kubeadmconstants.ControllerManagerKubeConfigFileName, - }, - { - longName: "certificate embedded in the kubeconfig file for the scheduler manager to use", - fileName: kubeadmconstants.SchedulerKubeConfigFileName, - }, - //NB. we are excluding KubeletKubeConfig from renewal because management of this certificate is delegated to kubelet - } - - // create a CertificateRenewHandler for each kubeConfig file - for _, kubeConfig := range kubeConfigs { - // create a ReadWriter for certificates embedded in kubeConfig files - kubeConfigReadWriter := newKubeconfigReadWriter(kubernetesDir, kubeConfig.fileName, - rm.cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) - - // adds the certificateRenewHandler. - // Certificates embedded kubeConfig files in are indexed by fileName, that is a well know constant defined - // in the kubeadm constants package and that can be reused across all the kubeadm codebase - rm.certificates[kubeConfig.fileName] = &CertificateRenewHandler{ - Name: kubeConfig.fileName, // we are using fileName as name, because there is nothing similar outside - LongName: kubeConfig.longName, - FileName: kubeConfig.fileName, - CABaseName: kubeadmconstants.CACertAndKeyBaseName, // all certificates in kubeConfig files are signed by the Kubernetes CA - readwriter: kubeConfigReadWriter, - } - } - - return rm, nil -} - -// Certificates returns the list of certificates controlled by this Manager -func (rm *Manager) Certificates() []*CertificateRenewHandler { - certificates := []*CertificateRenewHandler{} - for _, h := range rm.certificates { - certificates = append(certificates, h) - } - - sort.Slice(certificates, func(i, j int) bool { return certificates[i].Name < certificates[j].Name }) - - return certificates -} - -// CAs returns the list of CAs related to the certificates that are controlled by this manager -func (rm *Manager) CAs() []*CAExpirationHandler { - cas := []*CAExpirationHandler{} - for _, h := range rm.cas { - cas = append(cas, h) - } - - sort.Slice(cas, func(i, j int) bool { return cas[i].Name < cas[j].Name }) - - return cas -} - -// RenewUsingLocalCA executes certificate renewal using local certificate authorities for generating new certs. -// For PKI certificates, use the name defined in the certsphase package, while for certificates -// embedded in the kubeConfig files, use the kubeConfig file name defined in the kubeadm constants package. -// If you use the CertificateRenewHandler returned by Certificates func, handler.Name already contains the right value. -func (rm *Manager) RenewUsingLocalCA(name string) (bool, error) { - handler, ok := rm.certificates[name] - if !ok { - return false, errors.Errorf("%s is not a valid certificate for this cluster", name) - } - - // checks if the certificate is externally managed (CA certificate provided without the certificate key) - externallyManaged, err := rm.IsExternallyManaged(handler.CABaseName) - if err != nil { - return false, err - } - - // in case of external CA it is not possible to renew certificates, then return early - if externallyManaged { - return false, nil - } - - // reads the current certificate - cert, err := handler.readwriter.Read() - if err != nil { - return false, err - } - - // extract the certificate config - cfg := &pkiutil.CertConfig{ - Config: certToConfig(cert), - PublicKeyAlgorithm: rm.cfg.PublicKeyAlgorithm(), - } - - // reads the CA - caCert, caKey, err := certsphase.LoadCertificateAuthority(rm.cfg.CertificatesDir, handler.CABaseName) - if err != nil { - return false, err - } - - // create a new certificate with the same config - newCert, newKey, err := NewFileRenewer(caCert, caKey).Renew(cfg) - if err != nil { - return false, errors.Wrapf(err, "failed to renew certificate %s", name) - } - - // writes the new certificate to disk - err = handler.readwriter.Write(newCert, newKey) - if err != nil { - return false, err - } - - return true, nil -} - -// CreateRenewCSR generates CSR request for certificate renewal. -// For PKI certificates, use the name defined in the certsphase package, while for certificates -// embedded in the kubeConfig files, use the kubeConfig file name defined in the kubeadm constants package. -// If you use the CertificateRenewHandler returned by Certificates func, handler.Name already contains the right value. -func (rm *Manager) CreateRenewCSR(name, outdir string) error { - handler, ok := rm.certificates[name] - if !ok { - return errors.Errorf("%s is not a known certificate", name) - } - - // reads the current certificate - cert, err := handler.readwriter.Read() - if err != nil { - return err - } - - // extracts the certificate config - cfg := &pkiutil.CertConfig{ - Config: certToConfig(cert), - PublicKeyAlgorithm: rm.cfg.PublicKeyAlgorithm(), - } - - // generates the CSR request and save it - csr, key, err := pkiutil.NewCSRAndKey(cfg) - if err != nil { - return errors.Wrapf(err, "failure while generating %s CSR and key", name) - } - if err := pkiutil.WriteKey(outdir, name, key); err != nil { - return errors.Wrapf(err, "failure while saving %s key", name) - } - - if err := pkiutil.WriteCSR(outdir, name, csr); err != nil { - return errors.Wrapf(err, "failure while saving %s CSR", name) - } - - return nil -} - -// CertificateExists returns true if a certificate exists. -func (rm *Manager) CertificateExists(name string) (bool, error) { - handler, ok := rm.certificates[name] - if !ok { - return false, errors.Errorf("%s is not a known certificate", name) - } - - return handler.readwriter.Exists(), nil -} - -// GetCertificateExpirationInfo returns certificate expiration info. -// For PKI certificates, use the name defined in the certsphase package, while for certificates -// embedded in the kubeConfig files, use the kubeConfig file name defined in the kubeadm constants package. -// If you use the CertificateRenewHandler returned by Certificates func, handler.Name already contains the right value. -func (rm *Manager) GetCertificateExpirationInfo(name string) (*ExpirationInfo, error) { - handler, ok := rm.certificates[name] - if !ok { - return nil, errors.Errorf("%s is not a known certificate", name) - } - - // checks if the certificate is externally managed (CA certificate provided without the certificate key) - externallyManaged, err := rm.IsExternallyManaged(handler.CABaseName) - if err != nil { - return nil, err - } - - // reads the current certificate - cert, err := handler.readwriter.Read() - if err != nil { - return nil, err - } - - // returns the certificate expiration info - return newExpirationInfo(name, cert, externallyManaged), nil -} - -// CAExists returns true if a certificate authority exists. -func (rm *Manager) CAExists(name string) (bool, error) { - handler, ok := rm.cas[name] - if !ok { - return false, errors.Errorf("%s is not a known certificate", name) - } - - return handler.readwriter.Exists(), nil -} - -// GetCAExpirationInfo returns CA expiration info. -func (rm *Manager) GetCAExpirationInfo(name string) (*ExpirationInfo, error) { - handler, ok := rm.cas[name] - if !ok { - return nil, errors.Errorf("%s is not a known CA", name) - } - - // checks if the CA is externally managed (CA certificate provided without the certificate key) - externallyManaged, err := rm.IsExternallyManaged(handler.FileName) - if err != nil { - return nil, err - } - - // reads the current CA - ca, err := handler.readwriter.Read() - if err != nil { - return nil, err - } - - // returns the CA expiration info - return newExpirationInfo(name, ca, externallyManaged), nil -} - -// IsExternallyManaged checks if we are in the external CA case (CA certificate provided without the certificate key) -func (rm *Manager) IsExternallyManaged(caBaseName string) (bool, error) { - switch caBaseName { - case kubeadmconstants.CACertAndKeyBaseName: - externallyManaged, err := certsphase.UsingExternalCA(rm.cfg) - if err != nil { - return false, errors.Wrapf(err, "Error checking external CA condition for %s certificate authority", caBaseName) - } - return externallyManaged, nil - case kubeadmconstants.FrontProxyCACertAndKeyBaseName: - externallyManaged, err := certsphase.UsingExternalFrontProxyCA(rm.cfg) - if err != nil { - return false, errors.Wrapf(err, "Error checking external CA condition for %s certificate authority", caBaseName) - } - return externallyManaged, nil - case kubeadmconstants.EtcdCACertAndKeyBaseName: - return false, nil - default: - return false, errors.Errorf("unknown certificate authority %s", caBaseName) - } -} - -func certToConfig(cert *x509.Certificate) certutil.Config { - return certutil.Config{ - CommonName: cert.Subject.CommonName, - Organization: cert.Subject.Organization, - AltNames: certutil.AltNames{ - IPs: cert.IPAddresses, - DNSNames: cert.DNSNames, - }, - Usages: cert.ExtKeyUsage, - } -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/manager_test.go b/cmd/kubeadm/app/phases/certs/renewal/manager_test.go deleted file mode 100644 index 85b1325a7ac9f..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/manager_test.go +++ /dev/null @@ -1,274 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto/x509" - "crypto/x509/pkix" - "fmt" - "net" - "os" - "path/filepath" - "testing" - "time" - - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - certtestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -var ( - testCACertCfg = &pkiutil.CertConfig{ - Config: certutil.Config{CommonName: "kubernetes"}, - } - - testCACert, testCAKey, _ = pkiutil.NewCertificateAuthority(testCACertCfg) - - testCertCfg = &pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "test-common-name", - Organization: []string{"sig-cluster-lifecycle"}, - AltNames: certutil.AltNames{ - IPs: []net.IP{net.ParseIP("10.100.0.1")}, - DNSNames: []string{"test-domain.space"}, - }, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - } -) - -func TestNewManager(t *testing.T) { - tests := []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - expectedCertificates int - }{ - { - name: "cluster with local etcd", - cfg: &kubeadmapi.ClusterConfiguration{}, - expectedCertificates: 10, //[admin apiserver apiserver-etcd-client apiserver-kubelet-client controller-manager etcd/healthcheck-client etcd/peer etcd/server front-proxy-client scheduler] - }, - { - name: "cluster with external etcd", - cfg: &kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{}, - }, - }, - expectedCertificates: 6, // [admin apiserver apiserver-kubelet-client controller-manager front-proxy-client scheduler] - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - rm, err := NewManager(test.cfg, "") - if err != nil { - t.Fatalf("Failed to create the certificate renewal manager: %v", err) - } - - if len(rm.Certificates()) != test.expectedCertificates { - t.Errorf("Expected %d certificates, saw %d", test.expectedCertificates, len(rm.Certificates())) - } - }) - } -} - -func TestRenewUsingLocalCA(t *testing.T) { - dir := testutil.SetupTempDir(t) - defer os.RemoveAll(dir) - - if err := pkiutil.WriteCertAndKey(dir, "ca", testCACert, testCAKey); err != nil { - t.Fatalf("couldn't write out CA certificate to %s", dir) - } - - cfg := &kubeadmapi.ClusterConfiguration{ - CertificatesDir: dir, - } - rm, err := NewManager(cfg, dir) - if err != nil { - t.Fatalf("Failed to create the certificate renewal manager: %v", err) - } - - tests := []struct { - name string - certName string - createCertFunc func() *x509.Certificate - }{ - { - name: "Certificate renewal for a PKI certificate", - certName: "apiserver", - createCertFunc: func() *x509.Certificate { - return writeTestCertificate(t, dir, "apiserver", testCACert, testCAKey) - }, - }, - { - name: "Certificate renewal for a certificate embedded in a kubeconfig file", - certName: "admin.conf", - createCertFunc: func() *x509.Certificate { - return writeTestKubeconfig(t, dir, "admin.conf", testCACert, testCAKey) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cert := test.createCertFunc() - - time.Sleep(1 * time.Second) - - _, err := rm.RenewUsingLocalCA(test.certName) - if err != nil { - t.Fatalf("error renewing certificate: %v", err) - } - - newCert, err := rm.certificates[test.certName].readwriter.Read() - if err != nil { - t.Fatalf("error reading renewed certificate: %v", err) - } - - if newCert.SerialNumber.Cmp(cert.SerialNumber) == 0 { - t.Fatal("expected new certificate, but renewed certificate has same serial number") - } - - if !newCert.NotAfter.After(cert.NotAfter) { - t.Fatalf("expected new certificate with updated expiration, but renewed certificate has same NotAfter value: saw %s, expected greather than %s", newCert.NotAfter, cert.NotAfter) - } - - certtestutil.AssertCertificateIsSignedByCa(t, newCert, testCACert) - certtestutil.AssertCertificateHasClientAuthUsage(t, newCert) - certtestutil.AssertCertificateHasOrganizations(t, newCert, testCertCfg.Organization...) - certtestutil.AssertCertificateHasCommonName(t, newCert, testCertCfg.CommonName) - certtestutil.AssertCertificateHasDNSNames(t, newCert, testCertCfg.AltNames.DNSNames...) - certtestutil.AssertCertificateHasIPAddresses(t, newCert, testCertCfg.AltNames.IPs...) - }) - } -} - -func TestCreateRenewCSR(t *testing.T) { - dir := testutil.SetupTempDir(t) - defer os.RemoveAll(dir) - - outdir := filepath.Join(dir, "out") - - if err := os.MkdirAll(outdir, 0755); err != nil { - t.Fatalf("couldn't create %s", outdir) - } - - if err := pkiutil.WriteCertAndKey(dir, "ca", testCACert, testCAKey); err != nil { - t.Fatalf("couldn't write out CA certificate to %s", dir) - } - - cfg := &kubeadmapi.ClusterConfiguration{ - CertificatesDir: dir, - } - rm, err := NewManager(cfg, dir) - if err != nil { - t.Fatalf("Failed to create the certificate renewal manager: %v", err) - } - - tests := []struct { - name string - certName string - createCertFunc func() *x509.Certificate - }{ - { - name: "Creation of a CSR request for renewal of a PKI certificate", - certName: "apiserver", - createCertFunc: func() *x509.Certificate { - return writeTestCertificate(t, dir, "apiserver", testCACert, testCAKey) - }, - }, - { - name: "Creation of a CSR request for renewal of a certificate embedded in a kubeconfig file", - certName: "admin.conf", - createCertFunc: func() *x509.Certificate { - return writeTestKubeconfig(t, dir, "admin.conf", testCACert, testCAKey) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - test.createCertFunc() - - time.Sleep(1 * time.Second) - - err := rm.CreateRenewCSR(test.certName, outdir) - if err != nil { - t.Fatalf("error renewing certificate: %v", err) - } - - file := fmt.Sprintf("%s.key", test.certName) - if _, err := os.Stat(filepath.Join(outdir, file)); os.IsNotExist(err) { - t.Errorf("Expected file %s does not exist", file) - } - - file = fmt.Sprintf("%s.csr", test.certName) - if _, err := os.Stat(filepath.Join(outdir, file)); os.IsNotExist(err) { - t.Errorf("Expected file %s does not exist", file) - } - }) - } - -} - -func TestCertToConfig(t *testing.T) { - expectedConfig := &certutil.Config{ - CommonName: "test-common-name", - Organization: []string{"sig-cluster-lifecycle"}, - AltNames: certutil.AltNames{ - IPs: []net.IP{net.ParseIP("10.100.0.1")}, - DNSNames: []string{"test-domain.space"}, - }, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - } - - cert := &x509.Certificate{ - Subject: pkix.Name{ - CommonName: "test-common-name", - Organization: []string{"sig-cluster-lifecycle"}, - }, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - DNSNames: []string{"test-domain.space"}, - IPAddresses: []net.IP{net.ParseIP("10.100.0.1")}, - } - - cfg := certToConfig(cert) - - if cfg.CommonName != expectedConfig.CommonName { - t.Errorf("expected common name %q, got %q", expectedConfig.CommonName, cfg.CommonName) - } - - if len(cfg.Organization) != 1 || cfg.Organization[0] != expectedConfig.Organization[0] { - t.Errorf("expected organization %v, got %v", expectedConfig.Organization, cfg.Organization) - - } - - if len(cfg.Usages) != 1 || cfg.Usages[0] != expectedConfig.Usages[0] { - t.Errorf("expected ext key usage %v, got %v", expectedConfig.Usages, cfg.Usages) - } - - if len(cfg.AltNames.IPs) != 1 || cfg.AltNames.IPs[0].String() != expectedConfig.AltNames.IPs[0].String() { - t.Errorf("expected SAN IPs %v, got %v", expectedConfig.AltNames.IPs, cfg.AltNames.IPs) - } - - if len(cfg.AltNames.DNSNames) != 1 || cfg.AltNames.DNSNames[0] != expectedConfig.AltNames.DNSNames[0] { - t.Errorf("expected SAN DNSNames %v, got %v", expectedConfig.AltNames.DNSNames, cfg.AltNames.DNSNames) - } -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/readwriter.go b/cmd/kubeadm/app/phases/certs/renewal/readwriter.go deleted file mode 100644 index 865c432cc5929..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/readwriter.go +++ /dev/null @@ -1,218 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto" - "crypto/x509" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -// certificateReadWriter defines the behavior of a component that -// read or write a certificate stored/embedded in a file -type certificateReadWriter interface { - //Exists return true if the certificate exists - Exists() bool - - // Read a certificate stored/embedded in a file - Read() (*x509.Certificate, error) - - // Write (update) a certificate stored/embedded in a file - Write(*x509.Certificate, crypto.Signer) error -} - -// pkiCertificateReadWriter defines a certificateReadWriter for certificate files -// in the K8s pki managed by kubeadm -type pkiCertificateReadWriter struct { - baseName string - certificateDir string -} - -// newPKICertificateReadWriter return a new pkiCertificateReadWriter -func newPKICertificateReadWriter(certificateDir string, baseName string) *pkiCertificateReadWriter { - return &pkiCertificateReadWriter{ - baseName: baseName, - certificateDir: certificateDir, - } -} - -// Exists checks if a certificate exist -func (rw *pkiCertificateReadWriter) Exists() bool { - certificatePath, _ := pkiutil.PathsForCertAndKey(rw.certificateDir, rw.baseName) - return fileExists(certificatePath) -} - -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -// Read a certificate from a file the K8s pki managed by kubeadm -func (rw *pkiCertificateReadWriter) Read() (*x509.Certificate, error) { - certificatePath, _ := pkiutil.PathsForCertAndKey(rw.certificateDir, rw.baseName) - certs, err := certutil.CertsFromFile(certificatePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to load existing certificate %s", rw.baseName) - } - - if len(certs) != 1 { - return nil, errors.Errorf("wanted exactly one certificate, got %d", len(certs)) - } - - return certs[0], nil -} - -// Write a certificate to files in the K8s pki managed by kubeadm -func (rw *pkiCertificateReadWriter) Write(newCert *x509.Certificate, newKey crypto.Signer) error { - if err := pkiutil.WriteCertAndKey(rw.certificateDir, rw.baseName, newCert, newKey); err != nil { - return errors.Wrapf(err, "failed to write new certificate %s", rw.baseName) - } - return nil -} - -// kubeConfigReadWriter defines a certificateReadWriter for certificate files -// embedded in the kubeConfig files managed by kubeadm, and more specifically -// for the client certificate of the AuthInfo -type kubeConfigReadWriter struct { - kubernetesDir string - kubeConfigFileName string - kubeConfigFilePath string - kubeConfig *clientcmdapi.Config - baseName string - certificateDir string - caCert *x509.Certificate -} - -// newKubeconfigReadWriter return a new kubeConfigReadWriter -func newKubeconfigReadWriter(kubernetesDir string, kubeConfigFileName string, certificateDir, baseName string) *kubeConfigReadWriter { - return &kubeConfigReadWriter{ - kubernetesDir: kubernetesDir, - kubeConfigFileName: kubeConfigFileName, - kubeConfigFilePath: filepath.Join(kubernetesDir, kubeConfigFileName), - certificateDir: certificateDir, - baseName: baseName, - } -} - -// Exists checks if a certificate embedded in kubeConfig file exists -func (rw *kubeConfigReadWriter) Exists() bool { - return fileExists(rw.kubeConfigFilePath) -} - -// Read a certificate embedded in kubeConfig file managed by kubeadm. -// Please note that the kubeConfig file itself is kept in the ReadWriter state thus allowing -// to preserve the attributes (Context, Servers, AuthInfo etc.) -func (rw *kubeConfigReadWriter) Read() (*x509.Certificate, error) { - // try to load the kubeConfig file - kubeConfig, err := clientcmd.LoadFromFile(rw.kubeConfigFilePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to load kubeConfig file %s", rw.kubeConfigFilePath) - } - - // The CA cert is required for updating kubeconfig files. - // For local CA renewal, the local CA on disk could have changed, thus a reload is needed. - // For CSR renewal we assume the same CA on disk is mounted for usage with KCM's - // '--cluster-signing-cert-file' flag. - caCert, _, err := certsphase.LoadCertificateAuthority(rw.certificateDir, rw.baseName) - if err != nil { - return nil, err - } - rw.caCert = caCert - - // get current context - if _, ok := kubeConfig.Contexts[kubeConfig.CurrentContext]; !ok { - return nil, errors.Errorf("invalid kubeConfig file %s: missing context %s", rw.kubeConfigFilePath, kubeConfig.CurrentContext) - } - - // get cluster info for current context and ensure a server certificate is embedded in it - clusterName := kubeConfig.Contexts[kubeConfig.CurrentContext].Cluster - if _, ok := kubeConfig.Clusters[clusterName]; !ok { - return nil, errors.Errorf("invalid kubeConfig file %s: missing cluster %s", rw.kubeConfigFilePath, clusterName) - } - - cluster := kubeConfig.Clusters[clusterName] - if len(cluster.CertificateAuthorityData) == 0 { - return nil, errors.Errorf("kubeConfig file %s does not have an embedded server certificate", rw.kubeConfigFilePath) - } - - // get auth info for current context and ensure a client certificate is embedded in it - authInfoName := kubeConfig.Contexts[kubeConfig.CurrentContext].AuthInfo - if _, ok := kubeConfig.AuthInfos[authInfoName]; !ok { - return nil, errors.Errorf("invalid kubeConfig file %s: missing authInfo %s", rw.kubeConfigFilePath, authInfoName) - } - - authInfo := kubeConfig.AuthInfos[authInfoName] - if len(authInfo.ClientCertificateData) == 0 { - return nil, errors.Errorf("kubeConfig file %s does not have an embedded client certificate", rw.kubeConfigFilePath) - } - - // parse the client certificate, retrive the cert config and then renew it - certs, err := certutil.ParseCertsPEM(authInfo.ClientCertificateData) - if err != nil { - return nil, errors.Wrapf(err, "kubeConfig file %s does not contain a valid client certificate", rw.kubeConfigFilePath) - } - - rw.kubeConfig = kubeConfig - - return certs[0], nil -} - -// Write a certificate embedded in kubeConfig file managed by kubeadm -// Please note that all the other attribute of the kubeConfig file are preserved, but this -// requires to call Read before Write -func (rw *kubeConfigReadWriter) Write(newCert *x509.Certificate, newKey crypto.Signer) error { - // check if Read was called before Write - if rw.kubeConfig == nil { - return errors.Errorf("failed to Write kubeConfig file with renewed certs. It is necessary to call Read before Write") - } - - // encodes the new key - encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(newKey) - if err != nil { - return errors.Wrapf(err, "failed to marshal private key to PEM") - } - - // Update the embedded CA in the kubeconfig file. - // This assumes that the user has kept the current context to the desired one. - clusterName := rw.kubeConfig.Contexts[rw.kubeConfig.CurrentContext].Cluster - cluster := rw.kubeConfig.Clusters[clusterName] - cluster.CertificateAuthorityData = pkiutil.EncodeCertPEM(rw.caCert) - - // get auth info for current context and ensure a client certificate is embedded in it - authInfoName := rw.kubeConfig.Contexts[rw.kubeConfig.CurrentContext].AuthInfo - - // create a kubeConfig copy with the new client certs - newConfig := rw.kubeConfig.DeepCopy() - newConfig.AuthInfos[authInfoName].ClientKeyData = encodedClientKey - newConfig.AuthInfos[authInfoName].ClientCertificateData = pkiutil.EncodeCertPEM(newCert) - - // writes the kubeConfig to disk - return clientcmd.WriteToFile(*newConfig, rw.kubeConfigFilePath) -} diff --git a/cmd/kubeadm/app/phases/certs/renewal/readwriter_test.go b/cmd/kubeadm/app/phases/certs/renewal/readwriter_test.go deleted file mode 100644 index d19c6e58138f1..0000000000000 --- a/cmd/kubeadm/app/phases/certs/renewal/readwriter_test.go +++ /dev/null @@ -1,194 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package renewal - -import ( - "crypto" - "crypto/x509" - "net" - "os" - "path/filepath" - "testing" - - "k8s.io/client-go/tools/clientcmd" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func TestPKICertificateReadWriter(t *testing.T) { - // creates a tmp folder - dir := testutil.SetupTempDir(t) - defer os.RemoveAll(dir) - - // creates a certificate - cert := writeTestCertificate(t, dir, "test", testCACert, testCAKey) - - // Creates a pkiCertificateReadWriter - pkiReadWriter := newPKICertificateReadWriter(dir, "test") - - // Reads the certificate - readCert, err := pkiReadWriter.Read() - if err != nil { - t.Fatalf("couldn't read certificate: %v", err) - } - - // Check if the certificate read from disk is equal to the original one - if !cert.Equal(readCert) { - t.Errorf("read cert does not match with expected cert") - } - - // Create a new cert - newCert, newkey, err := pkiutil.NewCertAndKey(testCACert, testCAKey, testCertCfg) - if err != nil { - t.Fatalf("couldn't generate certificate: %v", err) - } - - // Writes the new certificate - err = pkiReadWriter.Write(newCert, newkey) - if err != nil { - t.Fatalf("couldn't write new certificate: %v", err) - } - - // Reads back the new certificate - readCert, err = pkiReadWriter.Read() - if err != nil { - t.Fatalf("couldn't read new certificate: %v", err) - } - - // Check if the new certificate read from disk is equal to the original one - if !newCert.Equal(readCert) { - t.Error("read cert does not match with expected new cert") - } -} - -func TestKubeconfigReadWriter(t *testing.T) { - // creates tmp folders - dirKubernetes := testutil.SetupTempDir(t) - defer os.RemoveAll(dirKubernetes) - dirPKI := testutil.SetupTempDir(t) - defer os.RemoveAll(dirPKI) - - // write the CA cert and key to the temporary PKI dir - caName := kubeadmconstants.CACertAndKeyBaseName - if err := pkiutil.WriteCertAndKey( - dirPKI, - caName, - testCACert, - testCAKey); err != nil { - t.Fatalf("couldn't write out certificate %s to %s", caName, dirPKI) - } - - // creates a certificate and then embeds it into a kubeconfig file - cert := writeTestKubeconfig(t, dirKubernetes, "test", testCACert, testCAKey) - - // Creates a KubeconfigReadWriter - kubeconfigReadWriter := newKubeconfigReadWriter(dirKubernetes, "test", dirPKI, caName) - - // Reads the certificate embedded in a kubeconfig - readCert, err := kubeconfigReadWriter.Read() - if err != nil { - t.Fatalf("couldn't read embedded certificate: %v", err) - } - - // Check if the certificate read from disk is equal to the original one - if !cert.Equal(readCert) { - t.Errorf("read cert does not match with expected cert") - } - - // Create a new cert - newCert, newkey, err := pkiutil.NewCertAndKey(testCACert, testCAKey, testCertCfg) - if err != nil { - t.Fatalf("couldn't generate certificate: %v", err) - } - - // Writes the new certificate embedded in a kubeconfig - err = kubeconfigReadWriter.Write(newCert, newkey) - if err != nil { - t.Fatalf("couldn't write new embedded certificate: %v", err) - } - - // Reads back the new certificate embedded in a kubeconfig writer - readCert, err = kubeconfigReadWriter.Read() - if err != nil { - t.Fatalf("couldn't read new embedded certificate: %v", err) - } - - // Check if the new certificate read from disk is equal to the original one - if !newCert.Equal(readCert) { - t.Errorf("read cert does not match with expected new cert") - } -} - -// writeTestCertificate is a utility for creating a test certificate -func writeTestCertificate(t *testing.T, dir, name string, caCert *x509.Certificate, caKey crypto.Signer) *x509.Certificate { - cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, testCertCfg) - if err != nil { - t.Fatalf("couldn't generate certificate: %v", err) - } - - if err := pkiutil.WriteCertAndKey(dir, name, cert, key); err != nil { - t.Fatalf("couldn't write out certificate %s to %s", name, dir) - } - - return cert -} - -// writeTestKubeconfig is a utility for creating a test kubeconfig with an embedded certificate -func writeTestKubeconfig(t *testing.T, dir, name string, caCert *x509.Certificate, caKey crypto.Signer) *x509.Certificate { - - cfg := &pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "test-common-name", - Organization: []string{"sig-cluster-lifecycle"}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - AltNames: certutil.AltNames{ - IPs: []net.IP{net.ParseIP("10.100.0.1")}, - DNSNames: []string{"test-domain.space"}, - }, - }, - } - cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg) - if err != nil { - t.Fatalf("couldn't generate certificate: %v", err) - } - - encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(key) - if err != nil { - t.Fatalf("failed to marshal private key to PEM: %v", err) - } - - certificateAuthorityData := pkiutil.EncodeCertPEM(caCert) - - config := kubeconfigutil.CreateWithCerts( - "https://localhost:1234", - "kubernetes-test", - "user-test", - certificateAuthorityData, - encodedClientKey, - pkiutil.EncodeCertPEM(cert), - ) - - if err := clientcmd.WriteToFile(*config, filepath.Join(dir, name)); err != nil { - t.Fatalf("couldn't write out certificate") - } - - return cert -} diff --git a/cmd/kubeadm/app/phases/controlplane/BUILD b/cmd/kubeadm/app/phases/controlplane/BUILD deleted file mode 100644 index 373edab9c3a28..0000000000000 --- a/cmd/kubeadm/app/phases/controlplane/BUILD +++ /dev/null @@ -1,63 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = [ - "manifests_test.go", - "volumes_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "manifests.go", - "volumes.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go deleted file mode 100644 index 8181bea63a458..0000000000000 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ /dev/null @@ -1,354 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controlplane - -import ( - "fmt" - "net" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" - utilsnet "k8s.io/utils/net" -) - -// CreateInitStaticPodManifestFiles will write all static pod manifest files needed to bring up the control plane. -func CreateInitStaticPodManifestFiles(manifestDir, patchesDir string, cfg *kubeadmapi.InitConfiguration) error { - klog.V(1).Infoln("[control-plane] creating static Pod files") - return CreateStaticPodFiles(manifestDir, patchesDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler) -} - -// GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current configuration -// NB. this methods holds the information about how kubeadm creates static pod manifests. -func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) map[string]v1.Pod { - // Get the required hostpath mounts - mounts := getHostPathVolumesForTheControlPlane(cfg) - - // Prepare static pod specs - staticPodSpecs := map[string]v1.Pod{ - kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.KubeAPIServer, - Image: images.GetKubernetesImage(kubeadmconstants.KubeAPIServer, cfg), - ImagePullPolicy: v1.PullIfNotPresent, - Command: getAPIServerCommand(cfg, endpoint), - VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), - LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS), - ReadinessProbe: staticpodutil.ReadinessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/readyz", int(endpoint.BindPort), v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/livez", int(endpoint.BindPort), v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("250m"), - Env: kubeadmutil.GetProxyEnvVars(), - }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer), - map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}), - kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.KubeControllerManager, - Image: images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, cfg), - ImagePullPolicy: v1.PullIfNotPresent, - Command: getControllerManagerCommand(cfg), - VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), - LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("200m"), - Env: kubeadmutil.GetProxyEnvVars(), - }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil), - kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.KubeScheduler, - Image: images.GetKubernetesImage(kubeadmconstants.KubeScheduler, cfg), - ImagePullPolicy: v1.PullIfNotPresent, - Command: getSchedulerCommand(cfg), - VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), - LivenessProbe: staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS), - StartupProbe: staticpodutil.StartupProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS, cfg.APIServer.TimeoutForControlPlane), - Resources: staticpodutil.ComponentResources("100m"), - Env: kubeadmutil.GetProxyEnvVars(), - }, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil), - } - return staticPodSpecs -} - -// CreateStaticPodFiles creates all the requested static pod files. -func CreateStaticPodFiles(manifestDir, patchesDir string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, componentNames ...string) error { - // gets the StaticPodSpecs, actualized for the current ClusterConfiguration - klog.V(1).Infoln("[control-plane] getting StaticPodSpecs") - specs := GetStaticPodSpecs(cfg, endpoint) - - // creates required static pod specs - for _, componentName := range componentNames { - // retrieves the StaticPodSpec for given component - spec, exists := specs[componentName] - if !exists { - return errors.Errorf("couldn't retrieve StaticPodSpec for %q", componentName) - } - - // print all volumes that are mounted - for _, v := range spec.Spec.Volumes { - klog.V(2).Infof("[control-plane] adding volume %q for component %q", v.Name, componentName) - } - - // if patchesDir is defined, patch the static Pod manifest - if patchesDir != "" { - patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) - if err != nil { - return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", componentName) - } - spec = *patchedSpec - } - - // writes the StaticPodSpec to disk - if err := staticpodutil.WriteStaticPodToDisk(componentName, manifestDir, spec); err != nil { - return errors.Wrapf(err, "failed to create static pod manifest file for %q", componentName) - } - - klog.V(1).Infof("[control-plane] wrote static Pod manifest for component %q to %q\n", componentName, kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir)) - } - - return nil -} - -// getAPIServerCommand builds the right API server command from the given config object and version -func getAPIServerCommand(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint) []string { - defaultArguments := map[string]string{ - "advertise-address": localAPIEndpoint.AdvertiseAddress, - "insecure-port": "0", - "enable-admission-plugins": "NodeRestriction", - "service-cluster-ip-range": cfg.Networking.ServiceSubnet, - "service-account-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPublicKeyName), - "service-account-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName), - "service-account-issuer": fmt.Sprintf("https://kubernetes.default.svc.%s", cfg.Networking.DNSDomain), - "client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName), - "tls-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerCertName), - "tls-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKeyName), - "kubelet-client-certificate": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientCertName), - "kubelet-client-key": filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerKubeletClientKeyName), - "enable-bootstrap-token-auth": "true", - "secure-port": fmt.Sprintf("%d", localAPIEndpoint.BindPort), - "allow-privileged": "true", - "kubelet-preferred-address-types": "InternalIP,ExternalIP,Hostname", - // add options to configure the front proxy. Without the generated client cert, this will never be useable - // so add it unconditionally with recommended values - "requestheader-username-headers": "X-Remote-User", - "requestheader-group-headers": "X-Remote-Group", - "requestheader-extra-headers-prefix": "X-Remote-Extra-", - "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), - "requestheader-allowed-names": "front-proxy-client", - "proxy-client-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientCertName), - "proxy-client-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyClientKeyName), - } - - command := []string{"kube-apiserver"} - - // If the user set endpoints for an external etcd cluster - if cfg.Etcd.External != nil { - defaultArguments["etcd-servers"] = strings.Join(cfg.Etcd.External.Endpoints, ",") - - // Use any user supplied etcd certificates - if cfg.Etcd.External.CAFile != "" { - defaultArguments["etcd-cafile"] = cfg.Etcd.External.CAFile - } - if cfg.Etcd.External.CertFile != "" && cfg.Etcd.External.KeyFile != "" { - defaultArguments["etcd-certfile"] = cfg.Etcd.External.CertFile - defaultArguments["etcd-keyfile"] = cfg.Etcd.External.KeyFile - } - } else { - // Default to etcd static pod on localhost - // localhost IP family should be the same that the AdvertiseAddress - etcdLocalhostAddress := "127.0.0.1" - if utilsnet.IsIPv6String(localAPIEndpoint.AdvertiseAddress) { - etcdLocalhostAddress = "::1" - } - defaultArguments["etcd-servers"] = fmt.Sprintf("https://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdListenClientPort))) - defaultArguments["etcd-cafile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName) - defaultArguments["etcd-certfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientCertName) - defaultArguments["etcd-keyfile"] = filepath.Join(cfg.CertificatesDir, kubeadmconstants.APIServerEtcdClientKeyName) - - // Apply user configurations for local etcd - if cfg.Etcd.Local != nil { - if value, ok := cfg.Etcd.Local.ExtraArgs["advertise-client-urls"]; ok { - defaultArguments["etcd-servers"] = value - } - } - } - - // TODO: The following code should be removed after dual-stack is GA. - // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. - if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present { - defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled) - } - - if cfg.APIServer.ExtraArgs == nil { - cfg.APIServer.ExtraArgs = map[string]string{} - } - cfg.APIServer.ExtraArgs["authorization-mode"] = getAuthzModes(cfg.APIServer.ExtraArgs["authorization-mode"]) - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.APIServer.ExtraArgs)...) - - return command -} - -// getAuthzModes gets the authorization-related parameters to the api server -// Node,RBAC is the default mode if nothing is passed to kubeadm. User provided modes override -// the default. -func getAuthzModes(authzModeExtraArgs string) string { - defaultMode := []string{ - kubeadmconstants.ModeNode, - kubeadmconstants.ModeRBAC, - } - - if len(authzModeExtraArgs) > 0 { - mode := []string{} - for _, requested := range strings.Split(authzModeExtraArgs, ",") { - if isValidAuthzMode(requested) { - mode = append(mode, requested) - } else { - klog.Warningf("ignoring unknown kube-apiserver authorization-mode %q", requested) - } - } - - // only return the user provided mode if at least one was valid - if len(mode) > 0 { - if !compareAuthzModes(defaultMode, mode) { - klog.Warningf("the default kube-apiserver authorization-mode is %q; using %q", - strings.Join(defaultMode, ","), - strings.Join(mode, ","), - ) - } - return strings.Join(mode, ",") - } - } - return strings.Join(defaultMode, ",") -} - -// compareAuthzModes compares two given authz modes and returns false if they do not match -func compareAuthzModes(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i, m := range a { - if m != b[i] { - return false - } - } - return true -} - -func isValidAuthzMode(authzMode string) bool { - allModes := []string{ - kubeadmconstants.ModeNode, - kubeadmconstants.ModeRBAC, - kubeadmconstants.ModeWebhook, - kubeadmconstants.ModeABAC, - kubeadmconstants.ModeAlwaysAllow, - kubeadmconstants.ModeAlwaysDeny, - } - - for _, mode := range allModes { - if authzMode == mode { - return true - } - } - return false -} - -// getControllerManagerCommand builds the right controller manager command from the given config object and version -func getControllerManagerCommand(cfg *kubeadmapi.ClusterConfiguration) []string { - - kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) - caFile := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CACertName) - - defaultArguments := map[string]string{ - "port": "0", - "bind-address": "127.0.0.1", - "leader-elect": "true", - "kubeconfig": kubeconfigFile, - "authentication-kubeconfig": kubeconfigFile, - "authorization-kubeconfig": kubeconfigFile, - "client-ca-file": caFile, - "requestheader-client-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.FrontProxyCACertName), - "root-ca-file": caFile, - "service-account-private-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.ServiceAccountPrivateKeyName), - "cluster-signing-cert-file": caFile, - "cluster-signing-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName), - "use-service-account-credentials": "true", - "controllers": "*,bootstrapsigner,tokencleaner", - } - - // If using external CA, pass empty string to controller manager instead of ca.key/ca.crt path, - // so that the csrsigning controller fails to start - if res, _ := certphase.UsingExternalCA(cfg); res { - defaultArguments["cluster-signing-key-file"] = "" - defaultArguments["cluster-signing-cert-file"] = "" - } - - // Let the controller-manager allocate Node CIDRs for the Pod network. - // Each node will get a subspace of the address CIDR provided with --pod-network-cidr. - if cfg.Networking.PodSubnet != "" { - defaultArguments["allocate-node-cidrs"] = "true" - defaultArguments["cluster-cidr"] = cfg.Networking.PodSubnet - if cfg.Networking.ServiceSubnet != "" { - defaultArguments["service-cluster-ip-range"] = cfg.Networking.ServiceSubnet - } - } - - // Set cluster name - if cfg.ClusterName != "" { - defaultArguments["cluster-name"] = cfg.ClusterName - } - - // TODO: The following code should be remvoved after dual-stack is GA. - // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. - enabled, present := cfg.FeatureGates[features.IPv6DualStack] - if present { - defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled) - } - - command := []string{"kube-controller-manager"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.ControllerManager.ExtraArgs)...) - - return command -} - -// getSchedulerCommand builds the right scheduler command from the given config object and version -func getSchedulerCommand(cfg *kubeadmapi.ClusterConfiguration) []string { - kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) - defaultArguments := map[string]string{ - "port": "0", - "bind-address": "127.0.0.1", - "leader-elect": "true", - "kubeconfig": kubeconfigFile, - "authentication-kubeconfig": kubeconfigFile, - "authorization-kubeconfig": kubeconfigFile, - } - - // TODO: The following code should be remvoved after dual-stack is GA. - // Note: The user still retains the ability to explicitly set feature-gates and that value will overwrite this base value. - if enabled, present := cfg.FeatureGates[features.IPv6DualStack]; present { - defaultArguments["feature-gates"] = fmt.Sprintf("%s=%t", features.IPv6DualStack, enabled) - } - - command := []string{"kube-scheduler"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Scheduler.ExtraArgs)...) - return command -} diff --git a/cmd/kubeadm/app/phases/controlplane/manifests_test.go b/cmd/kubeadm/app/phases/controlplane/manifests_test.go deleted file mode 100644 index 0a443670d16cd..0000000000000 --- a/cmd/kubeadm/app/phases/controlplane/manifests_test.go +++ /dev/null @@ -1,1117 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controlplane - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "sort" - "strings" - "testing" - - "github.com/lithammer/dedent" - - "k8s.io/apimachinery/pkg/util/sets" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -const ( - testCertsDir = "/var/lib/certs" -) - -var cpVersion = kubeadmconstants.MinimumControlPlaneVersion.WithPreRelease("beta.2").String() - -func TestGetStaticPodSpecs(t *testing.T) { - - // Creates a Cluster Configuration - cfg := &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.9.0", - } - - // Executes GetStaticPodSpecs - specs := GetStaticPodSpecs(cfg, &kubeadmapi.APIEndpoint{}) - - var tests = []struct { - name string - staticPodName string - }{ - { - name: "KubeAPIServer", - staticPodName: kubeadmconstants.KubeAPIServer, - }, - { - name: "KubeControllerManager", - staticPodName: kubeadmconstants.KubeControllerManager, - }, - { - name: "KubeScheduler", - staticPodName: kubeadmconstants.KubeScheduler, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - // assert the spec for the staticPodName exists - if spec, ok := specs[tc.staticPodName]; ok { - - // Assert each specs refers to the right pod - if spec.Spec.Containers[0].Name != tc.staticPodName { - t.Errorf("getKubeConfigSpecs spec for %s contains pod %s, expects %s", tc.staticPodName, spec.Spec.Containers[0].Name, tc.staticPodName) - } - - } else { - t.Errorf("getStaticPodSpecs didn't create spec for %s ", tc.staticPodName) - } - }) - } -} - -func TestCreateStaticPodFilesAndWrappers(t *testing.T) { - - var tests = []struct { - name string - components []string - }{ - { - name: "KubeAPIServer KubeAPIServer KubeScheduler", - components: []string{kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeScheduler}, - }, - { - name: "KubeAPIServer", - components: []string{kubeadmconstants.KubeAPIServer}, - }, - { - name: "KubeControllerManager", - components: []string{kubeadmconstants.KubeControllerManager}, - }, - { - name: "KubeScheduler", - components: []string{kubeadmconstants.KubeScheduler}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Creates a Cluster Configuration - cfg := &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.9.0", - } - - // Execute createStaticPodFunction - manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) - err := CreateStaticPodFiles(manifestPath, "", cfg, &kubeadmapi.APIEndpoint{}, test.components...) - if err != nil { - t.Errorf("Error executing createStaticPodFunction: %v", err) - return - } - - // Assert expected files are there - testutil.AssertFilesCount(t, manifestPath, len(test.components)) - - for _, fileName := range test.components { - testutil.AssertFileExists(t, manifestPath, fileName+".yaml") - } - }) - } -} - -func TestCreateStaticPodFilesWithPatches(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Creates a Cluster Configuration - cfg := &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.9.0", - } - - patchesPath := filepath.Join(tmpdir, "patch-files") - err := os.MkdirAll(patchesPath, 0777) - if err != nil { - t.Fatalf("Couldn't create %s", patchesPath) - } - - patchString := dedent.Dedent(` - metadata: - annotations: - patched: "true" - `) - - err = ioutil.WriteFile(filepath.Join(patchesPath, kubeadmconstants.KubeAPIServer+".yaml"), []byte(patchString), 0644) - if err != nil { - t.Fatalf("WriteFile returned unexpected error: %v", err) - } - - // Execute createStaticPodFunction with patches - manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) - err = CreateStaticPodFiles(manifestPath, patchesPath, cfg, &kubeadmapi.APIEndpoint{}, kubeadmconstants.KubeAPIServer) - if err != nil { - t.Errorf("Error executing createStaticPodFunction: %v", err) - return - } - - pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, fmt.Sprintf("%s.yaml", kubeadmconstants.KubeAPIServer))) - if err != nil { - t.Errorf("Error executing ReadStaticPodFromDisk: %v", err) - return - } - - if _, ok := pod.ObjectMeta.Annotations["patched"]; !ok { - t.Errorf("Patches were not applied to %s", kubeadmconstants.KubeAPIServer) - } -} - -func TestGetAPIServerCommand(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - endpoint *kubeadmapi.APIEndpoint - expected []string - }{ - { - name: "testing defaults", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=1.2.3.4", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - }, - }, - { - name: "ipv6 advertise address", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=2001:db8::1", - fmt.Sprintf("--etcd-servers=https://[::1]:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - }, - }, - { - name: "an external etcd with custom ca, certs and keys", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"https://[2001:abcd:bcda::1]:2379", "https://[2001:abcd:bcda::2]:2379"}, - CAFile: "fuz", - CertFile: "fiz", - KeyFile: "faz", - }, - }, - CertificatesDir: testCertsDir, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--enable-bootstrap-token-auth=true", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=2001:db8::1", - "--etcd-servers=https://[2001:abcd:bcda::1]:2379,https://[2001:abcd:bcda::2]:2379", - "--etcd-cafile=fuz", - "--etcd-certfile=fiz", - "--etcd-keyfile=faz", - }, - }, - { - name: "an insecure etcd", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"http://[::1]:2379", "http://[::1]:2380"}, - }, - }, - CertificatesDir: testCertsDir, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "2001:db8::1"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - fmt.Sprintf("--secure-port=%d", 123), - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--enable-bootstrap-token-auth=true", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=2001:db8::1", - "--etcd-servers=http://[::1]:2379,http://[::1]:2380", - }, - }, - { - name: "test APIServer.ExtraArgs works as expected", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - APIServer: kubeadmapi.APIServer{ - ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "service-cluster-ip-range": "baz", - "advertise-address": "9.9.9.9", - "audit-policy-file": "/etc/config/audit.yaml", - "audit-log-path": "/var/log/kubernetes", - }, - }, - }, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=baz", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=9.9.9.9", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - "--audit-policy-file=/etc/config/audit.yaml", - "--audit-log-path=/var/log/kubernetes", - }, - }, - { - name: "authorization-mode extra-args ABAC", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - APIServer: kubeadmapi.APIServer{ - ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "authorization-mode": kubeadmconstants.ModeABAC, - }, - }, - }, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=ABAC", - "--advertise-address=1.2.3.4", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - }, - }, - { - name: "insecure-port extra-args", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - APIServer: kubeadmapi.APIServer{ - ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "insecure-port": "1234", - }, - }, - }, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=1234", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC", - "--advertise-address=1.2.3.4", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - }, - }, - { - name: "authorization-mode extra-args Webhook", - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ServiceSubnet: "bar", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - APIServer: kubeadmapi.APIServer{ - ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - "authorization-mode": strings.Join([]string{ - kubeadmconstants.ModeNode, - kubeadmconstants.ModeRBAC, - kubeadmconstants.ModeWebhook, - }, ","), - }, - }, - }, - }, - endpoint: &kubeadmapi.APIEndpoint{BindPort: 123, AdvertiseAddress: "1.2.3.4"}, - expected: []string{ - "kube-apiserver", - "--insecure-port=0", - "--enable-admission-plugins=NodeRestriction", - "--service-cluster-ip-range=bar", - "--service-account-key-file=" + testCertsDir + "/sa.pub", - "--service-account-signing-key-file=" + testCertsDir + "/sa.key", - "--service-account-issuer=https://kubernetes.default.svc.cluster.local", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--tls-cert-file=" + testCertsDir + "/apiserver.crt", - "--tls-private-key-file=" + testCertsDir + "/apiserver.key", - "--kubelet-client-certificate=" + testCertsDir + "/apiserver-kubelet-client.crt", - "--kubelet-client-key=" + testCertsDir + "/apiserver-kubelet-client.key", - "--enable-bootstrap-token-auth=true", - "--secure-port=123", - "--allow-privileged=true", - "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname", - "--proxy-client-cert-file=/var/lib/certs/front-proxy-client.crt", - "--proxy-client-key-file=/var/lib/certs/front-proxy-client.key", - "--requestheader-username-headers=X-Remote-User", - "--requestheader-group-headers=X-Remote-Group", - "--requestheader-extra-headers-prefix=X-Remote-Extra-", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--requestheader-allowed-names=front-proxy-client", - "--authorization-mode=Node,RBAC,Webhook", - "--advertise-address=1.2.3.4", - fmt.Sprintf("--etcd-servers=https://127.0.0.1:%d", kubeadmconstants.EtcdListenClientPort), - "--etcd-cafile=" + testCertsDir + "/etcd/ca.crt", - "--etcd-certfile=" + testCertsDir + "/apiserver-etcd-client.crt", - "--etcd-keyfile=" + testCertsDir + "/apiserver-etcd-client.key", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := getAPIServerCommand(rt.cfg, rt.endpoint) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - errorDiffArguments(t, rt.name, actual, rt.expected) - } - }) - } -} - -func errorDiffArguments(t *testing.T, name string, actual, expected []string) { - expectedShort := removeCommon(expected, actual) - actualShort := removeCommon(actual, expected) - t.Errorf( - "[%s] failed getAPIServerCommand:\nexpected:\n%v\nsaw:\n%v"+ - "\nexpectedShort:\n%v\nsawShort:\n%v\n", - name, expected, actual, - expectedShort, actualShort) -} - -// removeCommon removes common items from left list -// makes compairing two cmdline (with lots of arguments) easier -func removeCommon(left, right []string) []string { - origSet := sets.NewString(left...) - origSet.Delete(right...) - return origSet.List() -} - -func TestGetControllerManagerCommand(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - expected []string - }{ - { - name: "custom cluster name for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: cpVersion, - CertificatesDir: testCertsDir, - ClusterName: "some-other-cluster-name", - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--cluster-name=some-other-cluster-name", - }, - }, - { - name: "custom certs dir for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - }, - }, - { - name: "custom cluster-cidr for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16", DNSDomain: "cluster.local"}, - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--allocate-node-cidrs=true", - "--cluster-cidr=10.0.1.15/16", - }, - }, - { - name: "custom service-cluster-ip-range for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - PodSubnet: "10.0.1.15/16", - ServiceSubnet: "172.20.0.0/24", - DNSDomain: "cluster.local", - }, - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--allocate-node-cidrs=true", - "--cluster-cidr=10.0.1.15/16", - "--service-cluster-ip-range=172.20.0.0/24", - }, - }, - { - name: "custom extra-args for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{PodSubnet: "10.0.1.15/16", DNSDomain: "cluster.local"}, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size": "20"}, - }, - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--allocate-node-cidrs=true", - "--cluster-cidr=10.0.1.15/16", - "--node-cidr-mask-size=20", - }, - }, - { - name: "custom IPv6 networking for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - PodSubnet: "2001:db8::/64", - ServiceSubnet: "fd03::/112", - DNSDomain: "cluster.local", - }, - - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--allocate-node-cidrs=true", - "--cluster-cidr=2001:db8::/64", - "--service-cluster-ip-range=fd03::/112", - }, - }, - { - name: "dual-stack networking for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - PodSubnet: "2001:db8::/64,10.1.0.0/16", - ServiceSubnet: "fd03::/112,192.168.0.0/16", - DNSDomain: "cluster.local", - }, - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--feature-gates=IPv6DualStack=true", - "--allocate-node-cidrs=true", - "--cluster-cidr=2001:db8::/64,10.1.0.0/16", - "--service-cluster-ip-range=fd03::/112,192.168.0.0/16", - }, - }, - { - name: "dual-stack networking custom extra-args for " + cpVersion, - cfg: &kubeadmapi.ClusterConfiguration{ - Networking: kubeadmapi.Networking{ - PodSubnet: "10.0.1.15/16,2001:db8::/64", - DNSDomain: "cluster.local", - }, - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{"node-cidr-mask-size-ipv4": "20", "node-cidr-mask-size-ipv6": "80"}, - }, - CertificatesDir: testCertsDir, - KubernetesVersion: cpVersion, - FeatureGates: map[string]bool{features.IPv6DualStack: true}, - }, - expected: []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + testCertsDir + "/ca.crt", - "--service-account-private-key-file=" + testCertsDir + "/sa.key", - "--cluster-signing-cert-file=" + testCertsDir + "/ca.crt", - "--cluster-signing-key-file=" + testCertsDir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + testCertsDir + "/ca.crt", - "--requestheader-client-ca-file=" + testCertsDir + "/front-proxy-ca.crt", - "--feature-gates=IPv6DualStack=true", - "--allocate-node-cidrs=true", - "--cluster-cidr=10.0.1.15/16,2001:db8::/64", - "--node-cidr-mask-size-ipv4=20", - "--node-cidr-mask-size-ipv6=80", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := getControllerManagerCommand(rt.cfg) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - errorDiffArguments(t, rt.name, actual, rt.expected) - } - }) - } -} - -func TestGetControllerManagerCommandExternalCA(t *testing.T) { - tests := []struct { - name string - cfg *kubeadmapi.InitConfiguration - caKeyPresent bool - expectedArgFunc func(dir string) []string - }{ - { - name: "caKeyPresent-false for " + cpVersion, - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - KubernetesVersion: cpVersion, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - }, - }, - caKeyPresent: false, - expectedArgFunc: func(tmpdir string) []string { - return []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + tmpdir + "/ca.crt", - "--service-account-private-key-file=" + tmpdir + "/sa.key", - "--cluster-signing-cert-file=", - "--cluster-signing-key-file=", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + tmpdir + "/ca.crt", - "--requestheader-client-ca-file=" + tmpdir + "/front-proxy-ca.crt", - } - }, - }, - { - name: "caKeyPresent true for " + cpVersion, - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - KubernetesVersion: cpVersion, - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - }, - }, - caKeyPresent: true, - expectedArgFunc: func(tmpdir string) []string { - return []string{ - "kube-controller-manager", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--root-ca-file=" + tmpdir + "/ca.crt", - "--service-account-private-key-file=" + tmpdir + "/sa.key", - "--cluster-signing-cert-file=" + tmpdir + "/ca.crt", - "--cluster-signing-key-file=" + tmpdir + "/ca.key", - "--use-service-account-credentials=true", - "--controllers=*,bootstrapsigner,tokencleaner", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/controller-manager.conf", - "--client-ca-file=" + tmpdir + "/ca.crt", - "--requestheader-client-ca-file=" + tmpdir + "/front-proxy-ca.crt", - } - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - test.cfg.CertificatesDir = tmpdir - - if err := certs.CreatePKIAssets(test.cfg); err != nil { - t.Errorf("failed creating pki assets: %v", err) - } - - // delete ca.key and front-proxy-ca.key if test.caKeyPresent is false - if !test.caKeyPresent { - if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, kubeadmconstants.CAKeyName)); err != nil { - t.Errorf("failed removing %s: %v", kubeadmconstants.CAKeyName, err) - } - if err := os.Remove(filepath.Join(test.cfg.CertificatesDir, kubeadmconstants.FrontProxyCAKeyName)); err != nil { - t.Errorf("failed removing %s: %v", kubeadmconstants.FrontProxyCAKeyName, err) - } - } - - actual := getControllerManagerCommand(&test.cfg.ClusterConfiguration) - expected := test.expectedArgFunc(tmpdir) - sort.Strings(actual) - sort.Strings(expected) - if !reflect.DeepEqual(actual, expected) { - errorDiffArguments(t, test.name, actual, expected) - } - }) - } -} - -func TestGetSchedulerCommand(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - expected []string - }{ - { - name: "scheduler defaults", - cfg: &kubeadmapi.ClusterConfiguration{}, - expected: []string{ - "kube-scheduler", - "--port=0", - "--bind-address=127.0.0.1", - "--leader-elect=true", - "--kubeconfig=" + kubeadmconstants.KubernetesDir + "/scheduler.conf", - "--authentication-kubeconfig=" + kubeadmconstants.KubernetesDir + "/scheduler.conf", - "--authorization-kubeconfig=" + kubeadmconstants.KubernetesDir + "/scheduler.conf", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := getSchedulerCommand(rt.cfg) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - errorDiffArguments(t, rt.name, actual, rt.expected) - } - }) - } -} - -func TestGetAuthzModes(t *testing.T) { - var tests = []struct { - name string - authMode []string - expected string - }{ - { - name: "default if empty", - authMode: []string{}, - expected: "Node,RBAC", - }, - { - name: "default non empty", - authMode: []string{kubeadmconstants.ModeNode, kubeadmconstants.ModeRBAC}, - expected: "Node,RBAC", - }, - { - name: "single unspecified returning default", - authMode: []string{"FooAuthzMode"}, - expected: "Node,RBAC", - }, - { - name: "multiple ignored", - authMode: []string{kubeadmconstants.ModeNode, "foo", kubeadmconstants.ModeRBAC, "bar"}, - expected: "Node,RBAC", - }, - { - name: "single mode", - authMode: []string{kubeadmconstants.ModeAlwaysDeny}, - expected: "AlwaysDeny", - }, - { - name: "multiple special order", - authMode: []string{kubeadmconstants.ModeNode, kubeadmconstants.ModeWebhook, kubeadmconstants.ModeRBAC, kubeadmconstants.ModeABAC}, - expected: "Node,Webhook,RBAC,ABAC", - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := getAuthzModes(strings.Join(rt.authMode, ",")) - if actual != rt.expected { - t.Errorf("failed getAuthzModes:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) - } - }) - } -} - -func TestIsValidAuthzMode(t *testing.T) { - var tests = []struct { - mode string - valid bool - }{ - { - mode: "Node", - valid: true, - }, - { - mode: "RBAC", - valid: true, - }, - { - mode: "ABAC", - valid: true, - }, - { - mode: "AlwaysAllow", - valid: true, - }, - { - mode: "Webhook", - valid: true, - }, - { - mode: "AlwaysDeny", - valid: true, - }, - { - mode: "Foo", - valid: false, - }, - } - - for _, rt := range tests { - t.Run(rt.mode, func(t *testing.T) { - isValid := isValidAuthzMode(rt.mode) - if isValid != rt.valid { - t.Errorf("failed isValidAuthzMode:\nexpected:\n%v\nsaw:\n%v", rt.valid, isValid) - } - }) - } -} - -func TestCompareAuthzModes(t *testing.T) { - var tests = []struct { - name string - modesA []string - modesB []string - equal bool - }{ - { - name: "modes match", - modesA: []string{"a", "b", "c"}, - modesB: []string{"a", "b", "c"}, - equal: true, - }, - { - name: "modes order does not match", - modesA: []string{"a", "c", "b"}, - modesB: []string{"a", "b", "c"}, - }, - { - name: "modes do not match; A has less modes", - modesA: []string{"a", "b"}, - modesB: []string{"a", "b", "c"}, - }, - { - name: "modes do not match; B has less modes", - modesA: []string{"a", "b", "c"}, - modesB: []string{"a", "b"}, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - equal := compareAuthzModes(rt.modesA, rt.modesB) - if equal != rt.equal { - t.Errorf("failed compareAuthzModes:\nexpected:\n%v\nsaw:\n%v", rt.equal, equal) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/controlplane/volumes.go b/cmd/kubeadm/app/phases/controlplane/volumes.go deleted file mode 100644 index 7f2f7ceec5215..0000000000000 --- a/cmd/kubeadm/app/phases/controlplane/volumes.go +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controlplane - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" -) - -const ( - caCertsVolumeName = "ca-certs" - caCertsVolumePath = "/etc/ssl/certs" - flexvolumeDirVolumeName = "flexvolume-dir" - defaultFlexvolumeDirVolumePath = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" -) - -// caCertsExtraVolumePaths specifies the paths that can be conditionally mounted into the apiserver and controller-manager containers -// as /etc/ssl/certs might be or contain a symlink to them. It's a variable since it may be changed in unit testing. This var MUST -// NOT be changed in normal codepaths during runtime. -var caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates", "/usr/local/share/ca-certificates", "/etc/ca-certificates"} - -// getHostPathVolumesForTheControlPlane gets the required hostPath volumes and mounts for the control plane -func getHostPathVolumesForTheControlPlane(cfg *kubeadmapi.ClusterConfiguration) controlPlaneHostPathMounts { - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - hostPathFileOrCreate := v1.HostPathFileOrCreate - mounts := newControlPlaneHostPathMounts() - - // HostPath volumes for the API Server - // Read-only mount for the certificates directory - // TODO: Always mount the K8s Certificates directory to a static path inside of the container - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate) - // Read-only mount for the ca certs (/etc/ssl/certs) directory - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate) - - // If external etcd is specified, mount the directories needed for accessing the CA/serving certs and the private key - if cfg.Etcd.External != nil { - etcdVols, etcdVolMounts := getEtcdCertVolumes(cfg.Etcd.External, cfg.CertificatesDir) - mounts.AddHostPathMounts(kubeadmconstants.KubeAPIServer, etcdVols, etcdVolMounts) - } - - // HostPath volumes for the controller manager - // Read-only mount for the certificates directory - // TODO: Always mount the K8s Certificates directory to a static path inside of the container - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeCertificatesVolumeName, cfg.CertificatesDir, cfg.CertificatesDir, true, &hostPathDirectoryOrCreate) - // Read-only mount for the ca certs (/etc/ssl/certs) directory - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsVolumeName, caCertsVolumePath, caCertsVolumePath, true, &hostPathDirectoryOrCreate) - // Read-only mount for the controller manager kubeconfig file - controllerManagerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ControllerManagerKubeConfigFileName) - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, kubeadmconstants.KubeConfigVolumeName, controllerManagerKubeConfigFile, controllerManagerKubeConfigFile, true, &hostPathFileOrCreate) - // Mount for the flexvolume directory (/usr/libexec/kubernetes/kubelet-plugins/volume/exec by default) - // Flexvolume dir must NOT be readonly as it is used for third-party plugins to integrate with their storage backends via unix domain socket. - flexvolumeDirVolumePath, ok := cfg.ControllerManager.ExtraArgs["flex-volume-plugin-dir"] - if !ok { - flexvolumeDirVolumePath = defaultFlexvolumeDirVolumePath - } - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, flexvolumeDirVolumeName, flexvolumeDirVolumePath, flexvolumeDirVolumePath, false, &hostPathDirectoryOrCreate) - - // HostPath volumes for the scheduler - // Read-only mount for the scheduler kubeconfig file - schedulerKubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.SchedulerKubeConfigFileName) - mounts.NewHostPathMount(kubeadmconstants.KubeScheduler, kubeadmconstants.KubeConfigVolumeName, schedulerKubeConfigFile, schedulerKubeConfigFile, true, &hostPathFileOrCreate) - - // On some systems were we host-mount /etc/ssl/certs, it is also required to mount additional directories. - // This is needed due to symlinks pointing from files in /etc/ssl/certs to these directories. - for _, caCertsExtraVolumePath := range caCertsExtraVolumePaths { - if isExtraVolumeMountNeeded(caCertsExtraVolumePath) { - caCertsExtraVolumeName := strings.Replace(caCertsExtraVolumePath, "/", "-", -1)[1:] - mounts.NewHostPathMount(kubeadmconstants.KubeAPIServer, caCertsExtraVolumeName, caCertsExtraVolumePath, caCertsExtraVolumePath, true, &hostPathDirectoryOrCreate) - mounts.NewHostPathMount(kubeadmconstants.KubeControllerManager, caCertsExtraVolumeName, caCertsExtraVolumePath, caCertsExtraVolumePath, true, &hostPathDirectoryOrCreate) - } - } - - // Merge user defined mounts and ensure unique volume and volume mount - // names - mounts.AddExtraHostPathMounts(kubeadmconstants.KubeAPIServer, cfg.APIServer.ExtraVolumes) - mounts.AddExtraHostPathMounts(kubeadmconstants.KubeControllerManager, cfg.ControllerManager.ExtraVolumes) - mounts.AddExtraHostPathMounts(kubeadmconstants.KubeScheduler, cfg.Scheduler.ExtraVolumes) - - return mounts -} - -// controlPlaneHostPathMounts is a helper struct for handling all the control plane's hostPath mounts in an easy way -type controlPlaneHostPathMounts struct { - // volumes is a nested map that forces a unique volumes. The outer map's - // keys are a string that should specify the target component to add the - // volume to. The values (inner map) of the outer map are maps with string - // keys and v1.Volume values. The inner map's key should specify the volume - // name. - volumes map[string]map[string]v1.Volume - // volumeMounts is a nested map that forces a unique volume mounts. The - // outer map's keys are a string that should specify the target component - // to add the volume mount to. The values (inner map) of the outer map are - // maps with string keys and v1.VolumeMount values. The inner map's key - // should specify the volume mount name. - volumeMounts map[string]map[string]v1.VolumeMount -} - -func newControlPlaneHostPathMounts() controlPlaneHostPathMounts { - return controlPlaneHostPathMounts{ - volumes: map[string]map[string]v1.Volume{}, - volumeMounts: map[string]map[string]v1.VolumeMount{}, - } -} - -func (c *controlPlaneHostPathMounts) NewHostPathMount(component, mountName, hostPath, containerPath string, readOnly bool, hostPathType *v1.HostPathType) { - vol := staticpodutil.NewVolume(mountName, hostPath, hostPathType) - c.addComponentVolume(component, vol) - volMount := staticpodutil.NewVolumeMount(mountName, containerPath, readOnly) - c.addComponentVolumeMount(component, volMount) -} - -func (c *controlPlaneHostPathMounts) AddHostPathMounts(component string, vols []v1.Volume, volMounts []v1.VolumeMount) { - for _, v := range vols { - c.addComponentVolume(component, v) - } - for _, v := range volMounts { - c.addComponentVolumeMount(component, v) - } -} - -// AddExtraHostPathMounts adds host path mounts and overwrites the default -// paths in the case that a user specifies the same volume/volume mount name. -func (c *controlPlaneHostPathMounts) AddExtraHostPathMounts(component string, extraVols []kubeadmapi.HostPathMount) { - for _, extraVol := range extraVols { - hostPathType := extraVol.PathType - c.NewHostPathMount(component, extraVol.Name, extraVol.HostPath, extraVol.MountPath, extraVol.ReadOnly, &hostPathType) - } -} - -func (c *controlPlaneHostPathMounts) GetVolumes(component string) map[string]v1.Volume { - return c.volumes[component] -} - -func (c *controlPlaneHostPathMounts) GetVolumeMounts(component string) map[string]v1.VolumeMount { - return c.volumeMounts[component] -} - -func (c *controlPlaneHostPathMounts) addComponentVolume(component string, vol v1.Volume) { - if _, ok := c.volumes[component]; !ok { - c.volumes[component] = map[string]v1.Volume{} - } - c.volumes[component][vol.Name] = vol -} - -func (c *controlPlaneHostPathMounts) addComponentVolumeMount(component string, volMount v1.VolumeMount) { - if _, ok := c.volumeMounts[component]; !ok { - c.volumeMounts[component] = map[string]v1.VolumeMount{} - } - c.volumeMounts[component][volMount.Name] = volMount -} - -// getEtcdCertVolumes returns the volumes/volumemounts needed for talking to an external etcd cluster -func getEtcdCertVolumes(etcdCfg *kubeadmapi.ExternalEtcd, k8sCertificatesDir string) ([]v1.Volume, []v1.VolumeMount) { - certPaths := []string{etcdCfg.CAFile, etcdCfg.CertFile, etcdCfg.KeyFile} - certDirs := sets.NewString() - for _, certPath := range certPaths { - certDir := filepath.Dir(certPath) - // Ignore ".", which is the result of passing an empty path. - // Also ignore the cert directories that already may be mounted; /etc/ssl/certs, /etc/pki or Kubernetes CertificatesDir - // If the etcd certs are in there, it's okay, we don't have to do anything - extraVolumePath := false - for _, caCertsExtraVolumePath := range caCertsExtraVolumePaths { - if strings.HasPrefix(certDir, caCertsExtraVolumePath) { - extraVolumePath = true - break - } - } - if certDir == "." || extraVolumePath || strings.HasPrefix(certDir, caCertsVolumePath) || strings.HasPrefix(certDir, k8sCertificatesDir) { - continue - } - // Filter out any existing hostpath mounts in the list that contains a subset of the path - alreadyExists := false - for _, existingCertDir := range certDirs.List() { - // If the current directory is a parent of an existing one, remove the already existing one - if strings.HasPrefix(existingCertDir, certDir) { - certDirs.Delete(existingCertDir) - } else if strings.HasPrefix(certDir, existingCertDir) { - // If an existing directory is a parent of the current one, don't add the current one - alreadyExists = true - } - } - if alreadyExists { - continue - } - certDirs.Insert(certDir) - } - - volumes := []v1.Volume{} - volumeMounts := []v1.VolumeMount{} - pathType := v1.HostPathDirectoryOrCreate - for i, certDir := range certDirs.List() { - name := fmt.Sprintf("etcd-certs-%d", i) - volumes = append(volumes, staticpodutil.NewVolume(name, certDir, &pathType)) - volumeMounts = append(volumeMounts, staticpodutil.NewVolumeMount(name, certDir, true)) - } - return volumes, volumeMounts -} - -// isExtraVolumeMountNeeded specifies whether /etc/pki should be host-mounted into the containers -// On some systems were we host-mount /etc/ssl/certs, it is also required to mount /etc/pki. This is needed -// due to symlinks pointing from files in /etc/ssl/certs into /etc/pki/ -func isExtraVolumeMountNeeded(caCertsExtraVolumePath string) bool { - if _, err := os.Stat(caCertsExtraVolumePath); err == nil { - return true - } - return false -} diff --git a/cmd/kubeadm/app/phases/controlplane/volumes_test.go b/cmd/kubeadm/app/phases/controlplane/volumes_test.go deleted file mode 100644 index da7253dcd1096..0000000000000 --- a/cmd/kubeadm/app/phases/controlplane/volumes_test.go +++ /dev/null @@ -1,643 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controlplane - -import ( - "fmt" - "io/ioutil" - "os" - "reflect" - "testing" - - "k8s.io/api/core/v1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestGetEtcdCertVolumes(t *testing.T) { - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - k8sCertificatesDir := "/etc/kubernetes/pki" - var tests = []struct { - name, ca, cert, key string - vol []v1.Volume - volMount []v1.VolumeMount - }{ - { - name: "Should ignore files in /etc/ssl/certs", - ca: "/etc/ssl/certs/my-etcd-ca.crt", - cert: "/etc/ssl/certs/my-etcd.crt", - key: "/etc/ssl/certs/my-etcd.key", - vol: []v1.Volume{}, - volMount: []v1.VolumeMount{}, - }, - { - name: "Should ignore files in subdirs of /etc/ssl/certs", - ca: "/etc/ssl/certs/etcd/my-etcd-ca.crt", - cert: "/etc/ssl/certs/etcd/my-etcd.crt", - key: "/etc/ssl/certs/etcd/my-etcd.key", - vol: []v1.Volume{}, - volMount: []v1.VolumeMount{}, - }, - { - name: "Should ignore files in /etc/pki", - ca: "/etc/pki/my-etcd-ca.crt", - cert: "/etc/pki/my-etcd.crt", - key: "/etc/pki/my-etcd.key", - vol: []v1.Volume{}, - volMount: []v1.VolumeMount{}, - }, - { - name: "Should ignore files in Kubernetes PKI directory (and subdirs)", - ca: k8sCertificatesDir + "/ca/my-etcd-ca.crt", - cert: k8sCertificatesDir + "/my-etcd.crt", - key: k8sCertificatesDir + "/my-etcd.key", - vol: []v1.Volume{}, - volMount: []v1.VolumeMount{}, - }, - { - name: "All certs are in the same dir", - ca: "/var/lib/certs/etcd/my-etcd-ca.crt", - cert: "/var/lib/certs/etcd/my-etcd.crt", - key: "/var/lib/certs/etcd/my-etcd.key", - vol: []v1.Volume{ - { - Name: "etcd-certs-0", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - volMount: []v1.VolumeMount{ - { - Name: "etcd-certs-0", - MountPath: "/var/lib/certs/etcd", - ReadOnly: true, - }, - }, - }, - { - name: "One file + two files in separate dirs", - ca: "/etc/certs/etcd/my-etcd-ca.crt", - cert: "/var/lib/certs/etcd/my-etcd.crt", - key: "/var/lib/certs/etcd/my-etcd.key", - vol: []v1.Volume{ - { - Name: "etcd-certs-0", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "etcd-certs-1", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - volMount: []v1.VolumeMount{ - { - Name: "etcd-certs-0", - MountPath: "/etc/certs/etcd", - ReadOnly: true, - }, - { - Name: "etcd-certs-1", - MountPath: "/var/lib/certs/etcd", - ReadOnly: true, - }, - }, - }, - { - name: "All three files in different directories", - ca: "/etc/certs/etcd/my-etcd-ca.crt", - cert: "/var/lib/certs/etcd/my-etcd.crt", - key: "/var/lib/certs/private/my-etcd.key", - vol: []v1.Volume{ - { - Name: "etcd-certs-0", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "etcd-certs-1", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "etcd-certs-2", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/certs/private", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - volMount: []v1.VolumeMount{ - { - Name: "etcd-certs-0", - MountPath: "/etc/certs/etcd", - ReadOnly: true, - }, - { - Name: "etcd-certs-1", - MountPath: "/var/lib/certs/etcd", - ReadOnly: true, - }, - { - Name: "etcd-certs-2", - MountPath: "/var/lib/certs/private", - ReadOnly: true, - }, - }, - }, - { - name: "The most top-level dir should be used", - ca: "/etc/certs/etcd/my-etcd-ca.crt", - cert: "/etc/certs/etcd/serving/my-etcd.crt", - key: "/etc/certs/etcd/serving/my-etcd.key", - vol: []v1.Volume{ - { - Name: "etcd-certs-0", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - volMount: []v1.VolumeMount{ - { - Name: "etcd-certs-0", - MountPath: "/etc/certs/etcd", - ReadOnly: true, - }, - }, - }, - { - name: "The most top-level dir should be used, regardless of order", - ca: "/etc/certs/etcd/ca/my-etcd-ca.crt", - cert: "/etc/certs/etcd/my-etcd.crt", - key: "/etc/certs/etcd/my-etcd.key", - vol: []v1.Volume{ - { - Name: "etcd-certs-0", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - volMount: []v1.VolumeMount{ - { - Name: "etcd-certs-0", - MountPath: "/etc/certs/etcd", - ReadOnly: true, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualVol, actualVolMount := getEtcdCertVolumes(&kubeadmapi.ExternalEtcd{ - CAFile: rt.ca, - CertFile: rt.cert, - KeyFile: rt.key, - }, k8sCertificatesDir) - if !reflect.DeepEqual(actualVol, rt.vol) { - t.Errorf( - "failed getEtcdCertVolumes:\n\texpected: %v\n\t actual: %v", - rt.vol, - actualVol, - ) - } - if !reflect.DeepEqual(actualVolMount, rt.volMount) { - t.Errorf( - "failed getEtcdCertVolumes:\n\texpected: %v\n\t actual: %v", - rt.volMount, - actualVolMount, - ) - } - }) - } -} - -func TestGetHostPathVolumesForTheControlPlane(t *testing.T) { - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - hostPathFileOrCreate := v1.HostPathFileOrCreate - volMap := make(map[string]map[string]v1.Volume) - volMap[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{} - volMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{ - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: testCertsDir, - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.Volume{ - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{} - volMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{ - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: testCertsDir, - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.Volume{ - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.Volume{ - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/controller-manager.conf", - Type: &hostPathFileOrCreate, - }, - }, - } - volMap[kubeadmconstants.KubeScheduler] = map[string]v1.Volume{} - volMap[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.Volume{ - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/scheduler.conf", - Type: &hostPathFileOrCreate, - }, - }, - } - volMountMap := make(map[string]map[string]v1.VolumeMount) - volMountMap[kubeadmconstants.KubeAPIServer] = map[string]v1.VolumeMount{} - volMountMap[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.VolumeMount{ - Name: "k8s-certs", - MountPath: testCertsDir, - ReadOnly: true, - } - volMountMap[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.VolumeMount{ - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - ReadOnly: true, - } - volMountMap[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{} - volMountMap[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{ - Name: "k8s-certs", - MountPath: testCertsDir, - ReadOnly: true, - } - volMountMap[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.VolumeMount{ - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - ReadOnly: true, - } - volMountMap[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.VolumeMount{ - Name: "kubeconfig", - MountPath: "/etc/kubernetes/controller-manager.conf", - ReadOnly: true, - } - volMountMap[kubeadmconstants.KubeScheduler] = map[string]v1.VolumeMount{} - volMountMap[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.VolumeMount{ - Name: "kubeconfig", - MountPath: "/etc/kubernetes/scheduler.conf", - ReadOnly: true, - } - - volMap2 := make(map[string]map[string]v1.Volume) - volMap2[kubeadmconstants.KubeAPIServer] = map[string]v1.Volume{} - volMap2[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.Volume{ - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: testCertsDir, - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.Volume{ - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-0"] = v1.Volume{ - Name: "etcd-certs-0", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/certs/etcd", - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-1"] = v1.Volume{ - Name: "etcd-certs-1", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/var/lib/etcd/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeControllerManager] = map[string]v1.Volume{} - volMap2[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.Volume{ - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: testCertsDir, - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.Volume{ - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.Volume{ - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/controller-manager.conf", - Type: &hostPathFileOrCreate, - }, - }, - } - volMap2[kubeadmconstants.KubeScheduler] = map[string]v1.Volume{} - volMap2[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.Volume{ - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/scheduler.conf", - Type: &hostPathFileOrCreate, - }, - }, - } - volMountMap2 := make(map[string]map[string]v1.VolumeMount) - volMountMap2[kubeadmconstants.KubeAPIServer] = map[string]v1.VolumeMount{} - volMountMap2[kubeadmconstants.KubeAPIServer]["k8s-certs"] = v1.VolumeMount{ - Name: "k8s-certs", - MountPath: testCertsDir, - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeAPIServer]["ca-certs"] = v1.VolumeMount{ - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-0"] = v1.VolumeMount{ - Name: "etcd-certs-0", - MountPath: "/etc/certs/etcd", - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeAPIServer]["etcd-certs-1"] = v1.VolumeMount{ - Name: "etcd-certs-1", - MountPath: "/var/lib/etcd/certs", - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeControllerManager] = map[string]v1.VolumeMount{} - volMountMap2[kubeadmconstants.KubeControllerManager]["k8s-certs"] = v1.VolumeMount{ - Name: "k8s-certs", - MountPath: testCertsDir, - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeControllerManager]["ca-certs"] = v1.VolumeMount{ - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeControllerManager]["kubeconfig"] = v1.VolumeMount{ - Name: "kubeconfig", - MountPath: "/etc/kubernetes/controller-manager.conf", - ReadOnly: true, - } - volMountMap2[kubeadmconstants.KubeScheduler] = map[string]v1.VolumeMount{} - volMountMap2[kubeadmconstants.KubeScheduler]["kubeconfig"] = v1.VolumeMount{ - Name: "kubeconfig", - MountPath: "/etc/kubernetes/scheduler.conf", - ReadOnly: true, - } - var tests = []struct { - name string - cfg *kubeadmapi.ClusterConfiguration - vol map[string]map[string]v1.Volume - volMount map[string]map[string]v1.VolumeMount - }{ - { - name: "Should ignore files in /etc/ssl/certs", - cfg: &kubeadmapi.ClusterConfiguration{ - CertificatesDir: testCertsDir, - Etcd: kubeadmapi.Etcd{}, - }, - vol: volMap, - volMount: volMountMap, - }, - { - name: "Should ignore files in /etc/ssl/certs and in CertificatesDir", - cfg: &kubeadmapi.ClusterConfiguration{ - CertificatesDir: testCertsDir, - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"foo"}, - CAFile: "/etc/certs/etcd/my-etcd-ca.crt", - CertFile: testCertsDir + "/etcd/my-etcd.crt", - KeyFile: "/var/lib/etcd/certs/my-etcd.key", - }, - }, - }, - vol: volMap2, - volMount: volMountMap2, - }, - } - - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - // set up tmp caCertsExtraVolumePaths for testing - caCertsExtraVolumePaths = []string{fmt.Sprintf("%s/etc/pki", tmpdir), fmt.Sprintf("%s/usr/share/ca-certificates", tmpdir)} - defer func() { caCertsExtraVolumePaths = []string{"/etc/pki", "/usr/share/ca-certificates"} }() - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - mounts := getHostPathVolumesForTheControlPlane(rt.cfg) - - // Avoid unit test errors when the flexvolume is mounted - delete(mounts.volumes[kubeadmconstants.KubeControllerManager], flexvolumeDirVolumeName) - delete(mounts.volumeMounts[kubeadmconstants.KubeControllerManager], flexvolumeDirVolumeName) - if !reflect.DeepEqual(mounts.volumes, rt.vol) { - t.Errorf( - "failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v", - rt.vol, - mounts.volumes, - ) - } - if !reflect.DeepEqual(mounts.volumeMounts, rt.volMount) { - t.Errorf( - "failed getHostPathVolumesForTheControlPlane:\n\texpected: %v\n\t actual: %v", - rt.volMount, - mounts.volumeMounts, - ) - } - }) - } -} - -func TestAddExtraHostPathMounts(t *testing.T) { - mounts := newControlPlaneHostPathMounts() - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - hostPathFileOrCreate := v1.HostPathFileOrCreate - vols := []v1.Volume{ - { - Name: "foo", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/tmp/foo", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "bar", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/tmp/bar", - Type: &hostPathFileOrCreate, - }, - }, - }, - } - volMounts := []v1.VolumeMount{ - { - Name: "foo", - MountPath: "/tmp/foo", - ReadOnly: true, - }, - { - Name: "bar", - MountPath: "/tmp/bar", - ReadOnly: true, - }, - } - mounts.AddHostPathMounts("component", vols, volMounts) - hostPathMounts := []kubeadmapi.HostPathMount{ - { - Name: "foo-0", - HostPath: "/tmp/qux-0", - MountPath: "/tmp/qux-0", - ReadOnly: true, - PathType: v1.HostPathFile, - }, - { - Name: "bar-0", - HostPath: "/tmp/asd-0", - MountPath: "/tmp/asd-0", - ReadOnly: false, - PathType: v1.HostPathDirectory, - }, - { - Name: "foo-1", - HostPath: "/tmp/qux-1", - MountPath: "/tmp/qux-1", - ReadOnly: true, - PathType: v1.HostPathFileOrCreate, - }, - { - Name: "bar-1", - HostPath: "/tmp/asd-1", - MountPath: "/tmp/asd-1", - ReadOnly: false, - PathType: v1.HostPathDirectoryOrCreate, - }, - } - mounts.AddExtraHostPathMounts("component", hostPathMounts) - for _, hostMount := range hostPathMounts { - t.Run(hostMount.Name, func(t *testing.T) { - volumeName := hostMount.Name - if _, ok := mounts.volumes["component"][volumeName]; !ok { - t.Errorf("Expected to find volume %q", volumeName) - } - vol := mounts.volumes["component"][volumeName] - if vol.Name != volumeName { - t.Errorf("Expected volume name %q", volumeName) - } - if vol.HostPath.Path != hostMount.HostPath { - t.Errorf("Expected host path %q", hostMount.HostPath) - } - if _, ok := mounts.volumeMounts["component"][volumeName]; !ok { - t.Errorf("Expected to find volume mount %q", volumeName) - } - if *vol.HostPath.Type != v1.HostPathType(hostMount.PathType) { - t.Errorf("Expected to host path type %q", hostMount.PathType) - } - volMount := mounts.volumeMounts["component"][volumeName] - if volMount.Name != volumeName { - t.Errorf("Expected volume mount name %q", volumeName) - } - if volMount.MountPath != hostMount.MountPath { - t.Errorf("Expected container path %q", hostMount.MountPath) - } - if volMount.ReadOnly != hostMount.ReadOnly { - t.Errorf("Expected volume readOnly setting %t", hostMount.ReadOnly) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/copycerts/BUILD b/cmd/kubeadm/app/phases/copycerts/BUILD deleted file mode 100644 index 60c2e6ec31ff8..0000000000000 --- a/cmd/kubeadm/app/phases/copycerts/BUILD +++ /dev/null @@ -1,59 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["copycerts.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/copycerts", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/crypto:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = ["copycerts_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util/crypto:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/phases/copycerts/copycerts.go b/cmd/kubeadm/app/phases/copycerts/copycerts.go deleted file mode 100644 index 199fc4845a586..0000000000000 --- a/cmd/kubeadm/app/phases/copycerts/copycerts.go +++ /dev/null @@ -1,295 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package copycerts - -import ( - "context" - "encoding/hex" - "fmt" - "io/ioutil" - "os" - "path" - "strings" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - clientset "k8s.io/client-go/kubernetes" - certutil "k8s.io/client-go/util/cert" - keyutil "k8s.io/client-go/util/keyutil" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - nodebootstraptokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - cryptoutil "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto" -) - -const ( - externalEtcdCA = "external-etcd-ca.crt" - externalEtcdCert = "external-etcd.crt" - externalEtcdKey = "external-etcd.key" -) - -// createShortLivedBootstrapToken creates the token used to manager kubeadm-certs -// and return the tokenID -func createShortLivedBootstrapToken(client clientset.Interface) (string, error) { - tokenStr, err := bootstraputil.GenerateBootstrapToken() - if err != nil { - return "", errors.Wrap(err, "error generating token to upload certs") - } - token, err := kubeadmapi.NewBootstrapTokenString(tokenStr) - if err != nil { - return "", errors.Wrap(err, "error creating upload certs token") - } - tokens := []kubeadmapi.BootstrapToken{{ - Token: token, - Description: "Proxy for managing TTL for the kubeadm-certs secret", - TTL: &metav1.Duration{ - Duration: kubeadmconstants.DefaultCertTokenDuration, - }, - }} - - if err := nodebootstraptokenphase.CreateNewTokens(client, tokens); err != nil { - return "", errors.Wrap(err, "error creating token") - } - return tokens[0].Token.ID, nil -} - -//CreateCertificateKey returns a cryptographically secure random key -func CreateCertificateKey() (string, error) { - randBytes, err := cryptoutil.CreateRandBytes(kubeadmconstants.CertificateKeySize) - if err != nil { - return "", err - } - return hex.EncodeToString(randBytes), nil -} - -//UploadCerts save certs needs to join a new control-plane on kubeadm-certs sercret. -func UploadCerts(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, key string) error { - fmt.Printf("[upload-certs] Storing the certificates in Secret %q in the %q Namespace\n", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem) - decodedKey, err := hex.DecodeString(key) - if err != nil { - return errors.Wrap(err, "error decoding certificate key") - } - tokenID, err := createShortLivedBootstrapToken(client) - if err != nil { - return err - } - - secretData, err := getDataFromDisk(cfg, decodedKey) - if err != nil { - return err - } - ref, err := getSecretOwnerRef(client, tokenID) - if err != nil { - return err - } - - err = apiclient.CreateOrUpdateSecret(client, &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeadmCertsSecret, - Namespace: metav1.NamespaceSystem, - OwnerReferences: ref, - }, - Data: secretData, - }) - if err != nil { - return err - } - - return createRBAC(client) -} - -func createRBAC(client clientset.Interface) error { - err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeadmCertsClusterRoleName, - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - ResourceNames: []string{kubeadmconstants.KubeadmCertsSecret}, - }, - }, - }) - if err != nil { - return err - } - - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeadmCertsClusterRoleName, - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: kubeadmconstants.KubeadmCertsClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} - -func getSecretOwnerRef(client clientset.Interface, tokenID string) ([]metav1.OwnerReference, error) { - secretName := bootstraputil.BootstrapTokenSecretName(tokenID) - secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{}) - if err != nil { - return nil, errors.Wrap(err, "error to get token reference") - } - - gvk := schema.GroupVersionKind{Version: "v1", Kind: "Secret"} - ref := metav1.NewControllerRef(secret, gvk) - return []metav1.OwnerReference{*ref}, nil -} - -func loadAndEncryptCert(certPath string, key []byte) ([]byte, error) { - cert, err := ioutil.ReadFile(certPath) - if err != nil { - return nil, err - } - return cryptoutil.EncryptBytes(cert, key) -} - -func certsToTransfer(cfg *kubeadmapi.InitConfiguration) map[string]string { - certsDir := cfg.CertificatesDir - certs := map[string]string{ - kubeadmconstants.CACertName: path.Join(certsDir, kubeadmconstants.CACertName), - kubeadmconstants.CAKeyName: path.Join(certsDir, kubeadmconstants.CAKeyName), - kubeadmconstants.FrontProxyCACertName: path.Join(certsDir, kubeadmconstants.FrontProxyCACertName), - kubeadmconstants.FrontProxyCAKeyName: path.Join(certsDir, kubeadmconstants.FrontProxyCAKeyName), - kubeadmconstants.ServiceAccountPublicKeyName: path.Join(certsDir, kubeadmconstants.ServiceAccountPublicKeyName), - kubeadmconstants.ServiceAccountPrivateKeyName: path.Join(certsDir, kubeadmconstants.ServiceAccountPrivateKeyName), - } - - if cfg.Etcd.External == nil { - certs[kubeadmconstants.EtcdCACertName] = path.Join(certsDir, kubeadmconstants.EtcdCACertName) - certs[kubeadmconstants.EtcdCAKeyName] = path.Join(certsDir, kubeadmconstants.EtcdCAKeyName) - } else { - certs[externalEtcdCA] = cfg.Etcd.External.CAFile - certs[externalEtcdCert] = cfg.Etcd.External.CertFile - certs[externalEtcdKey] = cfg.Etcd.External.KeyFile - } - - return certs -} - -func getDataFromDisk(cfg *kubeadmapi.InitConfiguration, key []byte) (map[string][]byte, error) { - secretData := map[string][]byte{} - for certName, certPath := range certsToTransfer(cfg) { - cert, err := loadAndEncryptCert(certPath, key) - if err == nil || os.IsNotExist(err) { - secretData[certOrKeyNameToSecretName(certName)] = cert - } else { - return nil, err - } - } - return secretData, nil -} - -// DownloadCerts downloads the certificates needed to join a new control plane. -func DownloadCerts(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, key string) error { - fmt.Printf("[download-certs] Downloading the certificates in Secret %q in the %q Namespace\n", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem) - - decodedKey, err := hex.DecodeString(key) - if err != nil { - return errors.Wrap(err, "error decoding certificate key") - } - - secret, err := getSecret(client) - if err != nil { - return errors.Wrap(err, "error downloading the secret") - } - - secretData, err := getDataFromSecret(secret, decodedKey) - if err != nil { - return errors.Wrap(err, "error decoding secret data with provided key") - } - - for certOrKeyName, certOrKeyPath := range certsToTransfer(cfg) { - certOrKeyData, found := secretData[certOrKeyNameToSecretName(certOrKeyName)] - if !found { - return errors.Errorf("the Secret does not include the required certificate or key - name: %s, path: %s", certOrKeyName, certOrKeyPath) - } - if len(certOrKeyData) == 0 { - klog.V(1).Infof("[download-certs] Not saving %q to disk, since it is empty in the %q Secret\n", certOrKeyName, kubeadmconstants.KubeadmCertsSecret) - continue - } - if err := writeCertOrKey(certOrKeyPath, certOrKeyData); err != nil { - return err - } - } - - return nil -} - -func writeCertOrKey(certOrKeyPath string, certOrKeyData []byte) error { - if _, err := keyutil.ParsePublicKeysPEM(certOrKeyData); err == nil { - return keyutil.WriteKey(certOrKeyPath, certOrKeyData) - } else if _, err := certutil.ParseCertsPEM(certOrKeyData); err == nil { - return certutil.WriteCert(certOrKeyPath, certOrKeyData) - } - return errors.New("unknown data found in Secret entry") -} - -func getSecret(client clientset.Interface) (*v1.Secret, error) { - secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.KubeadmCertsSecret, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return nil, errors.Errorf("Secret %q was not found in the %q Namespace. This Secret might have expired. Please, run `kubeadm init phase upload-certs --upload-certs` on a control plane to generate a new one", kubeadmconstants.KubeadmCertsSecret, metav1.NamespaceSystem) - } - return nil, err - } - return secret, nil -} - -func getDataFromSecret(secret *v1.Secret, key []byte) (map[string][]byte, error) { - secretData := map[string][]byte{} - for secretName, encryptedSecret := range secret.Data { - // In some cases the secret might have empty data if the secrets were not present on disk - // when uploading. This can specially happen with external insecure etcd (no certs) - if len(encryptedSecret) > 0 { - cert, err := cryptoutil.DecryptBytes(encryptedSecret, key) - if err != nil { - // If any of the decrypt operations fail do not return a partial result, - // return an empty result immediately - return map[string][]byte{}, err - } - secretData[secretName] = cert - } else { - secretData[secretName] = []byte{} - } - } - return secretData, nil -} - -func certOrKeyNameToSecretName(certOrKeyName string) string { - return strings.Replace(certOrKeyName, "/", "-", -1) -} diff --git a/cmd/kubeadm/app/phases/copycerts/copycerts_test.go b/cmd/kubeadm/app/phases/copycerts/copycerts_test.go deleted file mode 100644 index be2032aab6a4e..0000000000000 --- a/cmd/kubeadm/app/phases/copycerts/copycerts_test.go +++ /dev/null @@ -1,281 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package copycerts - -import ( - "context" - "encoding/hex" - "io/ioutil" - "os" - "path" - "regexp" - "testing" - - "github.com/lithammer/dedent" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fakeclient "k8s.io/client-go/kubernetes/fake" - certutil "k8s.io/client-go/util/cert" - keyutil "k8s.io/client-go/util/keyutil" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - cryptoutil "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func TestGetDataFromInitConfig(t *testing.T) { - certData := []byte("cert-data") - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - cfg := &kubeadmapi.InitConfiguration{} - cfg.CertificatesDir = tmpdir - - key, err := CreateCertificateKey() - if err != nil { - t.Fatalf(dedent.Dedent("failed to create key.\nfatal error: %v"), err) - } - decodedKey, err := hex.DecodeString(key) - if err != nil { - t.Fatalf(dedent.Dedent("failed to decode key.\nfatal error: %v"), err) - } - - if err := os.Mkdir(path.Join(tmpdir, "etcd"), 0755); err != nil { - t.Fatalf(dedent.Dedent("failed to create etcd cert dir.\nfatal error: %v"), err) - } - - certs := certsToTransfer(cfg) - for name, path := range certs { - if err := ioutil.WriteFile(path, certData, 0644); err != nil { - t.Fatalf(dedent.Dedent("failed to write cert: %s\nfatal error: %v"), name, err) - } - } - - secretData, err := getDataFromDisk(cfg, decodedKey) - if err != nil { - t.Fatalf("failed to get secret data. fatal error: %v", err) - } - - re := regexp.MustCompile(`[-._a-zA-Z0-9]+`) - for name, data := range secretData { - if !re.MatchString(name) { - t.Fatalf(dedent.Dedent("failed to validate secretData\n %s isn't a valid secret key"), name) - } - - decryptedData, err := cryptoutil.DecryptBytes(data, decodedKey) - if string(certData) != string(decryptedData) { - t.Fatalf(dedent.Dedent("can't decrypt cert: %s\nfatal error: %v"), name, err) - } - } -} - -func TestCertsToTransfer(t *testing.T) { - localEtcdCfg := &kubeadmapi.InitConfiguration{} - externalEtcdCfg := &kubeadmapi.InitConfiguration{} - externalEtcdCfg.Etcd = kubeadmapi.Etcd{} - externalEtcdCfg.Etcd.External = &kubeadmapi.ExternalEtcd{} - - commonExpectedCerts := []string{ - kubeadmconstants.CACertName, - kubeadmconstants.CAKeyName, - kubeadmconstants.FrontProxyCACertName, - kubeadmconstants.FrontProxyCAKeyName, - kubeadmconstants.ServiceAccountPublicKeyName, - kubeadmconstants.ServiceAccountPrivateKeyName, - } - - tests := map[string]struct { - config *kubeadmapi.InitConfiguration - expectedCerts []string - }{ - "local etcd": { - config: localEtcdCfg, - expectedCerts: append( - []string{kubeadmconstants.EtcdCACertName, kubeadmconstants.EtcdCAKeyName}, - commonExpectedCerts..., - ), - }, - "external etcd": { - config: externalEtcdCfg, - expectedCerts: append( - []string{externalEtcdCA, externalEtcdCert, externalEtcdKey}, - commonExpectedCerts..., - ), - }, - } - - for name, test := range tests { - t.Run(name, func(t2 *testing.T) { - certList := certsToTransfer(test.config) - for _, cert := range test.expectedCerts { - if _, found := certList[cert]; !found { - t2.Fatalf(dedent.Dedent("failed to get list of certs to upload\ncert %s not found"), cert) - } - } - }) - } -} - -func TestCertOrKeyNameToSecretName(t *testing.T) { - tests := []struct { - keyName string - expectedSecretName string - }{ - { - keyName: "apiserver-kubelet-client.crt", - expectedSecretName: "apiserver-kubelet-client.crt", - }, - { - keyName: "etcd/ca.crt", - expectedSecretName: "etcd-ca.crt", - }, - { - keyName: "etcd/healthcheck-client.crt", - expectedSecretName: "etcd-healthcheck-client.crt", - }, - } - - for _, tc := range tests { - secretName := certOrKeyNameToSecretName(tc.keyName) - if secretName != tc.expectedSecretName { - t.Fatalf("secret name %s didn't match expected name %s", secretName, tc.expectedSecretName) - } - } -} - -func TestUploadCerts(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - secretKey, err := CreateCertificateKey() - if err != nil { - t.Fatalf("could not create certificate key: %v", err) - } - - initConfiguration := testutil.GetDefaultInternalConfig(t) - initConfiguration.ClusterConfiguration.CertificatesDir = tmpdir - - if err := certs.CreatePKIAssets(initConfiguration); err != nil { - t.Fatalf("error creating PKI assets: %v", err) - } - - cs := fakeclient.NewSimpleClientset() - if err := UploadCerts(cs, initConfiguration, secretKey); err != nil { - t.Fatalf("error uploading certs: %v", err) - } - rawSecretKey, err := hex.DecodeString(secretKey) - if err != nil { - t.Fatalf("error decoding key: %v", err) - } - secretMap, err := cs.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.KubeadmCertsSecret, metav1.GetOptions{}) - if err != nil { - t.Fatalf("could not fetch secret: %v", err) - } - for certName, certPath := range certsToTransfer(initConfiguration) { - secretCertData, err := cryptoutil.DecryptBytes(secretMap.Data[certOrKeyNameToSecretName(certName)], rawSecretKey) - if err != nil { - t.Fatalf("error decrypting secret data: %v", err) - } - diskCertData, err := ioutil.ReadFile(certPath) - if err != nil { - t.Fatalf("error reading certificate from disk: %v", err) - } - // Check that the encrypted contents on the secret match the contents on disk, and that all - // the expected certificates are in the secret - if string(secretCertData) != string(diskCertData) { - t.Fatalf("cert %s does not have the expected contents. contents: %q; expected contents: %q", certName, string(secretCertData), string(diskCertData)) - } - } -} - -func TestDownloadCerts(t *testing.T) { - secretKey, err := CreateCertificateKey() - if err != nil { - t.Fatalf("could not create certificate key: %v", err) - } - - // Temporary directory where certificates will be generated - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - initConfiguration := testutil.GetDefaultInternalConfig(t) - initConfiguration.ClusterConfiguration.CertificatesDir = tmpdir - - // Temporary directory where certificates will be downloaded to - targetTmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(targetTmpdir) - initForDownloadConfiguration := testutil.GetDefaultInternalConfig(t) - initForDownloadConfiguration.ClusterConfiguration.CertificatesDir = targetTmpdir - - if err := certs.CreatePKIAssets(initConfiguration); err != nil { - t.Fatalf("error creating PKI assets: %v", err) - } - - kubeadmCertsSecret := createKubeadmCertsSecret(t, initConfiguration, secretKey) - cs := fakeclient.NewSimpleClientset(kubeadmCertsSecret) - if err := DownloadCerts(cs, initForDownloadConfiguration, secretKey); err != nil { - t.Fatalf("error downloading certs: %v", err) - } - - const keyFileMode = 0600 - const certFileMode = 0644 - - for certName, certPath := range certsToTransfer(initForDownloadConfiguration) { - diskCertData, err := ioutil.ReadFile(certPath) - if err != nil { - t.Errorf("error reading certificate from disk: %v", err) - } - // Check that the written files are either certificates or keys, and that they have - // the expected permissions - if _, err := keyutil.ParsePublicKeysPEM(diskCertData); err == nil { - if stat, err := os.Stat(certPath); err == nil { - if stat.Mode() != keyFileMode { - t.Errorf("key %q should have mode %#o, has %#o", certName, keyFileMode, stat.Mode()) - } - } else { - t.Errorf("could not stat key %q: %v", certName, err) - } - } else if _, err := certutil.ParseCertsPEM(diskCertData); err == nil { - if stat, err := os.Stat(certPath); err == nil { - if stat.Mode() != certFileMode { - t.Errorf("cert %q should have mode %#o, has %#o", certName, certFileMode, stat.Mode()) - } - } else { - t.Errorf("could not stat cert %q: %v", certName, err) - } - } else { - t.Errorf("secret %q was not identified as a cert or as a key", certName) - } - } -} - -func createKubeadmCertsSecret(t *testing.T, cfg *kubeadmapi.InitConfiguration, secretKey string) *v1.Secret { - decodedKey, err := hex.DecodeString(secretKey) - if err != nil { - t.Fatalf("error decoding key: %v", err) - } - secretData, err := getDataFromDisk(cfg, decodedKey) - if err != nil { - t.Fatalf("error creating secret data: %v", err) - } - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeadmCertsSecret, - Namespace: metav1.NamespaceSystem, - }, - Data: secretData, - } -} diff --git a/cmd/kubeadm/app/phases/etcd/BUILD b/cmd/kubeadm/app/phases/etcd/BUILD deleted file mode 100644 index 5fd4735ac5314..0000000000000 --- a/cmd/kubeadm/app/phases/etcd/BUILD +++ /dev/null @@ -1,54 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["local_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/etcd:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = ["local.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/etcd:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go deleted file mode 100644 index 1965e916eedf7..0000000000000 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package etcd - -import ( - "fmt" - "net" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - utilsnet "k8s.io/utils/net" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" -) - -const ( - etcdVolumeName = "etcd-data" - certsVolumeName = "etcd-certs" - etcdHealthyCheckInterval = 5 * time.Second - etcdHealthyCheckRetries = 8 -) - -// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file. -// This function is used by init - when the etcd cluster is empty - or by kubeadm -// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact) -func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { - if cfg.Etcd.External != nil { - return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd") - } - // gets etcd StaticPodSpec - spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{}) - - // if patchesDir is defined, patch the static Pod manifest - if patchesDir != "" { - patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) - if err != nil { - return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd) - } - spec = *patchedSpec - } - - // writes etcd StaticPod to disk - if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { - return err - } - - klog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir)) - return nil -} - -// CheckLocalEtcdClusterStatus verifies health state of local/stacked etcd cluster before installing a new etcd member -func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) error { - klog.V(1).Info("[etcd] Checking etcd cluster health") - - // creates an etcd client that connects to all the local/stacked etcd members - klog.V(1).Info("creating etcd client that connects to etcd pods") - etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) - if err != nil { - return err - } - - // Checking health state - err = etcdClient.CheckClusterHealth() - if err != nil { - return errors.Wrap(err, "etcd cluster is not healthy") - } - - return nil -} - -// RemoveStackedEtcdMemberFromCluster will remove a local etcd member from etcd cluster, -// when reset the control plane node. -func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error { - // creates an etcd client that connects to all the local/stacked etcd members - klog.V(1).Info("[etcd] creating etcd client that connects to etcd pods") - etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) - if err != nil { - return err - } - - members, err := etcdClient.ListMembers() - if err != nil { - return err - } - // If this is the only remaining stacked etcd member in the cluster, calling RemoveMember() - // is not needed. - if len(members) == 1 { - etcdClientAddress := etcdutil.GetClientURL(&cfg.LocalAPIEndpoint) - for _, endpoint := range etcdClient.Endpoints { - if endpoint == etcdClientAddress { - klog.V(1).Info("[etcd] This is the only remaining etcd member in the etcd cluster, skip removing it") - return nil - } - } - } - - // notifies the other members of the etcd cluster about the removing member - etcdPeerAddress := etcdutil.GetPeerURL(&cfg.LocalAPIEndpoint) - - klog.V(2).Infof("[etcd] get the member id from peer: %s", etcdPeerAddress) - id, err := etcdClient.GetMemberID(etcdPeerAddress) - if err != nil { - return err - } - - klog.V(1).Infof("[etcd] removing etcd member: %s, id: %d", etcdPeerAddress, id) - members, err = etcdClient.RemoveMember(id) - if err != nil { - return err - } - klog.V(1).Infof("[etcd] Updated etcd member list: %v", members) - - return nil -} - -// CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file -// for an additional etcd member that is joining an existing local/stacked etcd cluster. -// Other members of the etcd cluster will be notified of the joining node in beforehand as well. -func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error { - // creates an etcd client that connects to all the local/stacked etcd members - klog.V(1).Info("creating etcd client that connects to etcd pods") - etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) - if err != nil { - return err - } - - etcdPeerAddress := etcdutil.GetPeerURL(endpoint) - - klog.V(1).Infoln("[etcd] Getting the list of existing members") - initialCluster, err := etcdClient.ListMembers() - if err != nil { - return err - } - - // only add the new member if it doesn't already exists - var exists bool - klog.V(1).Infof("[etcd] Checking if the etcd member already exists: %s", etcdPeerAddress) - for _, member := range initialCluster { - if member.PeerURL == etcdPeerAddress { - exists = true - break - } - } - - if exists { - klog.V(1).Infof("[etcd] Etcd member already exists: %s", endpoint) - } else { - klog.V(1).Infof("[etcd] Adding etcd member: %s", etcdPeerAddress) - initialCluster, err = etcdClient.AddMember(nodeName, etcdPeerAddress) - if err != nil { - return err - } - - fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster") - klog.V(1).Infof("Updated etcd member list: %v", initialCluster) - } - - fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd) - - // gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members - spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster) - - // if patchesDir is defined, patch the static Pod manifest - if patchesDir != "" { - patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout) - if err != nil { - return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd) - } - spec = *patchedSpec - } - - // writes etcd StaticPod to disk - if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil { - return err - } - - fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries) - if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil { - return err - } - - return nil -} - -// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current configuration -// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests. -func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) v1.Pod { - pathType := v1.HostPathDirectoryOrCreate - etcdMounts := map[string]v1.Volume{ - etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType), - certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType), - } - // probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress - probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress)) - return staticpodutil.ComponentPod( - v1.Container{ - Name: kubeadmconstants.Etcd, - Command: getEtcdCommand(cfg, endpoint, nodeName, initialCluster), - Image: images.GetEtcdImage(cfg), - ImagePullPolicy: v1.PullIfNotPresent, - // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner - VolumeMounts: []v1.VolumeMount{ - staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false), - staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false), - }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("100Mi"), - }, - }, - LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health", probePort, probeScheme), - StartupProbe: staticpodutil.StartupProbe(probeHostname, "/health", probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane), - }, - etcdMounts, - // etcd will listen on the advertise address of the API server, in a different port (2379) - map[string]string{kubeadmconstants.EtcdAdvertiseClientUrlsAnnotationKey: etcdutil.GetClientURL(endpoint)}, - ) -} - -// getEtcdCommand builds the right etcd command from the given config object -func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string { - // localhost IP family should be the same that the AdvertiseAddress - etcdLocalhostAddress := "127.0.0.1" - if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) { - etcdLocalhostAddress = "::1" - } - defaultArguments := map[string]string{ - "name": nodeName, - "listen-client-urls": fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)), - "advertise-client-urls": etcdutil.GetClientURL(endpoint), - "listen-peer-urls": etcdutil.GetPeerURL(endpoint), - "initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint), - "data-dir": cfg.Etcd.Local.DataDir, - "cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName), - "key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName), - "trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), - "client-cert-auth": "true", - "peer-cert-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName), - "peer-key-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName), - "peer-trusted-ca-file": filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName), - "peer-client-cert-auth": "true", - "snapshot-count": "10000", - "listen-metrics-urls": fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))), - } - - if len(initialCluster) == 0 { - defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint)) - } else { - // NB. the joining etcd member should be part of the initialCluster list - endpoints := []string{} - for _, member := range initialCluster { - endpoints = append(endpoints, fmt.Sprintf("%s=%s", member.Name, member.PeerURL)) - } - - defaultArguments["initial-cluster"] = strings.Join(endpoints, ",") - defaultArguments["initial-cluster-state"] = "existing" - } - - command := []string{"etcd"} - command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...) - return command -} diff --git a/cmd/kubeadm/app/phases/etcd/local_test.go b/cmd/kubeadm/app/phases/etcd/local_test.go deleted file mode 100644 index d45df03337985..0000000000000 --- a/cmd/kubeadm/app/phases/etcd/local_test.go +++ /dev/null @@ -1,305 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package etcd - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "sort" - "testing" - - "github.com/lithammer/dedent" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func TestGetEtcdPodSpec(t *testing.T) { - // Creates a ClusterConfiguration - cfg := &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.7.0", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - DataDir: "/var/lib/etcd", - }, - }, - } - endpoint := &kubeadmapi.APIEndpoint{} - - // Executes GetEtcdPodSpec - spec := GetEtcdPodSpec(cfg, endpoint, "", []etcdutil.Member{}) - - // Assert each specs refers to the right pod - if spec.Spec.Containers[0].Name != kubeadmconstants.Etcd { - t.Errorf("getKubeConfigSpecs spec for etcd contains pod %s, expects %s", spec.Spec.Containers[0].Name, kubeadmconstants.Etcd) - } -} - -func TestCreateLocalEtcdStaticPodManifestFile(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - var tests = []struct { - cfg *kubeadmapi.ClusterConfiguration - expectedError bool - }{ - { - cfg: &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.7.0", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - DataDir: tmpdir + "/etcd", - }, - }, - }, - expectedError: false, - }, - { - cfg: &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.7.0", - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{ - "https://etcd-instance:2379", - }, - CAFile: "/etc/kubernetes/pki/etcd/ca.crt", - CertFile: "/etc/kubernetes/pki/etcd/apiserver-etcd-client.crt", - KeyFile: "/etc/kubernetes/pki/etcd/apiserver-etcd-client.key", - }, - }, - }, - expectedError: true, - }, - } - - for _, test := range tests { - // Execute createStaticPodFunction - manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) - err := CreateLocalEtcdStaticPodManifestFile(manifestPath, "", "", test.cfg, &kubeadmapi.APIEndpoint{}) - - if !test.expectedError { - if err != nil { - t.Errorf("CreateLocalEtcdStaticPodManifestFile failed when not expected: %v", err) - } - // Assert expected files are there - testutil.AssertFilesCount(t, manifestPath, 1) - testutil.AssertFileExists(t, manifestPath, kubeadmconstants.Etcd+".yaml") - } else { - testutil.AssertError(t, err, "etcd static pod manifest cannot be generated for cluster using external etcd") - } - } -} - -func TestCreateLocalEtcdStaticPodManifestFileWithPatches(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Creates a Cluster Configuration - cfg := &kubeadmapi.ClusterConfiguration{ - KubernetesVersion: "v1.7.0", - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - DataDir: tmpdir + "/etcd", - }, - }, - } - - patchesPath := filepath.Join(tmpdir, "patch-files") - err := os.MkdirAll(patchesPath, 0777) - if err != nil { - t.Fatalf("Couldn't create %s", patchesPath) - } - - patchString := dedent.Dedent(` - metadata: - annotations: - patched: "true" - `) - - err = ioutil.WriteFile(filepath.Join(patchesPath, kubeadmconstants.Etcd+".yaml"), []byte(patchString), 0644) - if err != nil { - t.Fatalf("WriteFile returned unexpected error: %v", err) - } - - manifestPath := filepath.Join(tmpdir, kubeadmconstants.ManifestsSubDirName) - err = CreateLocalEtcdStaticPodManifestFile(manifestPath, patchesPath, "", cfg, &kubeadmapi.APIEndpoint{}) - if err != nil { - t.Errorf("Error executing createStaticPodFunction: %v", err) - return - } - - pod, err := staticpodutil.ReadStaticPodFromDisk(filepath.Join(manifestPath, kubeadmconstants.Etcd+".yaml")) - if err != nil { - t.Errorf("Error executing ReadStaticPodFromDisk: %v", err) - return - } - - if _, ok := pod.ObjectMeta.Annotations["patched"]; !ok { - t.Errorf("Patches were not applied to %s", kubeadmconstants.Etcd) - } -} - -func TestGetEtcdCommand(t *testing.T) { - var tests = []struct { - name string - advertiseAddress string - nodeName string - extraArgs map[string]string - initialCluster []etcdutil.Member - expected []string - }{ - { - name: "Default args - with empty etcd initial cluster", - advertiseAddress: "1.2.3.4", - nodeName: "foo", - expected: []string{ - "etcd", - "--name=foo", - fmt.Sprintf("--listen-client-urls=https://127.0.0.1:%d,https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort), - fmt.Sprintf("--advertise-client-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - fmt.Sprintf("--initial-advertise-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - "--data-dir=/var/lib/etcd", - "--cert-file=" + kubeadmconstants.EtcdServerCertName, - "--key-file=" + kubeadmconstants.EtcdServerKeyName, - "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--client-cert-auth=true", - "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, - "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, - "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--snapshot-count=10000", - "--peer-client-cert-auth=true", - fmt.Sprintf("--initial-cluster=foo=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - }, - }, - { - name: "Default args - With an existing etcd cluster", - advertiseAddress: "1.2.3.4", - nodeName: "foo", - initialCluster: []etcdutil.Member{ - {Name: "foo", PeerURL: fmt.Sprintf("https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort)}, // NB. the joining etcd instance should be part of the initialCluster list - {Name: "bar", PeerURL: fmt.Sprintf("https://5.6.7.8:%d", kubeadmconstants.EtcdListenPeerPort)}, - }, - expected: []string{ - "etcd", - "--name=foo", - fmt.Sprintf("--listen-client-urls=https://127.0.0.1:%d,https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort), - fmt.Sprintf("--advertise-client-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - fmt.Sprintf("--initial-advertise-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - "--data-dir=/var/lib/etcd", - "--cert-file=" + kubeadmconstants.EtcdServerCertName, - "--key-file=" + kubeadmconstants.EtcdServerKeyName, - "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--client-cert-auth=true", - "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, - "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, - "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--snapshot-count=10000", - "--peer-client-cert-auth=true", - "--initial-cluster-state=existing", - fmt.Sprintf("--initial-cluster=foo=https://1.2.3.4:%d,bar=https://5.6.7.8:%d", kubeadmconstants.EtcdListenPeerPort, kubeadmconstants.EtcdListenPeerPort), - }, - }, - { - name: "Extra args", - advertiseAddress: "1.2.3.4", - nodeName: "bar", - extraArgs: map[string]string{ - "listen-client-urls": "https://10.0.1.10:2379", - "advertise-client-urls": "https://10.0.1.10:2379", - }, - expected: []string{ - "etcd", - "--name=bar", - "--listen-client-urls=https://10.0.1.10:2379", - fmt.Sprintf("--listen-metrics-urls=http://127.0.0.1:%d", kubeadmconstants.EtcdMetricsPort), - "--advertise-client-urls=https://10.0.1.10:2379", - fmt.Sprintf("--listen-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - fmt.Sprintf("--initial-advertise-peer-urls=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - "--data-dir=/var/lib/etcd", - "--cert-file=" + kubeadmconstants.EtcdServerCertName, - "--key-file=" + kubeadmconstants.EtcdServerKeyName, - "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--client-cert-auth=true", - "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, - "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, - "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--snapshot-count=10000", - "--peer-client-cert-auth=true", - fmt.Sprintf("--initial-cluster=bar=https://1.2.3.4:%d", kubeadmconstants.EtcdListenPeerPort), - }, - }, - { - name: "IPv6 advertise address", - advertiseAddress: "2001:db8::3", - nodeName: "foo", - expected: []string{ - "etcd", - "--name=foo", - fmt.Sprintf("--listen-client-urls=https://[::1]:%d,https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort, kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-metrics-urls=http://[::1]:%d", kubeadmconstants.EtcdMetricsPort), - fmt.Sprintf("--advertise-client-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenClientPort), - fmt.Sprintf("--listen-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort), - fmt.Sprintf("--initial-advertise-peer-urls=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort), - "--data-dir=/var/lib/etcd", - "--cert-file=" + kubeadmconstants.EtcdServerCertName, - "--key-file=" + kubeadmconstants.EtcdServerKeyName, - "--trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--client-cert-auth=true", - "--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName, - "--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName, - "--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName, - "--snapshot-count=10000", - "--peer-client-cert-auth=true", - fmt.Sprintf("--initial-cluster=foo=https://[2001:db8::3]:%d", kubeadmconstants.EtcdListenPeerPort), - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - endpoint := &kubeadmapi.APIEndpoint{ - AdvertiseAddress: rt.advertiseAddress, - } - cfg := &kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - DataDir: "/var/lib/etcd", - ExtraArgs: rt.extraArgs, - }, - }, - } - actual := getEtcdCommand(cfg, endpoint, rt.nodeName, rt.initialCluster) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/kubeconfig/BUILD b/cmd/kubeadm/app/phases/kubeconfig/BUILD deleted file mode 100644 index 066bf88dbbb28..0000000000000 --- a/cmd/kubeadm/app/phases/kubeconfig/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "doc.go", - "kubeconfig.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["kubeconfig_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/certs:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//cmd/kubeadm/test/kubeconfig:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/phases/kubeconfig/doc.go b/cmd/kubeadm/app/phases/kubeconfig/doc.go deleted file mode 100644 index f8d5afa12f5f5..0000000000000 --- a/cmd/kubeadm/app/phases/kubeconfig/doc.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeconfig - -/* - - PHASE: KUBECONFIG - - INPUTS: - From InitConfiguration - The API Server endpoint (AdvertiseAddress + BindPort) is required so the kubeconfig file knows where to find the control-plane - The KubernetesDir path is required for knowing where to put the kubeconfig files - The PKIPath is required for knowing where all certificates should be stored - - OUTPUTS: - Files to KubernetesDir (default /etc/kubernetes): - - admin.conf - - kubelet.conf - - scheduler.conf - - controller-manager.conf -*/ diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go deleted file mode 100644 index 48c5af8ef1b27..0000000000000 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go +++ /dev/null @@ -1,528 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeconfig - -import ( - "bytes" - "crypto" - "crypto/x509" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" - - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -const ( - errInvalid = "invalid argument" - errExist = "file already exists" -) - -// clientCertAuth struct holds info required to build a client certificate to provide authentication info in a kubeconfig object -type clientCertAuth struct { - CAKey crypto.Signer - Organizations []string -} - -// tokenAuth struct holds info required to use a token to provide authentication info in a kubeconfig object -type tokenAuth struct { - Token string `datapolicy:"token"` -} - -// kubeConfigSpec struct holds info required to build a KubeConfig object -type kubeConfigSpec struct { - CACert *x509.Certificate - APIServer string - ClientName string - TokenAuth *tokenAuth `datapolicy:"token"` - ClientCertAuth *clientCertAuth `datapolicy:"security-key"` -} - -// CreateJoinControlPlaneKubeConfigFiles will create and write to disk the kubeconfig files required by kubeadm -// join --control-plane workflow, plus the admin kubeconfig file used by the administrator and kubeadm itself; the -// kubelet.conf file must not be created because it will be created and signed by the kubelet TLS bootstrap process. -// When not using external CA mode, if a kubeconfig file already exists it is used only if evaluated equal, -// otherwise an error is returned. For external CA mode, the creation of kubeconfig files is skipped. -func CreateJoinControlPlaneKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration) error { - var externaCA bool - caKeyPath := filepath.Join(cfg.CertificatesDir, kubeadmconstants.CAKeyName) - if _, err := os.Stat(caKeyPath); os.IsNotExist(err) { - externaCA = true - } - - files := []string{ - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.ControllerManagerKubeConfigFileName, - kubeadmconstants.SchedulerKubeConfigFileName, - } - - for _, file := range files { - if externaCA { - fmt.Printf("[kubeconfig] External CA mode: Using user provided %s\n", file) - continue - } - if err := createKubeConfigFiles(outDir, cfg, file); err != nil { - return err - } - } - return nil -} - -// CreateKubeConfigFile creates a kubeconfig file. -// If the kubeconfig file already exists, it is used only if evaluated equal; otherwise an error is returned. -func CreateKubeConfigFile(kubeConfigFileName string, outDir string, cfg *kubeadmapi.InitConfiguration) error { - klog.V(1).Infof("creating kubeconfig file for %s", kubeConfigFileName) - return createKubeConfigFiles(outDir, cfg, kubeConfigFileName) -} - -// createKubeConfigFiles creates all the requested kubeconfig files. -// If kubeconfig files already exists, they are used only if evaluated equal; otherwise an error is returned. -func createKubeConfigFiles(outDir string, cfg *kubeadmapi.InitConfiguration, kubeConfigFileNames ...string) error { - - // gets the KubeConfigSpecs, actualized for the current InitConfiguration - specs, err := getKubeConfigSpecs(cfg) - if err != nil { - return err - } - - for _, kubeConfigFileName := range kubeConfigFileNames { - // retrieves the KubeConfigSpec for given kubeConfigFileName - spec, exists := specs[kubeConfigFileName] - if !exists { - return errors.Errorf("couldn't retrieve KubeConfigSpec for %s", kubeConfigFileName) - } - - // builds the KubeConfig object - config, err := buildKubeConfigFromSpec(spec, cfg.ClusterName) - if err != nil { - return err - } - - // writes the kubeconfig to disk if it does not exist - if err = createKubeConfigFileIfNotExists(outDir, kubeConfigFileName, config); err != nil { - return err - } - } - - return nil -} - -// getKubeConfigSpecs returns all KubeConfigSpecs actualized to the context of the current InitConfiguration -// NB. this method holds the information about how kubeadm creates kubeconfig files. -func getKubeConfigSpecs(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) { - caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) - if err != nil { - return nil, errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded") - } - // Validate period - certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert) - - configs, err := getKubeConfigSpecsBase(cfg) - if err != nil { - return nil, err - } - for _, spec := range configs { - spec.CACert = caCert - spec.ClientCertAuth.CAKey = caKey - } - return configs, nil -} - -// buildKubeConfigFromSpec creates a kubeconfig object for the given kubeConfigSpec -func buildKubeConfigFromSpec(spec *kubeConfigSpec, clustername string) (*clientcmdapi.Config, error) { - - // If this kubeconfig should use token - if spec.TokenAuth != nil { - // create a kubeconfig with a token - return kubeconfigutil.CreateWithToken( - spec.APIServer, - clustername, - spec.ClientName, - pkiutil.EncodeCertPEM(spec.CACert), - spec.TokenAuth.Token, - ), nil - } - - // otherwise, create a client certs - clientCertConfig := newClientCertConfigFromKubeConfigSpec(spec) - - clientCert, clientKey, err := pkiutil.NewCertAndKey(spec.CACert, spec.ClientCertAuth.CAKey, &clientCertConfig) - if err != nil { - return nil, errors.Wrapf(err, "failure while creating %s client certificate", spec.ClientName) - } - - encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(clientKey) - if err != nil { - return nil, errors.Wrapf(err, "failed to marshal private key to PEM") - } - // create a kubeconfig with the client certs - return kubeconfigutil.CreateWithCerts( - spec.APIServer, - clustername, - spec.ClientName, - pkiutil.EncodeCertPEM(spec.CACert), - encodedClientKey, - pkiutil.EncodeCertPEM(clientCert), - ), nil -} - -func newClientCertConfigFromKubeConfigSpec(spec *kubeConfigSpec) pkiutil.CertConfig { - return pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: spec.ClientName, - Organization: spec.ClientCertAuth.Organizations, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - } -} - -// validateKubeConfig check if the kubeconfig file exist and has the expected CA and server URL -func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) error { - kubeConfigFilePath := filepath.Join(outDir, filename) - - if _, err := os.Stat(kubeConfigFilePath); err != nil { - return err - } - - // The kubeconfig already exists, let's check if it has got the same CA and server URL - currentConfig, err := clientcmd.LoadFromFile(kubeConfigFilePath) - if err != nil { - return errors.Wrapf(err, "failed to load kubeconfig file %s that already exists on disk", kubeConfigFilePath) - } - - expectedCtx, exists := config.Contexts[config.CurrentContext] - if !exists { - return errors.Errorf("failed to find expected context %s", config.CurrentContext) - } - expectedCluster := expectedCtx.Cluster - currentCtx, exists := currentConfig.Contexts[currentConfig.CurrentContext] - if !exists { - return errors.Errorf("failed to find CurrentContext in Contexts of the kubeconfig file %s", kubeConfigFilePath) - } - currentCluster := currentCtx.Cluster - if currentConfig.Clusters[currentCluster] == nil { - return errors.Errorf("failed to find the given CurrentContext Cluster in Clusters of the kubeconfig file %s", kubeConfigFilePath) - } - - // Make sure the compared CAs are whitespace-trimmed. The function clientcmd.LoadFromFile() just decodes - // the base64 CA and places it raw in the v1.Config object. In case the user has extra whitespace - // in the CA they used to create a kubeconfig this comparison to a generated v1.Config will otherwise fail. - caCurrent := bytes.TrimSpace(currentConfig.Clusters[currentCluster].CertificateAuthorityData) - caExpected := bytes.TrimSpace(config.Clusters[expectedCluster].CertificateAuthorityData) - - // If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale - if !bytes.Equal(caCurrent, caExpected) { - return errors.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath) - } - // If the current API Server location on disk doesn't match the expected API server, show a warning - if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server { - klog.Warningf("a kubeconfig file %q exists already but has an unexpected API Server URL: expected: %s, got: %s", - kubeConfigFilePath, config.Clusters[expectedCluster].Server, currentConfig.Clusters[currentCluster].Server) - } - - return nil -} - -// createKubeConfigFileIfNotExists saves the KubeConfig object into a file if there isn't any file at the given path. -// If there already is a kubeconfig file at the given path; kubeadm tries to load it and check if the values in the -// existing and the expected config equals. If they do; kubeadm will just skip writing the file as it's up-to-date, -// but if a file exists but has old content or isn't a kubeconfig file, this function returns an error. -func createKubeConfigFileIfNotExists(outDir, filename string, config *clientcmdapi.Config) error { - kubeConfigFilePath := filepath.Join(outDir, filename) - - err := validateKubeConfig(outDir, filename, config) - if err != nil { - // Check if the file exist, and if it doesn't, just write it to disk - if !os.IsNotExist(err) { - return err - } - fmt.Printf("[kubeconfig] Writing %q kubeconfig file\n", filename) - err = kubeconfigutil.WriteToDisk(kubeConfigFilePath, config) - if err != nil { - return errors.Wrapf(err, "failed to save kubeconfig file %q on disk", kubeConfigFilePath) - } - return nil - } - // kubeadm doesn't validate the existing kubeconfig file more than this (kubeadm trusts the client certs to be valid) - // Basically, if we find a kubeconfig file with the same path; the same CA cert and the same server URL; - // kubeadm thinks those files are equal and doesn't bother writing a new file - fmt.Printf("[kubeconfig] Using existing kubeconfig file: %q\n", kubeConfigFilePath) - - return nil -} - -// WriteKubeConfigWithClientCert writes a kubeconfig file - with a client certificate as authentication info - to the given writer. -func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName string, organizations []string) error { - - // creates the KubeConfigSpecs, actualized for the current InitConfiguration - caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) - if err != nil { - return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded") - } - // Validate period - certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert) - - controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) - if err != nil { - return err - } - - spec := &kubeConfigSpec{ - ClientName: clientName, - APIServer: controlPlaneEndpoint, - CACert: caCert, - ClientCertAuth: &clientCertAuth{ - CAKey: caKey, - Organizations: organizations, - }, - } - - return writeKubeConfigFromSpec(out, spec, cfg.ClusterName) -} - -// WriteKubeConfigWithToken writes a kubeconfig file - with a token as client authentication info - to the given writer. -func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.InitConfiguration, clientName, token string) error { - - // creates the KubeConfigSpecs, actualized for the current InitConfiguration - caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) - if err != nil { - return errors.Wrap(err, "couldn't create a kubeconfig; the CA files couldn't be loaded") - } - // Validate period - certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert) - - controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) - if err != nil { - return err - } - - spec := &kubeConfigSpec{ - ClientName: clientName, - APIServer: controlPlaneEndpoint, - CACert: caCert, - TokenAuth: &tokenAuth{ - Token: token, - }, - } - - return writeKubeConfigFromSpec(out, spec, cfg.ClusterName) -} - -// writeKubeConfigFromSpec creates a kubeconfig object from a kubeConfigSpec and writes it to the given writer. -func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername string) error { - - // builds the KubeConfig object - config, err := buildKubeConfigFromSpec(spec, clustername) - if err != nil { - return err - } - - // writes the kubeconfig to disk if it not exists - configBytes, err := clientcmd.Write(*config) - if err != nil { - return errors.Wrap(err, "failure while serializing admin kubeconfig") - } - - fmt.Fprintln(out, string(configBytes)) - return nil -} - -// ValidateKubeconfigsForExternalCA check if the kubeconfig file exist and has the expected CA and server URL using kubeadmapi.InitConfiguration. -func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error { - // Creates a kubeconfig file with the target CA and server URL - // to be used as a input for validating user provided kubeconfig files - caCert, err := pkiutil.TryLoadCertFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName) - if err != nil { - return errors.Wrapf(err, "the CA file couldn't be loaded") - } - // Validate period - certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert) - - // validate user provided kubeconfig files for the scheduler and controller-manager - localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) - if err != nil { - return err - } - - validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert)) - kubeConfigFileNamesLocal := []string{ - kubeadmconstants.ControllerManagerKubeConfigFileName, - kubeadmconstants.SchedulerKubeConfigFileName, - } - - for _, kubeConfigFileName := range kubeConfigFileNamesLocal { - if err = validateKubeConfig(outDir, kubeConfigFileName, validationConfigLocal); err != nil { - return errors.Wrapf(err, "the %s file does not exists or it is not valid", kubeConfigFileName) - } - } - - // validate user provided kubeconfig files for the kubelet and admin - controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) - if err != nil { - return err - } - - validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert)) - kubeConfigFileNamesCPE := []string{ - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.KubeletKubeConfigFileName, - } - - for _, kubeConfigFileName := range kubeConfigFileNamesCPE { - if err = validateKubeConfig(outDir, kubeConfigFileName, validationConfigCPE); err != nil { - return errors.Wrapf(err, "the %s file does not exists or it is not valid", kubeConfigFileName) - } - } - - return nil -} - -func getKubeConfigSpecsBase(cfg *kubeadmapi.InitConfiguration) (map[string]*kubeConfigSpec, error) { - controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) - if err != nil { - return nil, err - } - localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) - if err != nil { - return nil, err - } - - return map[string]*kubeConfigSpec{ - kubeadmconstants.AdminKubeConfigFileName: { - APIServer: controlPlaneEndpoint, - ClientName: "kubernetes-admin", - ClientCertAuth: &clientCertAuth{ - Organizations: []string{kubeadmconstants.SystemPrivilegedGroup}, - }, - }, - kubeadmconstants.KubeletKubeConfigFileName: { - APIServer: controlPlaneEndpoint, - ClientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name), - ClientCertAuth: &clientCertAuth{ - Organizations: []string{kubeadmconstants.NodesGroup}, - }, - }, - kubeadmconstants.ControllerManagerKubeConfigFileName: { - APIServer: localAPIEndpoint, - ClientName: kubeadmconstants.ControllerManagerUser, - ClientCertAuth: &clientCertAuth{}, - }, - kubeadmconstants.SchedulerKubeConfigFileName: { - APIServer: localAPIEndpoint, - ClientName: kubeadmconstants.SchedulerUser, - ClientCertAuth: &clientCertAuth{}, - }, - }, nil -} - -func createKubeConfigAndCSR(kubeConfigDir string, kubeadmConfig *kubeadmapi.InitConfiguration, name string, spec *kubeConfigSpec) error { - if kubeConfigDir == "" { - return errors.Errorf("%s: kubeConfigDir was empty", errInvalid) - } - if kubeadmConfig == nil { - return errors.Errorf("%s: kubeadmConfig was nil", errInvalid) - } - if name == "" { - return errors.Errorf("%s: name was empty", errInvalid) - } - if spec == nil { - return errors.Errorf("%s: spec was nil", errInvalid) - } - kubeConfigPath := filepath.Join(kubeConfigDir, name) - if _, err := os.Stat(kubeConfigPath); err == nil { - return errors.Errorf("%s: kube config: %s", errExist, kubeConfigPath) - } else if !os.IsNotExist(err) { - return errors.Wrapf(err, "unexpected error while checking if file exists: %s", kubeConfigPath) - } - if pkiutil.CSROrKeyExist(kubeConfigDir, name) { - return errors.Errorf("%s: csr: %s", errExist, kubeConfigPath) - } - - clientCertConfig := newClientCertConfigFromKubeConfigSpec(spec) - - clientKey, err := pkiutil.NewPrivateKey(clientCertConfig.PublicKeyAlgorithm) - if err != nil { - return err - } - clientCSR, err := pkiutil.NewCSR(clientCertConfig, clientKey) - if err != nil { - return err - } - - encodedClientKey, err := keyutil.MarshalPrivateKeyToPEM(clientKey) - if err != nil { - return err - } - - var ( - emptyCACert []byte - emptyClientCert []byte - ) - - // create a kubeconfig with the client certs - config := kubeconfigutil.CreateWithCerts( - spec.APIServer, - kubeadmConfig.ClusterName, - spec.ClientName, - emptyCACert, - encodedClientKey, - emptyClientCert, - ) - - if err := kubeconfigutil.WriteToDisk(kubeConfigPath, config); err != nil { - return err - } - // Write CSR to disk - if err := pkiutil.WriteCSR(kubeConfigDir, name, clientCSR); err != nil { - return err - } - return nil -} - -// CreateDefaultKubeConfigsAndCSRFiles is used in ExternalCA mode to create -// kubeconfig files and adjacent CSR files. -func CreateDefaultKubeConfigsAndCSRFiles(out io.Writer, kubeConfigDir string, kubeadmConfig *kubeadmapi.InitConfiguration) error { - kubeConfigs, err := getKubeConfigSpecsBase(kubeadmConfig) - if err != nil { - return err - } - if out != nil { - fmt.Fprintf(out, "generating keys and CSRs in %s\n", kubeConfigDir) - } - for name, spec := range kubeConfigs { - if err := createKubeConfigAndCSR(kubeConfigDir, kubeadmConfig, name, spec); err != nil { - return err - } - if out != nil { - fmt.Fprintf(out, " %s\n", name) - } - } - return nil -} diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go deleted file mode 100644 index df5d47b64e12b..0000000000000 --- a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go +++ /dev/null @@ -1,692 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeconfig - -import ( - "bytes" - "crypto" - "crypto/x509" - "fmt" - "io" - "os" - "path/filepath" - "reflect" - "testing" - - "github.com/lithammer/dedent" - - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" - kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig" -) - -func TestGetKubeConfigSpecsFailsIfCADoesntExists(t *testing.T) { - // Create temp folder for the test case (without a CA) - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Creates an InitConfiguration pointing to the pkidir folder - cfg := &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: tmpdir, - }, - } - - // Executes getKubeConfigSpecs - if _, err := getKubeConfigSpecs(cfg); err == nil { - t.Error("getKubeConfigSpecs didnt failed when expected") - } -} - -func TestGetKubeConfigSpecs(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Adds a pki folder with a ca certs to the temp folder - pkidir := testutil.SetupPkiDirWithCertificateAuthority(t, tmpdir) - - // Creates InitConfigurations pointing to the pkidir folder - cfgs := []*kubeadmapi.InitConfiguration{ - { - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: pkidir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, - }, - { - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "api.k8s.io", - CertificatesDir: pkidir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, - }, - { - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "api.k8s.io:4321", - CertificatesDir: pkidir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, - }, - { - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "api.k8s.io", - CertificatesDir: pkidir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, - }, - { - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "api.k8s.io:4321", - CertificatesDir: pkidir, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-node-name"}, - }, - } - - for i, cfg := range cfgs { - var assertions = []struct { - kubeConfigFile string - clientName string - organizations []string - }{ - { - kubeConfigFile: kubeadmconstants.AdminKubeConfigFileName, - clientName: "kubernetes-admin", - organizations: []string{kubeadmconstants.SystemPrivilegedGroup}, - }, - { - kubeConfigFile: kubeadmconstants.KubeletKubeConfigFileName, - clientName: fmt.Sprintf("%s%s", kubeadmconstants.NodesUserPrefix, cfg.NodeRegistration.Name), - organizations: []string{kubeadmconstants.NodesGroup}, - }, - { - kubeConfigFile: kubeadmconstants.ControllerManagerKubeConfigFileName, - clientName: kubeadmconstants.ControllerManagerUser, - }, - { - kubeConfigFile: kubeadmconstants.SchedulerKubeConfigFileName, - clientName: kubeadmconstants.SchedulerUser, - }, - } - - for _, assertion := range assertions { - t.Run(fmt.Sprintf("%d-%s", i, assertion.clientName), func(t *testing.T) { - // Executes getKubeConfigSpecs - specs, err := getKubeConfigSpecs(cfg) - if err != nil { - t.Fatal("getKubeConfigSpecs failed!") - } - - var spec *kubeConfigSpec - var ok bool - - // assert the spec for the kubeConfigFile exists - if spec, ok = specs[assertion.kubeConfigFile]; !ok { - t.Errorf("getKubeConfigSpecs didn't create spec for %s ", assertion.kubeConfigFile) - return - } - - // Assert clientName - if spec.ClientName != assertion.clientName { - t.Errorf("getKubeConfigSpecs for %s clientName is %s, expected %s", assertion.kubeConfigFile, spec.ClientName, assertion.clientName) - } - - // Assert Organizations - if spec.ClientCertAuth == nil || !reflect.DeepEqual(spec.ClientCertAuth.Organizations, assertion.organizations) { - t.Errorf("getKubeConfigSpecs for %s Organizations is %v, expected %v", assertion.kubeConfigFile, spec.ClientCertAuth.Organizations, assertion.organizations) - } - - // Asserts InitConfiguration values injected into spec - controlPlaneEndpoint, err := kubeadmutil.GetControlPlaneEndpoint(cfg.ControlPlaneEndpoint, &cfg.LocalAPIEndpoint) - if err != nil { - t.Error(err) - } - localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint) - if err != nil { - t.Error(err) - } - - switch assertion.kubeConfigFile { - case kubeadmconstants.AdminKubeConfigFileName, kubeadmconstants.KubeletKubeConfigFileName: - if spec.APIServer != controlPlaneEndpoint { - t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s", - assertion.kubeConfigFile, controlPlaneEndpoint, spec.APIServer) - } - case kubeadmconstants.ControllerManagerKubeConfigFileName, kubeadmconstants.SchedulerKubeConfigFileName: - if spec.APIServer != localAPIEndpoint { - t.Errorf("expected getKubeConfigSpecs for %s to set cfg.APIServer to %s, got %s", - assertion.kubeConfigFile, localAPIEndpoint, spec.APIServer) - } - } - - // Asserts CA certs and CA keys loaded into specs - if spec.CACert == nil { - t.Errorf("getKubeConfigSpecs didn't loaded CACert into spec for %s!", assertion.kubeConfigFile) - } - if spec.ClientCertAuth == nil || spec.ClientCertAuth.CAKey == nil { - t.Errorf("getKubeConfigSpecs didn't loaded CAKey into spec for %s!", assertion.kubeConfigFile) - } - }) - } - } -} - -func TestBuildKubeConfigFromSpecWithClientAuth(t *testing.T) { - // Creates a CA - caCert, caKey := certstestutil.SetupCertificateAuthority(t) - - // Executes buildKubeConfigFromSpec passing a KubeConfigSpec with a ClientAuth - config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "myClientName", "test-cluster", "myOrg1", "myOrg2") - - // Asserts spec data are propagated to the kubeconfig - kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) - kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myClientName", "myOrg1", "myOrg2") -} - -func TestBuildKubeConfigFromSpecWithTokenAuth(t *testing.T) { - // Creates a CA - caCert, _ := certstestutil.SetupCertificateAuthority(t) - - // Executes buildKubeConfigFromSpec passing a KubeConfigSpec with a Token - config := setupdKubeConfigWithTokenAuth(t, caCert, "https://1.2.3.4:1234", "myClientName", "123456", "test-cluster") - - // Asserts spec data are propagated to the kubeconfig - kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) - kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myClientName", "123456") -} - -func TestCreateKubeConfigFileIfNotExists(t *testing.T) { - - // Creates a CAs - caCert, caKey := certstestutil.SetupCertificateAuthority(t) - anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthority(t) - - // build kubeconfigs (to be used to test kubeconfigs equality/not equality) - config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1", "myOrg2") - configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1", "myOrg2") - configWithAnotherClusterAddress := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://3.4.5.6:3456", "myOrg1", "test-cluster", "myOrg2") - invalidConfig := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1", "myOrg2") - invalidConfig.CurrentContext = "invalid context" - - var tests = []struct { - name string - existingKubeConfig *clientcmdapi.Config - kubeConfig *clientcmdapi.Config - expectedError bool - }{ - { // if there is no existing KubeConfig, creates the kubeconfig - name: "KubeConfig doesn't exist", - kubeConfig: config, - }, - { // if KubeConfig is invalid raise error - name: "KubeConfig is invalid", - existingKubeConfig: invalidConfig, - kubeConfig: invalidConfig, - expectedError: true, - }, - { // if KubeConfig is equal to the existingKubeConfig - refers to the same cluster -, use the existing (Test idempotency) - name: "KubeConfig refers to the same cluster", - existingKubeConfig: config, - kubeConfig: config, - }, - { // if KubeConfig is not equal to the existingKubeConfig - refers to the another cluster (a cluster with another Ca) -, raise error - name: "KubeConfig refers to the cluster with another CA", - existingKubeConfig: config, - kubeConfig: configWithAnotherClusterCa, - expectedError: true, - }, - { // if KubeConfig is not equal to the existingKubeConfig - tollerate custom server addresses - name: "KubeConfig referst to the cluster with another address", - existingKubeConfig: config, - kubeConfig: configWithAnotherClusterAddress, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Writes the existing kubeconfig file to disk - if test.existingKubeConfig != nil { - if err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.existingKubeConfig); err != nil { - t.Errorf("createKubeConfigFileIfNotExists failed") - } - } - - // Writes the kubeconfig file to disk - err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.kubeConfig) - if test.expectedError && err == nil { - t.Errorf("createKubeConfigFileIfNotExists didn't failed when expected to fail") - } - if !test.expectedError && err != nil { - t.Errorf("createKubeConfigFileIfNotExists failed") - } - - // Assert that the created file is there - testutil.AssertFileExists(t, tmpdir, "test.conf") - }) - } -} - -func TestCreateKubeconfigFilesAndWrappers(t *testing.T) { - var tests = []struct { - name string - createKubeConfigFunction func(outDir string, cfg *kubeadmapi.InitConfiguration) error - expectedFiles []string - expectedError bool - }{ - { // Test createKubeConfigFiles fails for unknown kubeconfig is requested - name: "createKubeConfigFiles", - createKubeConfigFunction: func(outDir string, cfg *kubeadmapi.InitConfiguration) error { - return createKubeConfigFiles(outDir, cfg, "unknown.conf") - }, - expectedError: true, - }, - { // Test CreateJoinControlPlaneKubeConfigFiles (wrapper to createKubeConfigFile) - name: "CreateJoinControlPlaneKubeConfigFiles", - createKubeConfigFunction: CreateJoinControlPlaneKubeConfigFiles, - expectedFiles: []string{ - kubeadmconstants.AdminKubeConfigFileName, - kubeadmconstants.ControllerManagerKubeConfigFileName, - kubeadmconstants.SchedulerKubeConfigFileName, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Create temp folder for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Adds a pki folder with a ca certs to the temp folder - pkidir := testutil.SetupPkiDirWithCertificateAuthority(t, tmpdir) - - // Creates an InitConfiguration pointing to the pkidir folder - cfg := &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: pkidir, - }, - } - - // Execs the createKubeConfigFunction - err := test.createKubeConfigFunction(tmpdir, cfg) - if test.expectedError && err == nil { - t.Errorf("createKubeConfigFunction didn't failed when expected to fail") - return - } - if !test.expectedError && err != nil { - t.Errorf("createKubeConfigFunction failed") - return - } - - // Assert expected files are there - testutil.AssertFileExists(t, tmpdir, test.expectedFiles...) - }) - } -} - -func TestWriteKubeConfigFailsIfCADoesntExists(t *testing.T) { - // Temporary folders for the test case (without a CA) - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Creates an InitConfiguration pointing to the tmpdir folder - cfg := &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: tmpdir, - }, - } - - var tests = []struct { - name string - writeKubeConfigFunction func(out io.Writer) error - }{ - { - name: "WriteKubeConfigWithClientCert", - writeKubeConfigFunction: func(out io.Writer) error { - return WriteKubeConfigWithClientCert(out, cfg, "myUser", []string{"myOrg"}) - }, - }, - { - name: "WriteKubeConfigWithToken", - writeKubeConfigFunction: func(out io.Writer) error { - return WriteKubeConfigWithToken(out, cfg, "myUser", "12345") - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - buf := new(bytes.Buffer) - - // executes writeKubeConfigFunction - if err := test.writeKubeConfigFunction(buf); err == nil { - t.Error("writeKubeConfigFunction didnt failed when expected") - } - }) - } -} - -func TestWriteKubeConfig(t *testing.T) { - // Temporary folders for the test case - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // Adds a pki folder with a ca cert to the temp folder - pkidir := testutil.SetupPkiDirWithCertificateAuthority(t, tmpdir) - - // Retrieves ca cert for assertions - caCert, _, err := pkiutil.TryLoadCertAndKeyFromDisk(pkidir, kubeadmconstants.CACertAndKeyBaseName) - if err != nil { - t.Fatalf("couldn't retrieve ca cert: %v", err) - } - - // Creates an InitConfiguration pointing to the pkidir folder - cfg := &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: pkidir, - }, - } - - var tests = []struct { - name string - writeKubeConfigFunction func(out io.Writer) error - withClientCert bool - withToken bool - }{ - { - name: "WriteKubeConfigWithClientCert", - writeKubeConfigFunction: func(out io.Writer) error { - return WriteKubeConfigWithClientCert(out, cfg, "myUser", []string{"myOrg"}) - }, - withClientCert: true, - }, - { - name: "WriteKubeConfigWithToken", - writeKubeConfigFunction: func(out io.Writer) error { - return WriteKubeConfigWithToken(out, cfg, "myUser", "12345") - }, - withToken: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - buf := new(bytes.Buffer) - - // executes writeKubeConfigFunction - if err := test.writeKubeConfigFunction(buf); err != nil { - t.Error("writeKubeConfigFunction failed") - return - } - - // reads kubeconfig written to stdout - config, err := clientcmd.Load(buf.Bytes()) - if err != nil { - t.Errorf("Couldn't read kubeconfig file from buffer: %v", err) - return - } - - // checks that CLI flags are properly propagated - kubeconfigtestutil.AssertKubeConfigCurrentCluster(t, config, "https://1.2.3.4:1234", caCert) - - if test.withClientCert { - // checks that kubeconfig files have expected client cert - kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithClientCert(t, config, caCert, "myUser") - } - - if test.withToken { - // checks that kubeconfig files have expected token - kubeconfigtestutil.AssertKubeConfigCurrentAuthInfoWithToken(t, config, "myUser", "12345") - } - }) - } -} - -func TestValidateKubeConfig(t *testing.T) { - caCert, caKey := certstestutil.SetupCertificateAuthority(t) - anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthority(t) - - config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") - configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") - configWithAnotherServerURL := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://4.3.2.1:4321", "test-cluster", "myOrg1") - - // create a valid config but with whitespace around the CA PEM. - // validateKubeConfig() should tollerate that. - configWhitespace := config.DeepCopy() - configWhitespaceCtx := configWhitespace.Contexts[configWhitespace.CurrentContext] - configWhitespaceCA := string(configWhitespace.Clusters[configWhitespaceCtx.Cluster].CertificateAuthorityData) - configWhitespaceCA = "\n" + configWhitespaceCA + "\n" - configWhitespace.Clusters[configWhitespaceCtx.Cluster].CertificateAuthorityData = []byte(configWhitespaceCA) - - tests := map[string]struct { - existingKubeConfig *clientcmdapi.Config - kubeConfig *clientcmdapi.Config - expectedError bool - }{ - "kubeconfig don't exist": { - kubeConfig: config, - expectedError: true, - }, - "kubeconfig exist and has invalid ca": { - existingKubeConfig: configWithAnotherClusterCa, - kubeConfig: config, - expectedError: true, - }, - "kubeconfig exist and has a different server url": { - existingKubeConfig: configWithAnotherServerURL, - kubeConfig: config, - }, - "kubeconfig exist and is valid": { - existingKubeConfig: config, - kubeConfig: config, - expectedError: false, - }, - "kubeconfig exist and is valid even if its CA contains whitespace": { - existingKubeConfig: configWhitespace, - kubeConfig: config, - expectedError: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - if test.existingKubeConfig != nil { - if err := createKubeConfigFileIfNotExists(tmpdir, "test.conf", test.existingKubeConfig); err != nil { - t.Errorf("createKubeConfigFileIfNotExists failed") - } - } - - err := validateKubeConfig(tmpdir, "test.conf", test.kubeConfig) - if (err != nil) != test.expectedError { - t.Fatalf(dedent.Dedent( - "validateKubeConfig failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), - name, - test.expectedError, - (err != nil), - err, - ) - } - }) - } -} - -func TestValidateKubeconfigsForExternalCA(t *testing.T) { - tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) - pkiDir := filepath.Join(tmpDir, "pki") - - initConfig := &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - CertificatesDir: pkiDir, - }, - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 1234, - AdvertiseAddress: "1.2.3.4", - }, - } - - // creates CA, write to pkiDir and remove ca.key to get into external CA condition - caCert, caKey := certstestutil.SetupCertificateAuthority(t) - if err := pkiutil.WriteCertAndKey(pkiDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil { - t.Fatalf("failure while saving CA certificate and key: %v", err) - } - if err := os.Remove(filepath.Join(pkiDir, kubeadmconstants.CAKeyName)); err != nil { - t.Fatalf("failure while deleting ca.key: %v", err) - } - - // create a valid config - config := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") - - // create a config with another CA - anotherCaCert, anotherCaKey := certstestutil.SetupCertificateAuthority(t) - configWithAnotherClusterCa := setupdKubeConfigWithClientAuth(t, anotherCaCert, anotherCaKey, "https://1.2.3.4:1234", "test-cluster", "myOrg1") - - // create a config with another server URL - configWithAnotherServerURL := setupdKubeConfigWithClientAuth(t, caCert, caKey, "https://4.3.2.1:4321", "test-cluster", "myOrg1") - - tests := map[string]struct { - filesToWrite map[string]*clientcmdapi.Config - initConfig *kubeadmapi.InitConfiguration - expectedError bool - }{ - "files don't exist": { - initConfig: initConfig, - expectedError: true, - }, - "some files don't exist": { - filesToWrite: map[string]*clientcmdapi.Config{ - kubeadmconstants.AdminKubeConfigFileName: config, - kubeadmconstants.KubeletKubeConfigFileName: config, - }, - initConfig: initConfig, - expectedError: true, - }, - "some files have invalid CA": { - filesToWrite: map[string]*clientcmdapi.Config{ - kubeadmconstants.AdminKubeConfigFileName: config, - kubeadmconstants.KubeletKubeConfigFileName: config, - kubeadmconstants.ControllerManagerKubeConfigFileName: configWithAnotherClusterCa, - kubeadmconstants.SchedulerKubeConfigFileName: config, - }, - initConfig: initConfig, - expectedError: true, - }, - "some files have a different Server URL": { - filesToWrite: map[string]*clientcmdapi.Config{ - kubeadmconstants.AdminKubeConfigFileName: config, - kubeadmconstants.KubeletKubeConfigFileName: config, - kubeadmconstants.ControllerManagerKubeConfigFileName: config, - kubeadmconstants.SchedulerKubeConfigFileName: configWithAnotherServerURL, - }, - initConfig: initConfig, - }, - "all files are valid": { - filesToWrite: map[string]*clientcmdapi.Config{ - kubeadmconstants.AdminKubeConfigFileName: config, - kubeadmconstants.KubeletKubeConfigFileName: config, - kubeadmconstants.ControllerManagerKubeConfigFileName: config, - kubeadmconstants.SchedulerKubeConfigFileName: config, - }, - initConfig: initConfig, - expectedError: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - for name, config := range test.filesToWrite { - if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil { - t.Errorf("createKubeConfigFileIfNotExists failed: %v", err) - } - } - - err := ValidateKubeconfigsForExternalCA(tmpdir, test.initConfig) - if (err != nil) != test.expectedError { - t.Fatalf(dedent.Dedent( - "ValidateKubeconfigsForExternalCA failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), - name, - test.expectedError, - (err != nil), - err, - ) - } - }) - } -} - -// setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth -func setupdKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey crypto.Signer, APIServer, clientName, clustername string, organizations ...string) *clientcmdapi.Config { - spec := &kubeConfigSpec{ - CACert: caCert, - APIServer: APIServer, - ClientName: clientName, - ClientCertAuth: &clientCertAuth{ - CAKey: caKey, - Organizations: organizations, - }, - } - - config, err := buildKubeConfigFromSpec(spec, clustername) - if err != nil { - t.Fatal("buildKubeConfigFromSpec failed!") - } - - return config -} - -// setupdKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token -func setupdKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, APIServer, clientName, token, clustername string) *clientcmdapi.Config { - spec := &kubeConfigSpec{ - CACert: caCert, - APIServer: APIServer, - ClientName: clientName, - TokenAuth: &tokenAuth{ - Token: token, - }, - } - - config, err := buildKubeConfigFromSpec(spec, clustername) - if err != nil { - t.Fatal("buildKubeConfigFromSpec failed!") - } - - return config -} diff --git a/cmd/kubeadm/app/phases/kubelet/BUILD b/cmd/kubeadm/app/phases/kubelet/BUILD deleted file mode 100644 index e989be8ec9490..0000000000000 --- a/cmd/kubeadm/app/phases/kubelet/BUILD +++ /dev/null @@ -1,63 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "config.go", - "flags.go", - "kubelet.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/initsystem:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "config_test.go", - "flags_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/phases/kubelet/config.go b/cmd/kubeadm/app/phases/kubelet/config.go deleted file mode 100644 index a0a07612901bb..0000000000000 --- a/cmd/kubeadm/app/phases/kubelet/config.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// WriteConfigToDisk writes the kubelet config object down to a file -// Used at "kubeadm init" and "kubeadm upgrade" time -func WriteConfigToDisk(cfg *kubeadmapi.ClusterConfiguration, kubeletDir string) error { - kubeletCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup] - if !ok { - return errors.New("no kubelet component config found") - } - - kubeletBytes, err := kubeletCfg.Marshal() - if err != nil { - return err - } - - return writeConfigBytesToDisk(kubeletBytes, kubeletDir) -} - -// CreateConfigMap creates a ConfigMap with the generic kubelet configuration. -// Used at "kubeadm init" and "kubeadm upgrade" time -func CreateConfigMap(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface) error { - - k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) - if err != nil { - return err - } - - configMapName := kubeadmconstants.GetKubeletConfigMapName(k8sVersion) - fmt.Printf("[kubelet] Creating a ConfigMap %q in namespace %s with the configuration for the kubelets in the cluster\n", configMapName, metav1.NamespaceSystem) - - kubeletCfg, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup] - if !ok { - return errors.New("no kubelet component config found in the active component config set") - } - - kubeletBytes, err := kubeletCfg.Marshal() - if err != nil { - return err - } - - configMap := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletBytes), - }, - } - - if !kubeletCfg.IsUserSupplied() { - componentconfigs.SignConfigMap(configMap) - } - - if err := apiclient.CreateOrUpdateConfigMap(client, configMap); err != nil { - return err - } - - if err := createConfigMapRBACRules(client, k8sVersion); err != nil { - return errors.Wrap(err, "error creating kubelet configuration configmap RBAC rules") - } - return nil -} - -// createConfigMapRBACRules creates the RBAC rules for exposing the base kubelet ConfigMap in the kube-system namespace to unauthenticated users -func createConfigMapRBACRules(client clientset.Interface, k8sVersion *version.Version) error { - if err := apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapRBACName(k8sVersion), - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{kubeadmconstants.GetKubeletConfigMapName(k8sVersion)}, - }, - }, - }); err != nil { - return err - } - - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapRBACName(k8sVersion), - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: configMapRBACName(k8sVersion), - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodesGroup, - }, - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, - }, - }, - }) -} - -// DownloadConfig downloads the kubelet configuration from a ConfigMap and writes it to disk. -// DEPRECATED: Do not use in new code! -func DownloadConfig(client clientset.Interface, kubeletVersionStr string, kubeletDir string) error { - // Parse the desired kubelet version - kubeletVersion, err := version.ParseSemantic(kubeletVersionStr) - if err != nil { - return err - } - - // Download the ConfigMap from the cluster based on what version the kubelet is - configMapName := kubeadmconstants.GetKubeletConfigMapName(kubeletVersion) - - fmt.Printf("[kubelet-start] Downloading configuration for the kubelet from the %q ConfigMap in the %s namespace\n", - configMapName, metav1.NamespaceSystem) - - kubeletCfgMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, configMapName) - if err != nil { - return err - } - - // Check for the key existence, otherwise we'll panic here - kubeletCfg, ok := kubeletCfgMap.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey] - if !ok { - return errors.Errorf("no key %q found in config map %s", kubeadmconstants.KubeletBaseConfigurationConfigMapKey, configMapName) - } - - return writeConfigBytesToDisk([]byte(kubeletCfg), kubeletDir) -} - -// configMapRBACName returns the name for the Role/RoleBinding for the kubelet config configmap for the right branch of k8s -func configMapRBACName(k8sVersion *version.Version) string { - return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor()) -} - -// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file -func writeConfigBytesToDisk(b []byte, kubeletDir string) error { - configFile := filepath.Join(kubeletDir, kubeadmconstants.KubeletConfigurationFileName) - fmt.Printf("[kubelet-start] Writing kubelet configuration to file %q\n", configFile) - - // creates target folder if not already exists - if err := os.MkdirAll(kubeletDir, 0700); err != nil { - return errors.Wrapf(err, "failed to create directory %q", kubeletDir) - } - - if err := ioutil.WriteFile(configFile, b, 0644); err != nil { - return errors.Wrapf(err, "failed to write kubelet configuration to the file %q", configFile) - } - return nil -} diff --git a/cmd/kubeadm/app/phases/kubelet/config_test.go b/cmd/kubeadm/app/phases/kubelet/config_test.go deleted file mode 100644 index 569bb195e020a..0000000000000 --- a/cmd/kubeadm/app/phases/kubelet/config_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -func TestCreateConfigMap(t *testing.T) { - nodeName := "fake-node" - client := fake.NewSimpleClientset() - client.PrependReactor("get", "nodes", func(action core.Action) (bool, runtime.Object, error) { - return true, &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - Spec: v1.NodeSpec{}, - }, nil - }) - client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - - clusterCfg := &kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: constants.CurrentKubernetesVersion.String(), - } - internalcfg, err := configutil.DefaultedInitConfiguration(&kubeadmapiv1beta2.InitConfiguration{}, clusterCfg) - if err != nil { - t.Fatalf("unexpected failure by DefaultedInitConfiguration: %v", err) - } - - if err := CreateConfigMap(&internalcfg.ClusterConfiguration, client); err != nil { - t.Errorf("CreateConfigMap: unexpected error %v", err) - } -} - -func TestCreateConfigMapRBACRules(t *testing.T) { - client := fake.NewSimpleClientset() - client.PrependReactor("create", "roles", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - client.PrependReactor("create", "rolebindings", func(action core.Action) (bool, runtime.Object, error) { - return true, nil, nil - }) - - if err := createConfigMapRBACRules(client, version.MustParseSemantic("v1.11.0")); err != nil { - t.Errorf("createConfigMapRBACRules: unexpected error %v", err) - } -} diff --git a/cmd/kubeadm/app/phases/kubelet/flags.go b/cmd/kubeadm/app/phases/kubelet/flags.go deleted file mode 100644 index 6288d498ba02e..0000000000000 --- a/cmd/kubeadm/app/phases/kubelet/flags.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -type kubeletFlagsOpts struct { - nodeRegOpts *kubeadmapi.NodeRegistrationOptions - pauseImage string - registerTaintsUsingFlags bool -} - -// GetNodeNameAndHostname obtains the name for this Node using the following precedence -// (from lower to higher): -// - actual hostname -// - NodeRegistrationOptions.Name (same as "--node-name" passed to "kubeadm init/join") -// - "hostname-overide" flag in NodeRegistrationOptions.KubeletExtraArgs -// It also returns the hostname or an error if getting the hostname failed. -func GetNodeNameAndHostname(cfg *kubeadmapi.NodeRegistrationOptions) (string, string, error) { - hostname, err := kubeadmutil.GetHostname("") - nodeName := hostname - if cfg.Name != "" { - nodeName = cfg.Name - } - if name, ok := cfg.KubeletExtraArgs["hostname-override"]; ok { - nodeName = name - } - return nodeName, hostname, err -} - -// WriteKubeletDynamicEnvFile writes an environment file with dynamic flags to the kubelet. -// Used at "kubeadm init" and "kubeadm join" time. -func WriteKubeletDynamicEnvFile(cfg *kubeadmapi.ClusterConfiguration, nodeReg *kubeadmapi.NodeRegistrationOptions, registerTaintsUsingFlags bool, kubeletDir string) error { - flagOpts := kubeletFlagsOpts{ - nodeRegOpts: nodeReg, - pauseImage: images.GetPauseImage(cfg), - registerTaintsUsingFlags: registerTaintsUsingFlags, - } - stringMap := buildKubeletArgMap(flagOpts) - argList := kubeadmutil.BuildArgumentListFromMap(stringMap, nodeReg.KubeletExtraArgs) - envFileContent := fmt.Sprintf("%s=%q\n", constants.KubeletEnvFileVariableName, strings.Join(argList, " ")) - - return writeKubeletFlagBytesToDisk([]byte(envFileContent), kubeletDir) -} - -//buildKubeletArgMapCommon takes a kubeletFlagsOpts object and builds based on that a string-string map with flags -//that are common to both Linux and Windows -func buildKubeletArgMapCommon(opts kubeletFlagsOpts) map[string]string { - kubeletFlags := map[string]string{} - - if opts.nodeRegOpts.CRISocket == constants.DefaultDockerCRISocket { - // These flags should only be set when running docker - kubeletFlags["network-plugin"] = "cni" - if opts.pauseImage != "" { - kubeletFlags["pod-infra-container-image"] = opts.pauseImage - } - } else { - kubeletFlags["container-runtime"] = "remote" - kubeletFlags["container-runtime-endpoint"] = opts.nodeRegOpts.CRISocket - } - - if opts.registerTaintsUsingFlags && opts.nodeRegOpts.Taints != nil && len(opts.nodeRegOpts.Taints) > 0 { - taintStrs := []string{} - for _, taint := range opts.nodeRegOpts.Taints { - taintStrs = append(taintStrs, taint.ToString()) - } - - kubeletFlags["register-with-taints"] = strings.Join(taintStrs, ",") - } - - // Pass the "--hostname-override" flag to the kubelet only if it's different from the hostname - nodeName, hostname, err := GetNodeNameAndHostname(opts.nodeRegOpts) - if err != nil { - klog.Warning(err) - } - if nodeName != hostname { - klog.V(1).Infof("setting kubelet hostname-override to %q", nodeName) - kubeletFlags["hostname-override"] = nodeName - } - - return kubeletFlags -} - -// writeKubeletFlagBytesToDisk writes a byte slice down to disk at the specific location of the kubelet flag overrides file -func writeKubeletFlagBytesToDisk(b []byte, kubeletDir string) error { - kubeletEnvFilePath := filepath.Join(kubeletDir, constants.KubeletEnvFileName) - fmt.Printf("[kubelet-start] Writing kubelet environment file with flags to file %q\n", kubeletEnvFilePath) - - // creates target folder if not already exists - if err := os.MkdirAll(kubeletDir, 0700); err != nil { - return errors.Wrapf(err, "failed to create directory %q", kubeletDir) - } - if err := ioutil.WriteFile(kubeletEnvFilePath, b, 0644); err != nil { - return errors.Wrapf(err, "failed to write kubelet configuration to the file %q", kubeletEnvFilePath) - } - return nil -} - -// buildKubeletArgMap takes a kubeletFlagsOpts object and builds based on that a string-string map with flags -// that should be given to the local kubelet daemon. -func buildKubeletArgMap(opts kubeletFlagsOpts) map[string]string { - return buildKubeletArgMapCommon(opts) -} diff --git a/cmd/kubeadm/app/phases/kubelet/flags_test.go b/cmd/kubeadm/app/phases/kubelet/flags_test.go deleted file mode 100644 index 4917a65f8c86b..0000000000000 --- a/cmd/kubeadm/app/phases/kubelet/flags_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "reflect" - "testing" - - v1 "k8s.io/api/core/v1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestBuildKubeletArgMap(t *testing.T) { - - tests := []struct { - name string - opts kubeletFlagsOpts - expected map[string]string - }{ - { - name: "the simplest case", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/dockershim.sock", - Taints: []v1.Taint{ // This should be ignored as registerTaintsUsingFlags is false - { - Key: "foo", - Value: "bar", - Effect: "baz", - }, - }, - }, - }, - expected: map[string]string{ - "network-plugin": "cni", - }, - }, - { - name: "hostname override from NodeRegistrationOptions.Name", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/dockershim.sock", - Name: "override-name", - }, - }, - expected: map[string]string{ - "network-plugin": "cni", - "hostname-override": "override-name", - }, - }, - { - name: "hostname override from NodeRegistrationOptions.KubeletExtraArgs", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/dockershim.sock", - KubeletExtraArgs: map[string]string{"hostname-override": "override-name"}, - }, - }, - expected: map[string]string{ - "network-plugin": "cni", - "hostname-override": "override-name", - }, - }, - { - name: "external CRI runtime", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/containerd.sock", - }, - }, - expected: map[string]string{ - "container-runtime": "remote", - "container-runtime-endpoint": "/var/run/containerd.sock", - }, - }, - { - name: "register with taints", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/containerd.sock", - Taints: []v1.Taint{ - { - Key: "foo", - Value: "bar", - Effect: "baz", - }, - { - Key: "key", - Value: "val", - Effect: "eff", - }, - }, - }, - registerTaintsUsingFlags: true, - }, - expected: map[string]string{ - "container-runtime": "remote", - "container-runtime-endpoint": "/var/run/containerd.sock", - "register-with-taints": "foo=bar:baz,key=val:eff", - }, - }, - { - name: "pause image is set", - opts: kubeletFlagsOpts{ - nodeRegOpts: &kubeadmapi.NodeRegistrationOptions{ - CRISocket: "/var/run/dockershim.sock", - }, - pauseImage: "gcr.io/pause:3.2", - }, - expected: map[string]string{ - "network-plugin": "cni", - "pod-infra-container-image": "gcr.io/pause:3.2", - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - actual := buildKubeletArgMap(test.opts) - if !reflect.DeepEqual(actual, test.expected) { - t.Errorf( - "failed buildKubeletArgMap:\n\texpected: %v\n\t actual: %v", - test.expected, - actual, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/kubelet/kubelet.go b/cmd/kubeadm/app/phases/kubelet/kubelet.go deleted file mode 100644 index a975920a24aa1..0000000000000 --- a/cmd/kubeadm/app/phases/kubelet/kubelet.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - - "k8s.io/kubernetes/cmd/kubeadm/app/util/initsystem" -) - -// TryStartKubelet attempts to bring up kubelet service -func TryStartKubelet() { - // If we notice that the kubelet service is inactive, try to start it - initSystem, err := initsystem.GetInitSystem() - if err != nil { - fmt.Println("[kubelet-start] no supported init system detected, won't make sure the kubelet is running properly.") - return - } - - if !initSystem.ServiceExists("kubelet") { - fmt.Println("[kubelet-start] couldn't detect a kubelet service, can't make sure the kubelet is running properly.") - } - - // This runs "systemctl daemon-reload && systemctl restart kubelet" - if err := initSystem.ServiceRestart("kubelet"); err != nil { - fmt.Printf("[kubelet-start] WARNING: unable to start the kubelet service: [%v]\n", err) - fmt.Printf("[kubelet-start] Please ensure kubelet is reloaded and running manually.\n") - } -} - -// TryStopKubelet attempts to bring down the kubelet service momentarily -func TryStopKubelet() { - // If we notice that the kubelet service is inactive, try to start it - initSystem, err := initsystem.GetInitSystem() - if err != nil { - fmt.Println("[kubelet-start] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.") - return - } - - if !initSystem.ServiceExists("kubelet") { - fmt.Println("[kubelet-start] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.") - } - - // This runs "systemctl daemon-reload && systemctl stop kubelet" - if err := initSystem.ServiceStop("kubelet"); err != nil { - fmt.Printf("[kubelet-start] WARNING: unable to stop the kubelet service momentarily: [%v]\n", err) - } -} - -// TryRestartKubelet attempts to restart the kubelet service -func TryRestartKubelet() { - // If we notice that the kubelet service is inactive, try to start it - initSystem, err := initsystem.GetInitSystem() - if err != nil { - fmt.Println("[kubelet-start] no supported init system detected, won't make sure the kubelet not running for a short period of time while setting up configuration for it.") - return - } - - if !initSystem.ServiceExists("kubelet") { - fmt.Println("[kubelet-start] couldn't detect a kubelet service, can't make sure the kubelet not running for a short period of time while setting up configuration for it.") - } - - // This runs "systemctl daemon-reload && systemctl stop kubelet" - if err := initSystem.ServiceRestart("kubelet"); err != nil { - fmt.Printf("[kubelet-start] WARNING: unable to restart the kubelet service momentarily: [%v]\n", err) - } -} diff --git a/cmd/kubeadm/app/phases/markcontrolplane/BUILD b/cmd/kubeadm/app/phases/markcontrolplane/BUILD deleted file mode 100644 index ec2db5e4d1e26..0000000000000 --- a/cmd/kubeadm/app/phases/markcontrolplane/BUILD +++ /dev/null @@ -1,45 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["markcontrolplane_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = ["markcontrolplane.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane", - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go deleted file mode 100644 index 63344c82ae347..0000000000000 --- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package markcontrolplane - -import ( - "fmt" - - "k8s.io/api/core/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// MarkControlPlane taints the control-plane and sets the control-plane label -func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error { - - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels \"%s=''\" and \"%s='' (deprecated)\"\n", - controlPlaneName, constants.LabelNodeRoleOldControlPlane, constants.LabelNodeRoleControlPlane) - - if len(taints) > 0 { - taintStrs := []string{} - for _, taint := range taints { - taintStrs = append(taintStrs, taint.ToString()) - } - fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the taints %v\n", controlPlaneName, taintStrs) - } - - return apiclient.PatchNode(client, controlPlaneName, func(n *v1.Node) { - markControlPlaneNode(n, taints) - }) -} - -func taintExists(taint v1.Taint, taints []v1.Taint) bool { - for _, t := range taints { - if t == taint { - return true - } - } - - return false -} - -func markControlPlaneNode(n *v1.Node, taints []v1.Taint) { - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - n.ObjectMeta.Labels[constants.LabelNodeRoleOldControlPlane] = "" - n.ObjectMeta.Labels[constants.LabelNodeRoleControlPlane] = "" - - for _, nt := range n.Spec.Taints { - if !taintExists(nt, taints) { - taints = append(taints, nt) - } - } - - n.Spec.Taints = taints -} diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go deleted file mode 100644 index 3233976e12e12..0000000000000 --- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package markcontrolplane - -import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestMarkControlPlane(t *testing.T) { - // Note: this test takes advantage of the deterministic marshalling of - // JSON provided by strategicpatch so that "expectedPatch" can use a - // string equality test instead of a logical JSON equality test. That - // will need to change if strategicpatch's behavior changes in the - // future. - tests := []struct { - name string - existingLabels []string - existingTaints []v1.Taint - newTaints []v1.Taint - expectedPatch string - }{ - { - name: "control-plane label and taint missing", - existingLabels: []string{""}, - existingTaints: nil, - newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, - }, - { - name: "control-plane label and taint missing but taint not wanted", - existingLabels: []string{""}, - existingTaints: nil, - newTaints: nil, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`, - }, - { - name: "control-plane label missing", - existingLabels: []string{""}, - existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`, - }, - { - name: "control-plane taint missing", - existingLabels: []string{ - kubeadmconstants.LabelNodeRoleOldControlPlane, - kubeadmconstants.LabelNodeRoleControlPlane, - }, - existingTaints: nil, - newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, - }, - { - name: "nothing missing", - existingLabels: []string{ - kubeadmconstants.LabelNodeRoleOldControlPlane, - kubeadmconstants.LabelNodeRoleControlPlane, - }, - existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{}`, - }, - { - name: "has taint and no new taints wanted", - existingLabels: []string{ - kubeadmconstants.LabelNodeRoleOldControlPlane, - kubeadmconstants.LabelNodeRoleControlPlane, - }, - existingTaints: []v1.Taint{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Effect: v1.TaintEffectNoSchedule, - }, - }, - newTaints: nil, - expectedPatch: `{}`, - }, - { - name: "has taint and should merge with wanted taint", - existingLabels: []string{ - kubeadmconstants.LabelNodeRoleOldControlPlane, - kubeadmconstants.LabelNodeRoleControlPlane, - }, - existingTaints: []v1.Taint{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Effect: v1.TaintEffectNoSchedule, - }, - }, - newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized"}]}}`, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - nodename := "node01" - controlPlaneNode := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodename, - Labels: map[string]string{ - v1.LabelHostname: nodename, - }, - }, - } - - for _, label := range tc.existingLabels { - controlPlaneNode.ObjectMeta.Labels[label] = "" - } - - if tc.existingTaints != nil { - controlPlaneNode.Spec.Taints = tc.existingTaints - } - - jsonNode, err := json.Marshal(controlPlaneNode) - if err != nil { - t.Fatalf("unexpected encoding error: %v", err) - } - - var patchRequest string - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - - if req.URL.Path != "/api/v1/nodes/"+nodename { - t.Errorf("request for unexpected HTTP resource: %v", req.URL.Path) - http.Error(w, "", http.StatusNotFound) - return - } - - switch req.Method { - case "GET": - case "PATCH": - buf := new(bytes.Buffer) - buf.ReadFrom(req.Body) - patchRequest = buf.String() - default: - t.Errorf("request for unexpected HTTP verb: %v", req.Method) - http.Error(w, "", http.StatusNotFound) - return - } - - w.WriteHeader(http.StatusOK) - w.Write(jsonNode) - })) - defer s.Close() - - cs, err := clientset.NewForConfig(&restclient.Config{Host: s.URL}) - if err != nil { - t.Fatalf("unexpected error building clientset: %v", err) - } - - if err := MarkControlPlane(cs, nodename, tc.newTaints); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if tc.expectedPatch != patchRequest { - t.Errorf("unexpected error: wanted patch %v, got %v", tc.expectedPatch, patchRequest) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/patchnode/BUILD b/cmd/kubeadm/app/phases/patchnode/BUILD deleted file mode 100644 index a7aed14209a73..0000000000000 --- a/cmd/kubeadm/app/phases/patchnode/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["patchnode.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = ["patchnode_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/phases/patchnode/patchnode.go b/cmd/kubeadm/app/phases/patchnode/patchnode.go deleted file mode 100644 index f3277fafc98dd..0000000000000 --- a/cmd/kubeadm/app/phases/patchnode/patchnode.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package patchnode - -import ( - "k8s.io/api/core/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// AnnotateCRISocket annotates the node with the given crisocket -func AnnotateCRISocket(client clientset.Interface, nodeName string, criSocket string) error { - - klog.V(1).Infof("[patchnode] Uploading the CRI Socket information %q to the Node API object %q as an annotation\n", criSocket, nodeName) - - return apiclient.PatchNode(client, nodeName, func(n *v1.Node) { - annotateNodeWithCRISocket(n, criSocket) - }) -} - -func annotateNodeWithCRISocket(n *v1.Node, criSocket string) { - if n.ObjectMeta.Annotations == nil { - n.ObjectMeta.Annotations = make(map[string]string) - } - n.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] = criSocket -} diff --git a/cmd/kubeadm/app/phases/patchnode/patchnode_test.go b/cmd/kubeadm/app/phases/patchnode/patchnode_test.go deleted file mode 100644 index 93787be0f1707..0000000000000 --- a/cmd/kubeadm/app/phases/patchnode/patchnode_test.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package patchnode - -import ( - "bytes" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestAnnotateCRISocket(t *testing.T) { - tests := []struct { - name string - currentCRISocketAnnotation string - newCRISocketAnnotation string - expectedPatch string - }{ - { - name: "CRI-socket annotation missing", - currentCRISocketAnnotation: "", - newCRISocketAnnotation: "/run/containerd/containerd.sock", - expectedPatch: `{"metadata":{"annotations":{"kubeadm.alpha.kubernetes.io/cri-socket":"/run/containerd/containerd.sock"}}}`, - }, - { - name: "CRI-socket annotation already exists", - currentCRISocketAnnotation: "/run/containerd/containerd.sock", - newCRISocketAnnotation: "/run/containerd/containerd.sock", - expectedPatch: `{}`, - }, - { - name: "CRI-socket annotation needs to be updated", - currentCRISocketAnnotation: "/var/run/dockershim.sock", - newCRISocketAnnotation: "/run/containerd/containerd.sock", - expectedPatch: `{"metadata":{"annotations":{"kubeadm.alpha.kubernetes.io/cri-socket":"/run/containerd/containerd.sock"}}}`, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - - nodename := "node01" - node := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodename, - Labels: map[string]string{ - v1.LabelHostname: nodename, - }, - Annotations: map[string]string{}, - }, - } - - if tc.currentCRISocketAnnotation != "" { - node.ObjectMeta.Annotations[kubeadmconstants.AnnotationKubeadmCRISocket] = tc.currentCRISocketAnnotation - } - - jsonNode, err := json.Marshal(node) - if err != nil { - t.Fatalf("unexpected encoding error: %v", err) - } - - var patchRequest string - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - - if req.URL.Path != "/api/v1/nodes/"+nodename { - t.Errorf("request for unexpected HTTP resource: %v", req.URL.Path) - http.Error(w, "", http.StatusNotFound) - return - } - - switch req.Method { - case "GET": - case "PATCH": - buf := new(bytes.Buffer) - buf.ReadFrom(req.Body) - patchRequest = buf.String() - default: - t.Errorf("request for unexpected HTTP verb: %v", req.Method) - http.Error(w, "", http.StatusNotFound) - return - } - - w.WriteHeader(http.StatusOK) - w.Write(jsonNode) - })) - defer s.Close() - - cs, err := clientset.NewForConfig(&restclient.Config{Host: s.URL}) - if err != nil { - t.Fatalf("unexpected error building clientset: %v", err) - } - - if err := AnnotateCRISocket(cs, nodename, tc.newCRISocketAnnotation); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if tc.expectedPatch != patchRequest { - t.Errorf("expected patch %v, got %v", tc.expectedPatch, patchRequest) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/selfhosting/BUILD b/cmd/kubeadm/app/phases/selfhosting/BUILD deleted file mode 100644 index c5eba468d2edc..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = [ - "podspec_mutation_test.go", - "selfhosting_test.go", - "selfhosting_volumes_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = [ - "podspec_mutation.go", - "selfhosting.go", - "selfhosting_volumes.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go deleted file mode 100644 index 0d4090798c67c..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "path/filepath" - "strings" - - v1 "k8s.io/api/core/v1" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const ( - // selfHostedKubeConfigDir sets the directory where kubeconfig files for the scheduler and controller-manager should be mounted - // Due to how the projected volume mount works (can only be a full directory, not mount individual files), we must change this from - // the default as mounts cannot be nested (/etc/kubernetes would override /etc/kubernetes/pki) - selfHostedKubeConfigDir = "/etc/kubernetes/kubeconfig" -) - -// PodSpecMutatorFunc is a function capable of mutating a PodSpec -type PodSpecMutatorFunc func(*v1.PodSpec) - -// GetDefaultMutators gets the mutator functions that always should be used -func GetDefaultMutators() map[string][]PodSpecMutatorFunc { - return map[string][]PodSpecMutatorFunc{ - kubeadmconstants.KubeAPIServer: { - addNodeSelectorToPodSpec, - setControlPlaneTolerationOnPodSpec, - setRightDNSPolicyOnPodSpec, - setHostIPOnPodSpec, - }, - kubeadmconstants.KubeControllerManager: { - addNodeSelectorToPodSpec, - setControlPlaneTolerationOnPodSpec, - setRightDNSPolicyOnPodSpec, - }, - kubeadmconstants.KubeScheduler: { - addNodeSelectorToPodSpec, - setControlPlaneTolerationOnPodSpec, - setRightDNSPolicyOnPodSpec, - }, - } -} - -// GetMutatorsFromFeatureGates returns all mutators needed based on the feature gates passed -func GetMutatorsFromFeatureGates(certsInSecrets bool) map[string][]PodSpecMutatorFunc { - // Here the map of different mutators to use for the control plane's podspec is stored - mutators := GetDefaultMutators() - - if certsInSecrets { - // Some extra work to be done if we should store the control plane certificates in Secrets - // Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them - mutators[kubeadmconstants.KubeAPIServer] = append(mutators[kubeadmconstants.KubeAPIServer], setSelfHostedVolumesForAPIServer) - mutators[kubeadmconstants.KubeControllerManager] = append(mutators[kubeadmconstants.KubeControllerManager], setSelfHostedVolumesForControllerManager) - mutators[kubeadmconstants.KubeScheduler] = append(mutators[kubeadmconstants.KubeScheduler], setSelfHostedVolumesForScheduler) - } - - return mutators -} - -// mutatePodSpec makes a Static Pod-hosted PodSpec suitable for self-hosting -func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpec *v1.PodSpec) { - // Get the mutator functions for the component in question, then loop through and execute them - mutatorsForComponent := mutators[name] - for _, mutateFunc := range mutatorsForComponent { - mutateFunc(podSpec) - } -} - -// addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the control-plane label -func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) { - if podSpec.NodeSelector == nil { - podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleOldControlPlane: ""} - return - } - - podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleOldControlPlane] = "" -} - -// setControlPlaneTolerationOnPodSpec makes the Pod tolerate the control-plane taint -func setControlPlaneTolerationOnPodSpec(podSpec *v1.PodSpec) { - if podSpec.Tolerations == nil { - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - podSpec.Tolerations = []v1.Toleration{kubeadmconstants.OldControlPlaneToleration} - return - } - - podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.OldControlPlaneToleration) -} - -// setHostIPOnPodSpec sets the environment variable HOST_IP using downward API -func setHostIPOnPodSpec(podSpec *v1.PodSpec) { - envVar := v1.EnvVar{ - Name: "HOST_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - } - - podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, envVar) - - for i := range podSpec.Containers[0].Command { - if strings.Contains(podSpec.Containers[0].Command[i], "advertise-address") { - podSpec.Containers[0].Command[i] = "--advertise-address=$(HOST_IP)" - } - } -} - -// setRightDNSPolicyOnPodSpec makes sure the self-hosted components can look up things via kube-dns if necessary -func setRightDNSPolicyOnPodSpec(podSpec *v1.PodSpec) { - podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet -} - -// setSelfHostedVolumesForAPIServer makes sure the self-hosted api server has the right volume source coming from a self-hosted cluster -func setSelfHostedVolumesForAPIServer(podSpec *v1.PodSpec) { - for i, v := range podSpec.Volumes { - // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted - if v.Name == kubeadmconstants.KubeCertificatesVolumeName { - podSpec.Volumes[i].VolumeSource = apiServerCertificatesVolumeSource() - } - } -} - -// setSelfHostedVolumesForControllerManager makes sure the self-hosted controller manager has the right volume source coming from a self-hosted cluster -func setSelfHostedVolumesForControllerManager(podSpec *v1.PodSpec) { - for i, v := range podSpec.Volumes { - // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted - if v.Name == kubeadmconstants.KubeCertificatesVolumeName { - podSpec.Volumes[i].VolumeSource = controllerManagerCertificatesVolumeSource() - } else if v.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName) - } - } - - // Change directory for the kubeconfig directory to selfHostedKubeConfigDir - for i, vm := range podSpec.Containers[0].VolumeMounts { - if vm.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir - } - } - - // Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki) - // This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes - // don't support that. - podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string { - controllerManagerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.ControllerManagerKubeConfigFileName) - argMap["kubeconfig"] = controllerManagerKubeConfigPath - if _, ok := argMap["authentication-kubeconfig"]; ok { - argMap["authentication-kubeconfig"] = controllerManagerKubeConfigPath - } - if _, ok := argMap["authorization-kubeconfig"]; ok { - argMap["authorization-kubeconfig"] = controllerManagerKubeConfigPath - } - return argMap - }) -} - -// setSelfHostedVolumesForScheduler makes sure the self-hosted scheduler has the right volume source coming from a self-hosted cluster -func setSelfHostedVolumesForScheduler(podSpec *v1.PodSpec) { - for i, v := range podSpec.Volumes { - // If the volume name matches the expected one; switch the volume source from hostPath to cluster-hosted - if v.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Volumes[i].VolumeSource = kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName) - } - } - - // Change directory for the kubeconfig directory to selfHostedKubeConfigDir - for i, vm := range podSpec.Containers[0].VolumeMounts { - if vm.Name == kubeadmconstants.KubeConfigVolumeName { - podSpec.Containers[0].VolumeMounts[i].MountPath = selfHostedKubeConfigDir - } - } - - // Rewrite the --kubeconfig path as the volume mount path may not overlap with certs dir, which it does by default (/etc/kubernetes and /etc/kubernetes/pki) - // This is not a problem with hostPath mounts as hostPath supports mounting one file only, instead of always a full directory. Secrets and Projected Volumes - // don't support that. - podSpec.Containers[0].Command = kubeadmutil.ReplaceArgument(podSpec.Containers[0].Command, func(argMap map[string]string) map[string]string { - schedulerKubeConfigPath := filepath.Join(selfHostedKubeConfigDir, kubeadmconstants.SchedulerKubeConfigFileName) - argMap["kubeconfig"] = schedulerKubeConfigPath - if _, ok := argMap["authentication-kubeconfig"]; ok { - argMap["authentication-kubeconfig"] = schedulerKubeConfigPath - } - if _, ok := argMap["authorization-kubeconfig"]; ok { - argMap["authorization-kubeconfig"] = schedulerKubeConfigPath - } - return argMap - }) -} diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go deleted file mode 100644 index 81e69b9e225a6..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go +++ /dev/null @@ -1,590 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "reflect" - "sort" - "testing" - - v1 "k8s.io/api/core/v1" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestMutatePodSpec(t *testing.T) { - var tests = []struct { - name string - component string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "mutate api server podspec", - component: kubeadmconstants.KubeAPIServer, - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=10.0.0.1", - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=$(HOST_IP)", - }, - Env: []v1.EnvVar{ - { - Name: "HOST_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - }, - }, - }, - }, - - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - { - name: "mutate controller manager podspec", - component: kubeadmconstants.KubeControllerManager, - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - { - name: "mutate scheduler podspec", - component: kubeadmconstants.KubeScheduler, - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - mutatePodSpec(GetDefaultMutators(), rt.component, rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed mutatePodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestAddNodeSelectorToPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "empty podspec", - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - }, - }, - { - name: "podspec with a valid node selector", - podSpec: &v1.PodSpec{ - NodeSelector: map[string]string{ - "foo": "bar", - }, - }, - expected: v1.PodSpec{ - NodeSelector: map[string]string{ - "foo": "bar", - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - addNodeSelectorToPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed addNodeSelectorToPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetControlPlaneTolerationOnPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "empty podspec", - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - Tolerations: []v1.Toleration{ - kubeadmconstants.OldControlPlaneToleration, - }, - }, - }, - { - name: "podspec with a valid toleration", - podSpec: &v1.PodSpec{ - Tolerations: []v1.Toleration{ - {Key: "foo", Value: "bar"}, - }, - }, - expected: v1.PodSpec{ - Tolerations: []v1.Toleration{ - {Key: "foo", Value: "bar"}, - kubeadmconstants.OldControlPlaneToleration, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setControlPlaneTolerationOnPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setControlPlaneTolerationOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetRightDNSPolicyOnPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "empty podspec", - podSpec: &v1.PodSpec{}, - expected: v1.PodSpec{ - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - { - name: "podspec with a v1.DNSClusterFirst policy", - podSpec: &v1.PodSpec{ - DNSPolicy: v1.DNSClusterFirst, - }, - expected: v1.PodSpec{ - DNSPolicy: v1.DNSClusterFirstWithHostNet, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setRightDNSPolicyOnPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setRightDNSPolicyOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetHostIPOnPodSpec(t *testing.T) { - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set HOST_IP env var on a podspec", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=10.0.0.1", - }, - Env: []v1.EnvVar{}, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "kube-apiserver", - Command: []string{ - "--advertise-address=$(HOST_IP)", - }, - Env: []v1.EnvVar{ - { - Name: "HOST_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.hostIP", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setHostIPOnPodSpec(rt.podSpec) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setHostIPOnPodSpec:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetSelfHostedVolumesForAPIServer(t *testing.T) { - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set selfhosted volumes for api server", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - }, - Command: []string{ - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/pki", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - }, - Command: []string{ - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: apiServerCertificatesVolumeSource(), - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setSelfHostedVolumesForAPIServer(rt.podSpec) - sort.Strings(rt.podSpec.Containers[0].Command) - sort.Strings(rt.expected.Containers[0].Command) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setSelfHostedVolumesForAPIServer:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetSelfHostedVolumesForControllerManager(t *testing.T) { - hostPathFileOrCreate := v1.HostPathFileOrCreate - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set selfhosted volumes for controller mananger", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/controller-manager.conf", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/controller-manager.conf", - "--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf", - "--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/pki", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/controller-manager.conf", - Type: &hostPathFileOrCreate, - }, - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "ca-certs", - MountPath: "/etc/ssl/certs", - }, - { - Name: "k8s-certs", - MountPath: "/etc/kubernetes/pki", - }, - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/kubeconfig", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", - "--authentication-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", - "--authorization-kubeconfig=/etc/kubernetes/kubeconfig/controller-manager.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ca-certs", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/ssl/certs", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "k8s-certs", - VolumeSource: controllerManagerCertificatesVolumeSource(), - }, - { - Name: "kubeconfig", - VolumeSource: kubeConfigVolumeSource(kubeadmconstants.ControllerManagerKubeConfigFileName), - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setSelfHostedVolumesForControllerManager(rt.podSpec) - sort.Strings(rt.podSpec.Containers[0].Command) - sort.Strings(rt.expected.Containers[0].Command) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setSelfHostedVolumesForControllerManager:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} - -func TestSetSelfHostedVolumesForScheduler(t *testing.T) { - hostPathFileOrCreate := v1.HostPathFileOrCreate - var tests = []struct { - name string - podSpec *v1.PodSpec - expected v1.PodSpec - }{ - { - name: "set selfhosted volumes for scheduler", - podSpec: &v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/scheduler.conf", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/scheduler.conf", - "--authentication-kubeconfig=/etc/kubernetes/scheduler.conf", - "--authorization-kubeconfig=/etc/kubernetes/scheduler.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "kubeconfig", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/kubernetes/scheduler.conf", - Type: &hostPathFileOrCreate, - }, - }, - }, - }, - }, - expected: v1.PodSpec{ - Containers: []v1.Container{ - { - VolumeMounts: []v1.VolumeMount{ - { - Name: "kubeconfig", - MountPath: "/etc/kubernetes/kubeconfig", - }, - }, - Command: []string{ - "--kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", - "--authentication-kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", - "--authorization-kubeconfig=/etc/kubernetes/kubeconfig/scheduler.conf", - "--foo=bar", - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "kubeconfig", - VolumeSource: kubeConfigVolumeSource(kubeadmconstants.SchedulerKubeConfigFileName), - }, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - setSelfHostedVolumesForScheduler(rt.podSpec) - sort.Strings(rt.podSpec.Containers[0].Command) - sort.Strings(rt.expected.Containers[0].Command) - - if !reflect.DeepEqual(*rt.podSpec, rt.expected) { - t.Errorf("failed setSelfHostedVolumesForScheduler:\nexpected:\n%v\nsaw:\n%v", rt.expected, *rt.podSpec) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting.go deleted file mode 100644 index 1d3ad4d4f093b..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting.go +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "fmt" - "io/ioutil" - "os" - "time" - - "k8s.io/klog/v2" - - "github.com/pkg/errors" - apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientset "k8s.io/client-go/kubernetes" - clientscheme "k8s.io/client-go/kubernetes/scheme" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -const ( - // selfHostingWaitTimeout describes the maximum amount of time a self-hosting wait process should wait before timing out - selfHostingWaitTimeout = 2 * time.Minute - - // selfHostingFailureThreshold describes how many times kubeadm will retry creating the DaemonSets - selfHostingFailureThreshold int = 5 -) - -// CreateSelfHostedControlPlane is responsible for turning a Static Pod-hosted control plane to a self-hosted one -// It achieves that task this way: -// 1. Load the Static Pod specification from disk (from /etc/kubernetes/manifests) -// 2. Extract the PodSpec from that Static Pod specification -// 3. Mutate the PodSpec to be compatible with self-hosting (add the right labels, taints, etc. so it can schedule correctly) -// 4. Build a new DaemonSet object for the self-hosted component in question. Use the above mentioned PodSpec -// 5. Create the DaemonSet resource. Wait until the Pods are running. -// 6. Remove the Static Pod manifest file. The kubelet will stop the original Static Pod-hosted component that was running. -// 7. The self-hosted containers should now step up and take over. -// 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue -// Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes -// 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop -func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.InitConfiguration, client clientset.Interface, waiter apiclient.Waiter, dryRun bool, certsInSecrets bool) error { - klog.V(1).Infoln("creating self hosted control plane") - // Adjust the timeout slightly to something self-hosting specific - waiter.SetTimeout(selfHostingWaitTimeout) - - // Here the map of different mutators to use for the control plane's PodSpec is stored - klog.V(1).Infoln("getting mutators") - mutators := GetMutatorsFromFeatureGates(certsInSecrets) - - if certsInSecrets { - // Upload the certificates and kubeconfig files from disk to the cluster as Secrets - if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil { - return err - } - if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil { - return err - } - } - - for _, componentName := range kubeadmconstants.ControlPlaneComponents { - start := time.Now() - manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir) - - // Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist - if _, err := os.Stat(manifestPath); err != nil { - fmt.Printf("[self-hosted] The Static Pod for the component %q doesn't seem to be on the disk; trying the next one\n", componentName) - continue - } - - // Load the Static Pod spec in order to be able to create a self-hosted variant of that file - podSpec, err := loadPodSpecFromFile(manifestPath) - if err != nil { - return err - } - - // Build a DaemonSet object from the loaded PodSpec - ds := BuildDaemonSet(componentName, podSpec, mutators) - - // Create or update the DaemonSet in the API Server, and retry selfHostingFailureThreshold times if it errors out - if err := apiclient.TryRunCommand(func() error { - return apiclient.CreateOrUpdateDaemonSet(client, ds) - }, selfHostingFailureThreshold); err != nil { - return err - } - - // Wait for the self-hosted component to come up - if err := waiter.WaitForPodsWithLabel(BuildSelfHostedComponentLabelQuery(componentName)); err != nil { - return err - } - - // Remove the old Static Pod manifest if not dryrunning - if !dryRun { - if err := os.RemoveAll(manifestPath); err != nil { - return errors.Wrapf(err, "unable to delete static pod manifest for %s ", componentName) - } - } - - // Wait for the mirror Pod hash to be removed; otherwise we'll run into race conditions here when the kubelet hasn't had time to - // remove the Static Pod (or the mirror Pod respectively). This implicitly also tests that the API server endpoint is healthy, - // because this blocks until the API server returns a 404 Not Found when getting the Static Pod - staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeRegistration.Name) - if err := waiter.WaitForPodToDisappear(staticPodName); err != nil { - return err - } - - // Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above) - if err := waiter.WaitForAPI(); err != nil { - return err - } - - fmt.Printf("[self-hosted] self-hosted %s ready after %f seconds\n", componentName, time.Since(start).Seconds()) - } - return nil -} - -// BuildDaemonSet is responsible for mutating the PodSpec and returns a DaemonSet which is suitable for self-hosting -func BuildDaemonSet(name string, podSpec *v1.PodSpec, mutators map[string][]PodSpecMutatorFunc) *apps.DaemonSet { - - // Mutate the PodSpec so it's suitable for self-hosting - mutatePodSpec(mutators, name, podSpec) - - // Return a DaemonSet based on that Spec - return &apps.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.AddSelfHostedPrefix(name), - Namespace: metav1.NamespaceSystem, - Labels: BuildSelfhostedComponentLabels(name), - }, - Spec: apps.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: BuildSelfhostedComponentLabels(name), - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: BuildSelfhostedComponentLabels(name), - }, - Spec: *podSpec, - }, - UpdateStrategy: apps.DaemonSetUpdateStrategy{ - // Make the DaemonSet utilize the RollingUpdate rollout strategy - Type: apps.RollingUpdateDaemonSetStrategyType, - }, - }, - } -} - -// BuildSelfhostedComponentLabels returns the labels for a self-hosted component -func BuildSelfhostedComponentLabels(component string) map[string]string { - return map[string]string{ - "k8s-app": kubeadmconstants.AddSelfHostedPrefix(component), - } -} - -// BuildSelfHostedComponentLabelQuery creates the right query for matching a self-hosted Pod -func BuildSelfHostedComponentLabelQuery(componentName string) string { - return fmt.Sprintf("k8s-app=%s", kubeadmconstants.AddSelfHostedPrefix(componentName)) -} - -func loadPodSpecFromFile(filePath string) (*v1.PodSpec, error) { - podDef, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to read file path %s", filePath) - } - - if len(podDef) == 0 { - return nil, errors.Errorf("file was empty: %s", filePath) - } - - codec := clientscheme.Codecs.UniversalDecoder() - pod := &v1.Pod{} - - if err = runtime.DecodeInto(codec, podDef, pod); err != nil { - return nil, errors.Wrap(err, "failed decoding pod") - } - - return &pod.Spec, nil -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go deleted file mode 100644 index e64db5007ceb4..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_test.go +++ /dev/null @@ -1,610 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "bytes" - "io/ioutil" - "os" - "testing" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const ( - testAPIServerPod = ` -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-apiserver - namespace: kube-system -spec: - containers: - - command: - - kube-apiserver - - --service-account-key-file=/etc/kubernetes/pki/sa.pub - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - - --secure-port=6443 - - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --requestheader-group-headers=X-Remote-Group - - --service-cluster-ip-range=10.96.0.0/12 - - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - - --advertise-address=192.168.1.115 - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - - --insecure-port=0 - - --experimental-bootstrap-token-auth=true - - --requestheader-username-headers=X-Remote-User - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --requestheader-allowed-names=front-proxy-client - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - - --allow-privileged=true - - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - - --authorization-mode=Node,RBAC - - --etcd-servers=http://127.0.0.1:2379 - image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 6443 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-apiserver - resources: - requests: - cpu: 250m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - hostNetwork: true - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki -status: {} -` - - testAPIServerDaemonSet = `apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-apiserver - name: self-hosted-kube-apiserver - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: self-hosted-kube-apiserver - template: - metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-apiserver - spec: - containers: - - command: - - kube-apiserver - - --service-account-key-file=/etc/kubernetes/pki/sa.pub - - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key - - --secure-port=6443 - - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --requestheader-group-headers=X-Remote-Group - - --service-cluster-ip-range=10.96.0.0/12 - - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt - - --advertise-address=$(HOST_IP) - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - - --insecure-port=0 - - --experimental-bootstrap-token-auth=true - - --requestheader-username-headers=X-Remote-User - - --requestheader-extra-headers-prefix=X-Remote-Extra- - - --requestheader-allowed-names=front-proxy-client - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - - --allow-privileged=true - - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key - - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - - --authorization-mode=Node,RBAC - - --etcd-servers=http://127.0.0.1:2379 - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - image: k8s.gcr.io/kube-apiserver-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 6443 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-apiserver - resources: - requests: - cpu: 250m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - priorityClassName: system-cluster-critical - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki - updateStrategy: - type: RollingUpdate -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -` - - testControllerManagerPod = ` -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-controller-manager - namespace: kube-system -spec: - containers: - - command: - - kube-controller-manager - - --leader-elect=true - - --controllers=*,bootstrapsigner,tokencleaner - - --kubeconfig=/etc/kubernetes/controller-manager.conf - - --root-ca-file=/etc/kubernetes/pki/ca.crt - - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - - --bind-address=127.0.0.1 - - --use-service-account-credentials=true - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10257 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-controller-manager - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/kubernetes/controller-manager.conf - name: kubeconfig - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - hostNetwork: true - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/kubernetes/controller-manager.conf - type: FileOrCreate - name: kubeconfig - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki -status: {} -` - - testControllerManagerDaemonSet = `apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-controller-manager - name: self-hosted-kube-controller-manager - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: self-hosted-kube-controller-manager - template: - metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-controller-manager - spec: - containers: - - command: - - kube-controller-manager - - --leader-elect=true - - --controllers=*,bootstrapsigner,tokencleaner - - --kubeconfig=/etc/kubernetes/controller-manager.conf - - --root-ca-file=/etc/kubernetes/pki/ca.crt - - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - - --bind-address=127.0.0.1 - - --use-service-account-credentials=true - - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - image: k8s.gcr.io/kube-controller-manager-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10257 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-controller-manager - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /etc/kubernetes/pki - name: k8s-certs - readOnly: true - - mountPath: /etc/ssl/certs - name: ca-certs - readOnly: true - - mountPath: /etc/kubernetes/controller-manager.conf - name: kubeconfig - readOnly: true - - mountPath: /etc/pki - name: ca-certs-etc-pki - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - priorityClassName: system-cluster-critical - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - hostPath: - path: /etc/kubernetes/pki - name: k8s-certs - - hostPath: - path: /etc/ssl/certs - name: ca-certs - - hostPath: - path: /etc/kubernetes/controller-manager.conf - type: FileOrCreate - name: kubeconfig - - hostPath: - path: /etc/pki - name: ca-certs-etc-pki - updateStrategy: - type: RollingUpdate -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -` - - testSchedulerPod = ` -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: kube-scheduler - namespace: kube-system -spec: - containers: - - command: - - kube-scheduler - - --leader-elect=true - - --kubeconfig=/etc/kubernetes/scheduler.conf - - --bind-address=127.0.0.1 - image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10259 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-scheduler - resources: - requests: - cpu: 100m - volumeMounts: - - mountPath: /etc/kubernetes/scheduler.conf - name: kubeconfig - readOnly: true - hostNetwork: true - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /etc/kubernetes/scheduler.conf - type: FileOrCreate - name: kubeconfig -status: {} -` - - testSchedulerDaemonSet = `apiVersion: apps/v1 -kind: DaemonSet -metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-scheduler - name: self-hosted-kube-scheduler - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: self-hosted-kube-scheduler - template: - metadata: - creationTimestamp: null - labels: - k8s-app: self-hosted-kube-scheduler - spec: - containers: - - command: - - kube-scheduler - - --leader-elect=true - - --kubeconfig=/etc/kubernetes/scheduler.conf - - --bind-address=127.0.0.1 - image: k8s.gcr.io/kube-scheduler-amd64:v1.7.4 - livenessProbe: - failureThreshold: 8 - httpGet: - host: 127.0.0.1 - path: /healthz - port: 10259 - scheme: HTTPS - initialDelaySeconds: 15 - timeoutSeconds: 15 - name: kube-scheduler - resources: - requests: - cpu: 100m - volumeMounts: - - mountPath: /etc/kubernetes/scheduler.conf - name: kubeconfig - readOnly: true - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/master: "" - priorityClassName: system-cluster-critical - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - hostPath: - path: /etc/kubernetes/scheduler.conf - type: FileOrCreate - name: kubeconfig - updateStrategy: - type: RollingUpdate -status: - currentNumberScheduled: 0 - desiredNumberScheduled: 0 - numberMisscheduled: 0 - numberReady: 0 -` -) - -func TestBuildDaemonSet(t *testing.T) { - var tests = []struct { - component string - podBytes []byte - dsBytes []byte - }{ - { - component: constants.KubeAPIServer, - podBytes: []byte(testAPIServerPod), - dsBytes: []byte(testAPIServerDaemonSet), - }, - { - component: constants.KubeControllerManager, - podBytes: []byte(testControllerManagerPod), - dsBytes: []byte(testControllerManagerDaemonSet), - }, - { - component: constants.KubeScheduler, - podBytes: []byte(testSchedulerPod), - dsBytes: []byte(testSchedulerDaemonSet), - }, - } - - for _, rt := range tests { - t.Run(rt.component, func(t *testing.T) { - tempFile, err := createTempFileWithContent(rt.podBytes) - if err != nil { - t.Errorf("error creating tempfile with content:%v", err) - } - defer os.Remove(tempFile) - - podSpec, err := loadPodSpecFromFile(tempFile) - if err != nil { - t.Fatalf("couldn't load the specified Pod Spec") - } - - ds := BuildDaemonSet(rt.component, podSpec, GetDefaultMutators()) - dsBytes, err := kubeadmutil.MarshalToYaml(ds, apps.SchemeGroupVersion) - if err != nil { - t.Fatalf("failed to marshal daemonset to YAML: %v", err) - } - - if !bytes.Equal(dsBytes, rt.dsBytes) { - t.Errorf("failed TestBuildDaemonSet:\nexpected:\n%s\nsaw:\n%s", rt.dsBytes, dsBytes) - } - }) - } -} - -func TestLoadPodSpecFromFile(t *testing.T) { - tests := []struct { - name string - content string - expectError bool - }{ - { - name: "no content", - content: "", - expectError: true, - }, - { - name: "valid YAML", - content: ` -apiVersion: v1 -kind: Pod -metadata: - name: testpod -spec: - containers: - - image: k8s.gcr.io/busybox -`, - expectError: false, - }, - { - name: "valid JSON", - content: ` -{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "name": "testpod" - }, - "spec": { - "containers": [ - { - "image": "k8s.gcr.io/busybox" - } - ] - } -}`, - expectError: false, - }, - { - name: "incorrect PodSpec", - content: ` -apiVersion: v1 -kind: Pod -metadata: - name: testpod -spec: - - image: k8s.gcr.io/busybox -`, - expectError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - tempFile, err := createTempFileWithContent([]byte(rt.content)) - if err != nil { - t.Errorf("error creating tempfile with content:%v", err) - } - defer os.Remove(tempFile) - - _, err = loadPodSpecFromFile(tempFile) - if (err != nil) != rt.expectError { - t.Errorf("failed TestLoadPodSpecFromFile:\nexpected error:\n%t\nsaw:\n%v", rt.expectError, err) - } - }) - } - - t.Run("empty file name", func(t *testing.T) { - _, err := loadPodSpecFromFile("") - if err == nil { - t.Error("unexpected success: loadPodSpecFromFile should return error when no file is given") - } - }) -} - -func createTempFileWithContent(content []byte) (string, error) { - tempFile, err := ioutil.TempFile("", "") - if err != nil { - return "", errors.Wrap(err, "cannot create temporary file") - } - if _, err = tempFile.Write([]byte(content)); err != nil { - return "", errors.Wrap(err, "cannot save temporary file") - } - if err = tempFile.Close(); err != nil { - return "", errors.Wrap(err, "cannot close temporary file") - } - return tempFile.Name(), nil -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go deleted file mode 100644 index a877ec34e53b2..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go +++ /dev/null @@ -1,356 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "strings" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -type tlsKeyPairPath struct { - name string - cert string - key string -} - -func apiServerCertificatesVolumeSource() v1.VolumeSource { - return v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.CACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.CACertName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.APIServerCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.APIServerKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.APIServerKubeletClientCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.APIServerKubeletClientKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ServiceAccountKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.ServiceAccountPublicKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.FrontProxyCACertName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.FrontProxyClientCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.FrontProxyClientKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: strings.Replace(kubeadmconstants.EtcdCACertAndKeyBaseName, "/", "-", -1), - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.EtcdCACertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.EtcdCAKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.APIServerEtcdClientCertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.APIServerEtcdClientKeyName, - }, - }, - }, - }, - }, - }, - } -} - -func controllerManagerCertificatesVolumeSource() v1.VolumeSource { - return v1.VolumeSource{ - Projected: &v1.ProjectedVolumeSource{ - Sources: []v1.VolumeProjection{ - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.CACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.CACertName, - }, - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.CAKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.ServiceAccountKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSPrivateKeyKey, - Path: kubeadmconstants.ServiceAccountPrivateKeyName, - }, - }, - }, - }, - { - Secret: &v1.SecretProjection{ - LocalObjectReference: v1.LocalObjectReference{ - Name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - }, - Items: []v1.KeyToPath{ - { - Key: v1.TLSCertKey, - Path: kubeadmconstants.FrontProxyCACertName, - }, - }, - }, - }, - }, - }, - } -} - -func kubeConfigVolumeSource(kubeconfigSecretName string) v1.VolumeSource { - return v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: strings.Replace(kubeconfigSecretName, "/", "-", -1), - }, - } -} - -func uploadTLSSecrets(client clientset.Interface, certDir string) error { - for _, tlsKeyPair := range getTLSKeyPairs() { - secret, err := createTLSSecretFromFiles( - tlsKeyPair.name, - filepath.Join(certDir, tlsKeyPair.cert), - filepath.Join(certDir, tlsKeyPair.key), - ) - if err != nil { - return err - } - - if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil { - return err - } - fmt.Printf("[self-hosted] Created TLS secret %q from %s and %s\n", tlsKeyPair.name, tlsKeyPair.cert, tlsKeyPair.key) - } - - return nil -} - -func uploadKubeConfigSecrets(client clientset.Interface, kubeConfigDir string) error { - files := []string{ - kubeadmconstants.SchedulerKubeConfigFileName, - kubeadmconstants.ControllerManagerKubeConfigFileName, - } - for _, file := range files { - kubeConfigPath := filepath.Join(kubeConfigDir, file) - secret, err := createOpaqueSecretFromFile(file, kubeConfigPath) - if err != nil { - return err - } - - if err := apiclient.CreateOrUpdateSecret(client, secret); err != nil { - return err - } - fmt.Printf("[self-hosted] Created secret for kubeconfig file %q\n", file) - } - - return nil -} - -func createTLSSecretFromFiles(secretName, crt, key string) (*v1.Secret, error) { - crtBytes, err := ioutil.ReadFile(crt) - if err != nil { - return nil, err - } - keyBytes, err := ioutil.ReadFile(key) - if err != nil { - return nil, err - } - - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: metav1.NamespaceSystem, - }, - Type: v1.SecretTypeTLS, - Data: map[string][]byte{ - v1.TLSCertKey: crtBytes, - v1.TLSPrivateKeyKey: keyBytes, - }, - }, nil -} - -func createOpaqueSecretFromFile(secretName, file string) (*v1.Secret, error) { - fileBytes, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: metav1.NamespaceSystem, - }, - Type: v1.SecretTypeOpaque, - Data: map[string][]byte{ - filepath.Base(file): fileBytes, - }, - }, nil -} - -func getTLSKeyPairs() []*tlsKeyPairPath { - return []*tlsKeyPairPath{ - { - name: kubeadmconstants.CACertAndKeyBaseName, - cert: kubeadmconstants.CACertName, - key: kubeadmconstants.CAKeyName, - }, - { - name: kubeadmconstants.APIServerCertAndKeyBaseName, - cert: kubeadmconstants.APIServerCertName, - key: kubeadmconstants.APIServerKeyName, - }, - { - name: kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, - cert: kubeadmconstants.APIServerKubeletClientCertName, - key: kubeadmconstants.APIServerKubeletClientKeyName, - }, - { - name: kubeadmconstants.ServiceAccountKeyBaseName, - cert: kubeadmconstants.ServiceAccountPublicKeyName, - key: kubeadmconstants.ServiceAccountPrivateKeyName, - }, - { - name: kubeadmconstants.FrontProxyCACertAndKeyBaseName, - cert: kubeadmconstants.FrontProxyCACertName, - key: kubeadmconstants.FrontProxyCAKeyName, - }, - { - name: kubeadmconstants.FrontProxyClientCertAndKeyBaseName, - cert: kubeadmconstants.FrontProxyClientCertName, - key: kubeadmconstants.FrontProxyClientKeyName, - }, - { - name: strings.Replace(kubeadmconstants.EtcdCACertAndKeyBaseName, "/", "-", -1), - cert: kubeadmconstants.EtcdCACertName, - key: kubeadmconstants.EtcdCAKeyName, - }, - { - name: kubeadmconstants.APIServerEtcdClientCertAndKeyBaseName, - cert: kubeadmconstants.APIServerEtcdClientCertName, - key: kubeadmconstants.APIServerEtcdClientKeyName, - }, - } -} diff --git a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go b/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go deleted file mode 100644 index bf2e51c1f168b..0000000000000 --- a/cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes_test.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package selfhosting - -import ( - "io/ioutil" - "log" - "os" - "testing" -) - -func createTemporaryFile(name string) *os.File { - content := []byte("foo") - tmpfile, err := ioutil.TempFile("", name) - if err != nil { - log.Fatal(err) - } - - if _, err := tmpfile.Write(content); err != nil { - log.Fatal(err) - } - - return tmpfile -} - -func TestCreateTLSSecretFromFile(t *testing.T) { - tmpCert := createTemporaryFile("foo.crt") - defer os.Remove(tmpCert.Name()) - tmpKey := createTemporaryFile("foo.key") - defer os.Remove(tmpKey.Name()) - - _, err := createTLSSecretFromFiles("foo", tmpCert.Name(), tmpKey.Name()) - if err != nil { - log.Fatal(err) - } - - if err := tmpCert.Close(); err != nil { - log.Fatal(err) - } - - if err := tmpKey.Close(); err != nil { - log.Fatal(err) - } -} - -func TestCreateOpaqueSecretFromFile(t *testing.T) { - tmpFile := createTemporaryFile("foo") - defer os.Remove(tmpFile.Name()) - - _, err := createOpaqueSecretFromFile("foo", tmpFile.Name()) - if err != nil { - log.Fatal(err) - } - - if err := tmpFile.Close(); err != nil { - log.Fatal(err) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/BUILD b/cmd/kubeadm/app/phases/upgrade/BUILD deleted file mode 100644 index dbe8d91c0a77d..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/BUILD +++ /dev/null @@ -1,105 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "compute.go", - "health.go", - "policy.go", - "postupgrade.go", - "preflight.go", - "staticpods.go", - "versiongetter.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/phases/addons/dns:go_default_library", - "//cmd/kubeadm/app/phases/addons/proxy:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/clusterinfo:go_default_library", - "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", - "//cmd/kubeadm/app/phases/controlplane:go_default_library", - "//cmd/kubeadm/app/phases/etcd:go_default_library", - "//cmd/kubeadm/app/phases/kubelet:go_default_library", - "//cmd/kubeadm/app/phases/patchnode:go_default_library", - "//cmd/kubeadm/app/phases/uploadconfig:go_default_library", - "//cmd/kubeadm/app/preflight:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/dryrun:go_default_library", - "//cmd/kubeadm/app/util/etcd:go_default_library", - "//cmd/kubeadm/app/util/image:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/batch/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//vendor/github.com/coredns/corefile-migration/migration:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/pointer:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = [ - "compute_test.go", - "policy_test.go", - "postupgrade_test.go", - "staticpods_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/phases/certs/renewal:go_default_library", - "//cmd/kubeadm/app/phases/controlplane:go_default_library", - "//cmd/kubeadm/app/phases/etcd:go_default_library", - "//cmd/kubeadm/app/phases/kubeconfig:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/certs:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//cmd/kubeadm/app/util/etcd:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/go.etcd.io/etcd/pkg/transport:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go deleted file mode 100644 index e0f45a82a13eb..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/compute.go +++ /dev/null @@ -1,318 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "strings" - - versionutil "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" -) - -// Upgrade defines an upgrade possibility to upgrade from a current version to a new one -type Upgrade struct { - Description string - Before ClusterState - After ClusterState -} - -// CanUpgradeKubelets returns whether an upgrade of any kubelet in the cluster is possible -func (u *Upgrade) CanUpgradeKubelets() bool { - // If there are multiple different versions now, an upgrade is possible (even if only for a subset of the nodes) - if len(u.Before.KubeletVersions) > 1 { - return true - } - // Don't report something available for upgrade if we don't know the current state - if len(u.Before.KubeletVersions) == 0 { - return false - } - - // if the same version number existed both before and after, we don't have to upgrade it - _, sameVersionFound := u.Before.KubeletVersions[u.After.KubeVersion] - return !sameVersionFound -} - -// CanUpgradeEtcd returns whether an upgrade of etcd is possible -func (u *Upgrade) CanUpgradeEtcd() bool { - return u.Before.EtcdVersion != u.After.EtcdVersion -} - -// ClusterState describes the state of certain versions for a cluster -type ClusterState struct { - // KubeVersion describes the version of the Kubernetes API Server, Controller Manager, Scheduler and Proxy. - KubeVersion string - // DNSType describes the type of DNS add-on used in the cluster. - DNSType kubeadmapi.DNSAddOnType - // DNSVersion describes the version of the DNS add-on. - DNSVersion string - // KubeadmVersion describes the version of the kubeadm CLI - KubeadmVersion string - // KubeletVersions is a map with a version number linked to the amount of kubelets running that version in the cluster - KubeletVersions map[string]uint16 - // EtcdVersion represents the version of etcd used in the cluster - EtcdVersion string -} - -// GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which -// kinds of upgrades can be performed -func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed, externalEtcd bool, dnsType kubeadmapi.DNSAddOnType, client clientset.Interface, manifestsDir string) ([]Upgrade, error) { - fmt.Println("[upgrade] Fetching available versions to upgrade to") - - // Collect the upgrades kubeadm can do in this list - upgrades := []Upgrade{} - - // Get the cluster version - clusterVersionStr, clusterVersion, err := versionGetterImpl.ClusterVersion() - if err != nil { - return upgrades, err - } - fmt.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr) - - // Get current kubeadm CLI version - kubeadmVersionStr, kubeadmVersion, err := versionGetterImpl.KubeadmVersion() - if err != nil { - return upgrades, err - } - fmt.Printf("[upgrade/versions] kubeadm version: %s\n", kubeadmVersionStr) - - // Get and output the current latest stable version - stableVersionStr, stableVersion, err := versionGetterImpl.VersionFromCILabel("stable", "stable version") - if err != nil { - fmt.Printf("[upgrade/versions] WARNING: %v\n", err) - fmt.Println("[upgrade/versions] WARNING: Falling back to current kubeadm version as latest stable version") - stableVersionStr, stableVersion = kubeadmVersionStr, kubeadmVersion - } else { - fmt.Printf("[upgrade/versions] Latest %s: %s\n", "stable version", stableVersionStr) - } - - // Get the kubelet versions in the cluster - kubeletVersions, err := versionGetterImpl.KubeletVersions() - if err != nil { - return upgrades, err - } - - // Get current stacked etcd version on the local node - var etcdVersion string - if !externalEtcd { - etcdVersion, err = GetEtcdImageTagFromStaticPod(manifestsDir) - if err != nil { - return upgrades, err - } - } - - currentDNSType, dnsVersion, err := dns.DeployedDNSAddon(client) - if err != nil { - return nil, err - } - - // Construct a descriptor for the current state of the world - beforeState := ClusterState{ - KubeVersion: clusterVersionStr, - DNSType: currentDNSType, - DNSVersion: dnsVersion, - KubeadmVersion: kubeadmVersionStr, - KubeletVersions: kubeletVersions, - EtcdVersion: etcdVersion, - } - - // Do a "dumb guess" that a new minor upgrade is available just because the latest stable version is higher than the cluster version - // This guess will be corrected once we know if there is a patch version available - canDoMinorUpgrade := clusterVersion.LessThan(stableVersion) - - // A patch version doesn't exist if the cluster version is higher than or equal to the current stable version - // in the case that a user is trying to upgrade from, let's say, v1.8.0-beta.2 to v1.8.0-rc.1 (given we support such upgrades experimentally) - // a stable-1.8 branch doesn't exist yet. Hence this check. - if patchVersionBranchExists(clusterVersion, stableVersion) { - currentBranch := getBranchFromVersion(clusterVersionStr) - versionLabel := fmt.Sprintf("stable-%s", currentBranch) - description := fmt.Sprintf("version in the v%s series", currentBranch) - - // Get and output the latest patch version for the cluster branch - patchVersionStr, patchVersion, err := versionGetterImpl.VersionFromCILabel(versionLabel, description) - if err != nil { - fmt.Printf("[upgrade/versions] WARNING: %v\n", err) - } else { - fmt.Printf("[upgrade/versions] Latest %s: %s\n", description, patchVersionStr) - - // Check if a minor version upgrade is possible when a patch release exists - // It's only possible if the latest patch version is higher than the current patch version - // If that's the case, they must be on different branches => a newer minor version can be upgraded to - canDoMinorUpgrade = minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion) - - // If the cluster version is lower than the newest patch version, we should inform about the possible upgrade - if patchUpgradePossible(clusterVersion, patchVersion) { - - // The kubeadm version has to be upgraded to the latest patch version - newKubeadmVer := patchVersionStr - if kubeadmVersion.AtLeast(patchVersion) { - // In this case, the kubeadm CLI version is new enough. Don't display an update suggestion for kubeadm by making .NewKubeadmVersion equal .CurrentKubeadmVersion - newKubeadmVer = kubeadmVersionStr - } - - upgrades = append(upgrades, Upgrade{ - Description: description, - Before: beforeState, - After: ClusterState{ - KubeVersion: patchVersionStr, - DNSType: dnsType, - DNSVersion: kubeadmconstants.GetDNSVersion(dnsType), - KubeadmVersion: newKubeadmVer, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, patchVersionStr), - // KubeletVersions is unset here as it is not used anywhere in .After - }, - }) - } - } - } - - if canDoMinorUpgrade { - upgrades = append(upgrades, Upgrade{ - Description: "stable version", - Before: beforeState, - After: ClusterState{ - KubeVersion: stableVersionStr, - DNSType: dnsType, - DNSVersion: kubeadmconstants.GetDNSVersion(dnsType), - KubeadmVersion: stableVersionStr, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, stableVersionStr), - // KubeletVersions is unset here as it is not used anywhere in .After - }, - }) - } - - if experimentalUpgradesAllowed || rcUpgradesAllowed { - // dl.k8s.io/release/latest.txt is ALWAYS an alpha.X version - // dl.k8s.io/release/latest-1.X.txt is first v1.X.0-alpha.0 -> v1.X.0-alpha.Y, then v1.X.0-beta.0 to v1.X.0-beta.Z, then v1.X.0-rc.1 to v1.X.0-rc.W. - // After the v1.X.0 release, latest-1.X.txt is always a beta.0 version. Let's say the latest stable version on the v1.7 branch is v1.7.3, then the - // latest-1.7 version is v1.7.4-beta.0 - - // Worth noticing is that when the release-1.X branch is cut; there are two versions tagged: v1.X.0-beta.0 AND v1.(X+1).alpha.0 - // The v1.(X+1).alpha.0 is pretty much useless and should just be ignored, as more betas may be released that have more features than the initial v1.(X+1).alpha.0 - - // So what we do below is getting the latest overall version, always an v1.X.0-alpha.Y version. Then we get latest-1.(X-1) version. This version may be anything - // between v1.(X-1).0-beta.0 and v1.(X-1).Z-beta.0. At some point in time, latest-1.(X-1) will point to v1.(X-1).0-rc.1. Then we should show it. - - // The flow looks like this (with time on the X axis): - // v1.8.0-alpha.1 -> v1.8.0-alpha.2 -> v1.8.0-alpha.3 | release-1.8 branch | v1.8.0-beta.0 -> v1.8.0-beta.1 -> v1.8.0-beta.2 -> v1.8.0-rc.1 -> v1.8.0 -> v1.8.1 - // v1.9.0-alpha.0 -> v1.9.0-alpha.1 -> v1.9.0-alpha.2 - - // Get and output the current latest unstable version - latestVersionStr, latestVersion, err := versionGetterImpl.VersionFromCILabel("latest", "experimental version") - if err != nil { - return upgrades, err - } - fmt.Printf("[upgrade/versions] Latest %s: %s\n", "experimental version", latestVersionStr) - - minorUnstable := latestVersion.Components()[1] - // Get and output the current latest unstable version - previousBranch := fmt.Sprintf("latest-1.%d", minorUnstable-1) - previousBranchLatestVersionStr, previousBranchLatestVersion, err := versionGetterImpl.VersionFromCILabel(previousBranch, "") - if err != nil { - return upgrades, err - } - fmt.Printf("[upgrade/versions] Latest %s: %s\n", "", previousBranchLatestVersionStr) - - // If that previous latest version is an RC, RCs are allowed and the cluster version is lower than the RC version, show the upgrade - if rcUpgradesAllowed && rcUpgradePossible(clusterVersion, previousBranchLatestVersion) { - upgrades = append(upgrades, Upgrade{ - Description: "release candidate version", - Before: beforeState, - After: ClusterState{ - KubeVersion: previousBranchLatestVersionStr, - DNSType: dnsType, - DNSVersion: kubeadmconstants.GetDNSVersion(dnsType), - KubeadmVersion: previousBranchLatestVersionStr, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, previousBranchLatestVersionStr), - // KubeletVersions is unset here as it is not used anywhere in .After - }, - }) - } - - // Show the possibility if experimental upgrades are allowed - if experimentalUpgradesAllowed && clusterVersion.LessThan(latestVersion) { - - // Default to assume that the experimental version to show is the unstable one - unstableKubeVersion := latestVersionStr - unstableKubeDNSVersion := kubeadmconstants.GetDNSVersion(dnsType) - - // Ẃe should not display alpha.0. The previous branch's beta/rc versions are more relevant due how the kube branching process works. - if latestVersion.PreRelease() == "alpha.0" { - unstableKubeVersion = previousBranchLatestVersionStr - unstableKubeDNSVersion = kubeadmconstants.GetDNSVersion(dnsType) - } - - upgrades = append(upgrades, Upgrade{ - Description: "experimental version", - Before: beforeState, - After: ClusterState{ - KubeVersion: unstableKubeVersion, - DNSType: dnsType, - DNSVersion: unstableKubeDNSVersion, - KubeadmVersion: unstableKubeVersion, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, unstableKubeVersion), - // KubeletVersions is unset here as it is not used anywhere in .After - }, - }) - } - } - - // Add a newline in the end of this output to leave some space to the next output section - fmt.Println("") - - return upgrades, nil -} - -func getBranchFromVersion(version string) string { - v := versionutil.MustParseGeneric(version) - return fmt.Sprintf("%d.%d", v.Major(), v.Minor()) -} - -func patchVersionBranchExists(clusterVersion, stableVersion *versionutil.Version) bool { - return stableVersion.AtLeast(clusterVersion) -} - -func patchUpgradePossible(clusterVersion, patchVersion *versionutil.Version) bool { - return clusterVersion.LessThan(patchVersion) -} - -func rcUpgradePossible(clusterVersion, previousBranchLatestVersion *versionutil.Version) bool { - return strings.HasPrefix(previousBranchLatestVersion.PreRelease(), "rc") && clusterVersion.LessThan(previousBranchLatestVersion) -} - -func minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion *versionutil.Version) bool { - return patchVersion.LessThan(stableVersion) -} - -func getSuggestedEtcdVersion(externalEtcd bool, kubernetesVersion string) string { - if externalEtcd { - return "" - } - etcdVersion, warning, err := kubeadmconstants.EtcdSupportedVersion(kubeadmconstants.SupportedEtcdVersion, kubernetesVersion) - if err != nil { - klog.Warningf("[upgrade/versions] could not retrieve an etcd version for the target Kubernetes version: %v", err) - return "N/A" - } - if warning != nil { - klog.Warningf("[upgrade/versions] %v", warning) - } - return etcdVersion.String() -} diff --git a/cmd/kubeadm/app/phases/upgrade/compute_test.go b/cmd/kubeadm/app/phases/upgrade/compute_test.go deleted file mode 100644 index 185bc4ab52e4e..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/compute_test.go +++ /dev/null @@ -1,918 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "io/ioutil" - "os" - "reflect" - "strings" - "testing" - - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - clientsetfake "k8s.io/client-go/kubernetes/fake" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -type fakeVersionGetter struct { - clusterVersion, kubeadmVersion, stableVersion, latestVersion, latestDevBranchVersion, stablePatchVersion, kubeletVersion string -} - -var _ VersionGetter = &fakeVersionGetter{} - -// ClusterVersion gets a fake API server version -func (f *fakeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) { - return f.clusterVersion, versionutil.MustParseSemantic(f.clusterVersion), nil -} - -// KubeadmVersion gets a fake kubeadm version -func (f *fakeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) { - return f.kubeadmVersion, versionutil.MustParseSemantic(f.kubeadmVersion), nil -} - -// VersionFromCILabel gets fake latest versions from CI -func (f *fakeVersionGetter) VersionFromCILabel(ciVersionLabel, _ string) (string, *versionutil.Version, error) { - if ciVersionLabel == "stable" { - return f.stableVersion, versionutil.MustParseSemantic(f.stableVersion), nil - } - if ciVersionLabel == "latest" { - return f.latestVersion, versionutil.MustParseSemantic(f.latestVersion), nil - } - if f.latestDevBranchVersion != "" && strings.HasPrefix(ciVersionLabel, "latest-") { - return f.latestDevBranchVersion, versionutil.MustParseSemantic(f.latestDevBranchVersion), nil - } - return f.stablePatchVersion, versionutil.MustParseSemantic(f.stablePatchVersion), nil -} - -// KubeletVersions gets the versions of the kubelets in the cluster -func (f *fakeVersionGetter) KubeletVersions() (map[string]uint16, error) { - return map[string]uint16{ - f.kubeletVersion: 1, - }, nil -} - -const fakeCurrentEtcdVersion = "3.1.12" -const etcdStaticPod = `apiVersion: v1 -kind: Pod -metadata: - labels: - component: etcd - tier: control-plane - name: etcd - namespace: kube-system -spec: - containers: - - name: etcd - image: k8s.gcr.io/etcd:` + fakeCurrentEtcdVersion - -func getEtcdVersion(v *versionutil.Version) string { - return constants.SupportedEtcdVersion[uint8(v.Minor())] -} - -const fakeCurrentCoreDNSVersion = "1.0.6" -const fakeCurrentKubeDNSVersion = "1.14.7" - -func TestGetAvailableUpgrades(t *testing.T) { - - // constansts for test cases - // variables are in the form v{MAJOR}{MINOR}{PATCH}, where MINOR is a variable so test are automatically uptodate to the latest MinimumControlPlaneVersion/ - - // v1.X series, e.g. v1.14 - v1X0 := constants.MinimumControlPlaneVersion.WithMinor(constants.MinimumControlPlaneVersion.Minor() - 1) - v1X5 := v1X0.WithPatch(5) - - // v1.Y series, where Y = X+1, e.g. v1.15 - v1Y0 := constants.MinimumControlPlaneVersion - v1Y0alpha0 := v1Y0.WithPreRelease("alpha.0") - v1Y0alpha1 := v1Y0.WithPreRelease("alpha.1") - v1Y1 := v1Y0.WithPatch(1) - v1Y2 := v1Y0.WithPatch(2) - v1Y3 := v1Y0.WithPatch(3) - v1Y5 := v1Y0.WithPatch(5) - - // v1.Z series, where Z = Y+1, e.g. v1.16 - v1Z0 := constants.CurrentKubernetesVersion - v1Z0alpha1 := v1Z0.WithPreRelease("alpha.1") - v1Z0alpha2 := v1Z0.WithPreRelease("alpha.2") - v1Z0beta1 := v1Z0.WithPreRelease("beta.1") - v1Z0rc1 := v1Z0.WithPreRelease("rc.1") - v1Z1 := v1Z0.WithPatch(1) - - tests := []struct { - name string - vg VersionGetter - expectedUpgrades []Upgrade - allowExperimental, allowRCs bool - errExpected bool - externalEtcd bool - beforeDNSType kubeadmapi.DNSAddOnType - beforeDNSVersion string - dnsType kubeadmapi.DNSAddOnType - }{ - { - name: "no action needed, already up-to-date", - vg: &fakeVersionGetter{ - clusterVersion: v1Y0.String(), - kubeletVersion: v1Y0.String(), - kubeadmVersion: v1Y0.String(), - - stablePatchVersion: v1Y0.String(), - stableVersion: v1Y0.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{}, - allowExperimental: false, - errExpected: false, - }, - { - name: "simple patch version upgrade", - vg: &fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y2.String(), - - stablePatchVersion: v1Y3.String(), - stableVersion: v1Y3.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, - }, - KubeadmVersion: v1Y2.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Y3.String(), - KubeadmVersion: v1Y3.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Y3), - }, - }, - }, - allowExperimental: false, - errExpected: false, - }, - { - name: "simple patch version upgrade with external etcd", - vg: &fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y2.String(), - - stablePatchVersion: v1Y3.String(), - stableVersion: v1Y3.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - externalEtcd: true, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, - }, - KubeadmVersion: v1Y2.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: "", - }, - After: ClusterState{ - KubeVersion: v1Y3.String(), - KubeadmVersion: v1Y3.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: "", - }, - }, - }, - allowExperimental: false, - errExpected: false, - }, - { - name: "no version provided to offline version getter does not change behavior", - vg: NewOfflineVersionGetter(&fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y2.String(), - - stablePatchVersion: v1Y3.String(), - stableVersion: v1Y3.String(), - }, ""), - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, - }, - KubeadmVersion: v1Y2.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Y3.String(), - KubeadmVersion: v1Y3.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Y3), - }, - }, - }, - allowExperimental: false, - errExpected: false, - }, - { - name: "minor version upgrade only", - vg: &fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Z0.String(), - - stablePatchVersion: v1Y1.String(), - stableVersion: v1Z0.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "stable version", - Before: ClusterState{ - KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, - }, - KubeadmVersion: v1Z0.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0.String(), - KubeadmVersion: v1Z0.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0), - }, - }, - }, - allowExperimental: false, - errExpected: false, - }, - { - name: "both minor version upgrade and patch version upgrade available", - vg: &fakeVersionGetter{ - clusterVersion: v1Y3.String(), - kubeletVersion: v1Y3.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y5.String(), - - stablePatchVersion: v1Y5.String(), - stableVersion: v1Z1.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y3.String(), - KubeletVersions: map[string]uint16{ - v1Y3.String(): 1, - }, - KubeadmVersion: v1Y5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Y5.String(), - KubeadmVersion: v1Y5.String(), // Note: The kubeadm version mustn't be "downgraded" here - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Y5), - }, - }, - { - Description: "stable version", - Before: ClusterState{ - KubeVersion: v1Y3.String(), - KubeletVersions: map[string]uint16{ - v1Y3.String(): 1, - }, - KubeadmVersion: v1Y5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z1.String(), - KubeadmVersion: v1Z1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z1), - }, - }, - }, - allowExperimental: false, - errExpected: false, - }, - { - name: "allow experimental upgrades, but no upgrade available", - vg: &fakeVersionGetter{ - clusterVersion: v1Z0alpha2.String(), - kubeletVersion: v1Y5.String(), - kubeadmVersion: v1Y5.String(), - - stablePatchVersion: v1Y5.String(), - stableVersion: v1Y5.String(), - latestVersion: v1Z0alpha2.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{}, - allowExperimental: true, - errExpected: false, - }, - { - name: "upgrade to an unstable version should be supported", - vg: &fakeVersionGetter{ - clusterVersion: v1Y5.String(), - kubeletVersion: v1Y5.String(), - kubeadmVersion: v1Y5.String(), - - stablePatchVersion: v1Y5.String(), - stableVersion: v1Y5.String(), - latestVersion: v1Z0alpha2.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "experimental version", - Before: ClusterState{ - KubeVersion: v1Y5.String(), - KubeletVersions: map[string]uint16{ - v1Y5.String(): 1, - }, - KubeadmVersion: v1Y5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0alpha2.String(), - KubeadmVersion: v1Z0alpha2.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0alpha2), - }, - }, - }, - allowExperimental: true, - errExpected: false, - }, - { - name: "upgrade from an unstable version to an unstable version should be supported", - vg: &fakeVersionGetter{ - clusterVersion: v1Z0alpha1.String(), - kubeletVersion: v1Y5.String(), - kubeadmVersion: v1Y5.String(), - - stablePatchVersion: v1Y5.String(), - stableVersion: v1Y5.String(), - latestVersion: v1Z0alpha2.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "experimental version", - Before: ClusterState{ - KubeVersion: v1Z0alpha1.String(), - KubeletVersions: map[string]uint16{ - v1Y5.String(): 1, - }, - KubeadmVersion: v1Y5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0alpha2.String(), - KubeadmVersion: v1Z0alpha2.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0alpha2), - }, - }, - }, - allowExperimental: true, - errExpected: false, - }, - { - name: "v1.X.0-alpha.0 should be ignored", - vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), - - stablePatchVersion: v1X5.String(), - stableVersion: v1X5.String(), - latestDevBranchVersion: v1Z0beta1.String(), - latestVersion: v1Y0alpha0.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "experimental version", - Before: ClusterState{ - KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, - }, - KubeadmVersion: v1X5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0beta1.String(), - KubeadmVersion: v1Z0beta1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0beta1), - }, - }, - }, - allowExperimental: true, - errExpected: false, - }, - { - name: "upgrade to an RC version should be supported", - vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), - - stablePatchVersion: v1X5.String(), - stableVersion: v1X5.String(), - latestDevBranchVersion: v1Z0rc1.String(), - latestVersion: v1Y0alpha1.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "release candidate version", - Before: ClusterState{ - KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, - }, - KubeadmVersion: v1X5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0rc1.String(), - KubeadmVersion: v1Z0rc1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0rc1), - }, - }, - }, - allowRCs: true, - errExpected: false, - }, - { - name: "it is possible (but very uncommon) that the latest version from the previous branch is an rc and the current latest version is alpha.0. In that case, show the RC", - vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), - - stablePatchVersion: v1X5.String(), - stableVersion: v1X5.String(), - latestDevBranchVersion: v1Z0rc1.String(), - latestVersion: v1Y0alpha0.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "experimental version", // Note that this is considered an experimental version in this uncommon scenario - Before: ClusterState{ - KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, - }, - KubeadmVersion: v1X5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0rc1.String(), - KubeadmVersion: v1Z0rc1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0rc1), - }, - }, - }, - allowExperimental: true, - errExpected: false, - }, - { - name: "upgrade to an RC version should be supported. There may also be an even newer unstable version.", - vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), - - stablePatchVersion: v1X5.String(), - stableVersion: v1X5.String(), - latestDevBranchVersion: v1Z0rc1.String(), - latestVersion: v1Y0alpha1.String(), - }, - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: "release candidate version", - Before: ClusterState{ - KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, - }, - KubeadmVersion: v1X5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0rc1.String(), - KubeadmVersion: v1Z0rc1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0rc1), - }, - }, - { - Description: "experimental version", - Before: ClusterState{ - KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, - }, - KubeadmVersion: v1X5.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Y0alpha1.String(), - KubeadmVersion: v1Y0alpha1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Y0alpha1), - }, - }, - }, - allowRCs: true, - allowExperimental: true, - errExpected: false, - }, - { - name: "offline version getter", - vg: NewOfflineVersionGetter(&fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y0.String(), - kubeadmVersion: v1Y1.String(), - }, v1Z1.String()), - beforeDNSType: kubeadmapi.CoreDNS, - beforeDNSVersion: fakeCurrentCoreDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y0.String(): 1, - }, - KubeadmVersion: v1Y1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z1.String(), - KubeadmVersion: v1Z1.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z1), - }, - }, - }, - }, - { - name: "kubedns to coredns", - vg: &fakeVersionGetter{ - clusterVersion: v1Y2.String(), - kubeletVersion: v1Y2.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Z0.String(), - - stablePatchVersion: v1Z0.String(), - stableVersion: v1Z0.String(), - }, - beforeDNSType: kubeadmapi.KubeDNS, - beforeDNSVersion: fakeCurrentKubeDNSVersion, - dnsType: kubeadmapi.CoreDNS, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y2.String(), - KubeletVersions: map[string]uint16{ - v1Y2.String(): 1, - }, - KubeadmVersion: v1Z0.String(), - DNSType: kubeadmapi.KubeDNS, - DNSVersion: fakeCurrentKubeDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0.String(), - KubeadmVersion: v1Z0.String(), - DNSType: kubeadmapi.CoreDNS, - DNSVersion: constants.CoreDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0), - }, - }, - }, - }, - { - name: "keep coredns", - vg: &fakeVersionGetter{ - clusterVersion: v1Y2.String(), - kubeletVersion: v1Y2.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Z0.String(), - - stablePatchVersion: v1Z0.String(), - stableVersion: v1Z0.String(), - }, - beforeDNSType: kubeadmapi.KubeDNS, - beforeDNSVersion: fakeCurrentKubeDNSVersion, - dnsType: kubeadmapi.KubeDNS, - expectedUpgrades: []Upgrade{ - { - Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), - Before: ClusterState{ - KubeVersion: v1Y2.String(), - KubeletVersions: map[string]uint16{ - v1Y2.String(): 1, - }, - KubeadmVersion: v1Z0.String(), - DNSType: kubeadmapi.KubeDNS, - DNSVersion: fakeCurrentKubeDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, - }, - After: ClusterState{ - KubeVersion: v1Z0.String(), - KubeadmVersion: v1Z0.String(), - DNSType: kubeadmapi.KubeDNS, - DNSVersion: constants.KubeDNSVersion, - EtcdVersion: getEtcdVersion(v1Z0), - }, - }, - }, - }, - } - - // Instantiating a fake etcd cluster for being able to get etcd version for a corresponding - // Kubernetes release. - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - - dnsName := constants.CoreDNSDeploymentName - if rt.beforeDNSType == kubeadmapi.KubeDNS { - dnsName = constants.KubeDNSDeploymentName - } - - client := clientsetfake.NewSimpleClientset(&apps.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: dnsName, - Namespace: "kube-system", - Labels: map[string]string{ - "k8s-app": "kube-dns", - }, - }, - Spec: apps.DeploymentSpec{ - Template: v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Image: "test:" + rt.beforeDNSVersion, - }, - }, - }, - }, - }, - }) - - manifestsDir, err := ioutil.TempDir("", "GetAvailableUpgrades-test-manifests") - if err != nil { - t.Fatalf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(manifestsDir) - - if err = ioutil.WriteFile(constants.GetStaticPodFilepath(constants.Etcd, manifestsDir), []byte(etcdStaticPod), 0644); err != nil { - t.Fatalf("Unable to create test static pod manifest: %v", err) - } - - actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.externalEtcd, rt.dnsType, client, manifestsDir) - if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { - t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) - } - if rt.errExpected && actualErr == nil { - t.Error("unexpected success") - } else if !rt.errExpected && actualErr != nil { - t.Errorf("unexpected failure: %v", actualErr) - } - if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { - t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) - } - }) - } -} - -func TestKubeletUpgrade(t *testing.T) { - tests := []struct { - name string - before map[string]uint16 - after string - expected bool - }{ - { - name: "upgrade from v1.10.1 to v1.10.3 is available", - before: map[string]uint16{ - "v1.10.1": 1, - }, - after: "v1.10.3", - expected: true, - }, - { - name: "upgrade from v1.10.1 and v1.10.3/100 to v1.10.3 is available", - before: map[string]uint16{ - "v1.10.1": 1, - "v1.10.3": 100, - }, - after: "v1.10.3", - expected: true, - }, - { - name: "upgrade from v1.10.3 to v1.10.3 is not available", - before: map[string]uint16{ - "v1.10.3": 1, - }, - after: "v1.10.3", - expected: false, - }, - { - name: "upgrade from v1.10.3/100 to v1.10.3 is not available", - before: map[string]uint16{ - "v1.10.3": 100, - }, - after: "v1.10.3", - expected: false, - }, - { - name: "upgrade is not available if we don't know anything about the earlier state", - before: map[string]uint16{}, - after: "v1.10.3", - expected: false, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - upgrade := Upgrade{ - Before: ClusterState{ - KubeletVersions: rt.before, - }, - After: ClusterState{ - KubeVersion: rt.after, - }, - } - actual := upgrade.CanUpgradeKubelets() - if actual != rt.expected { - t.Errorf("failed TestKubeletUpgrade\n\texpected: %t\n\tgot: %t\n\ttest object: %v", rt.expected, actual, upgrade) - } - }) - } -} - -func TestGetBranchFromVersion(t *testing.T) { - testCases := []struct { - version string - expectedVersion string - }{ - { - version: "v1.9.5", - expectedVersion: "1.9", - }, - { - version: "v1.9.0-alpha.2", - expectedVersion: "1.9", - }, - { - version: "v1.9.0-beta.0", - expectedVersion: "1.9", - }, - { - version: "v1.9.0-rc.1", - expectedVersion: "1.9", - }, - { - version: "v1.11.0-alpha.0", - expectedVersion: "1.11", - }, - - { - version: "v1.11.0-beta.1", - expectedVersion: "1.11", - }, - { - version: "v1.11.0-rc.0", - expectedVersion: "1.11", - }, - { - version: "1.12.5", - expectedVersion: "1.12", - }, - } - - for _, tc := range testCases { - t.Run(tc.version, func(t *testing.T) { - v := getBranchFromVersion(tc.version) - if v != tc.expectedVersion { - t.Errorf("expected version %s, got %s", tc.expectedVersion, v) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go deleted file mode 100644 index 3a238968d90d8..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/health.go +++ /dev/null @@ -1,314 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - utilpointer "k8s.io/utils/pointer" -) - -// healthCheck is a helper struct for easily performing healthchecks against the cluster and printing the output -type healthCheck struct { - name string - client clientset.Interface - cfg *kubeadmapi.ClusterConfiguration - // f is invoked with a k8s client and a kubeadm ClusterConfiguration passed to it. Should return an optional error - f func(clientset.Interface, *kubeadmapi.ClusterConfiguration) error -} - -// Check is part of the preflight.Checker interface -func (c *healthCheck) Check() (warnings, errors []error) { - if err := c.f(c.client, c.cfg); err != nil { - return nil, []error{err} - } - return nil, nil -} - -// Name is part of the preflight.Checker interface -func (c *healthCheck) Name() string { - return c.name -} - -// CheckClusterHealth makes sure: -// - the API /healthz endpoint is healthy -// - all control-plane Nodes are Ready -// - (if self-hosted) that there are DaemonSets with at least one Pod for all control plane components -// - (if static pod-hosted) that all required Static Pod manifests exist on disk -func CheckClusterHealth(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration, ignoreChecksErrors sets.String) error { - fmt.Println("[upgrade] Running cluster health checks") - - healthChecks := []preflight.Checker{ - &healthCheck{ - name: "CreateJob", - client: client, - cfg: cfg, - f: createJob, - }, - &healthCheck{ - name: "ControlPlaneNodesReady", - client: client, - f: controlPlaneNodesReady, - }, - &healthCheck{ - name: "StaticPodManifest", - client: client, - cfg: cfg, - f: staticPodManifestHealth, - }, - } - - return preflight.RunChecks(healthChecks, os.Stderr, ignoreChecksErrors) -} - -// CreateJob is a check that verifies that a Job can be created in the cluster -func createJob(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) (lastError error) { - const ( - jobName = "upgrade-health-check" - ns = metav1.NamespaceSystem - timeout = 15 * time.Second - ) - - // If client.Discovery().RESTClient() is nil, the fake client is used. - // Return early because the kubeadm dryrun dynamic client only handles the core/v1 GroupVersion. - if client.Discovery().RESTClient() == nil { - fmt.Printf("[dryrun] Would create the Job %q in namespace %q and wait until it completes\n", jobName, ns) - return nil - } - - // Prepare Job - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: jobName, - Namespace: ns, - }, - Spec: batchv1.JobSpec{ - BackoffLimit: utilpointer.Int32Ptr(0), - Template: v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyNever, - SecurityContext: &v1.PodSecurityContext{ - RunAsUser: utilpointer.Int64Ptr(999), - RunAsGroup: utilpointer.Int64Ptr(999), - RunAsNonRoot: utilpointer.BoolPtr(true), - }, - Tolerations: []v1.Toleration{ - { - Key: "node-role.kubernetes.io/master", - Effect: v1.TaintEffectNoSchedule, - }, - }, - Containers: []v1.Container{ - { - Name: jobName, - Image: images.GetPauseImage(cfg), - Args: []string{"-v"}, - }, - }, - }, - }, - }, - } - - // Check if the Job already exists and delete it - if _, err := client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{}); err == nil { - if err = deleteHealthCheckJob(client, ns, jobName); err != nil { - return err - } - } - - // Cleanup the Job on exit - defer func() { - lastError = deleteHealthCheckJob(client, ns, jobName) - }() - - // Create the Job, but retry in case it is being currently deleted - klog.V(2).Infof("Creating Job %q in the namespace %q", jobName, ns) - err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) { - if _, err := client.BatchV1().Jobs(ns).Create(context.TODO(), job, metav1.CreateOptions{}); err != nil { - klog.V(2).Infof("Could not create Job %q in the namespace %q, retrying: %v", jobName, ns, err) - lastError = err - return false, nil - } - return true, nil - }) - if err != nil { - return errors.Wrapf(lastError, "could not create Job %q in the namespace %q", jobName, ns) - } - - // Waiting and manually deleteing the Job is a workaround to not enabling the TTL controller. - // TODO: refactor this if the TTL controller is enabled in kubeadm once it goes Beta. - - // Wait for the Job to complete - err = wait.PollImmediate(time.Second*1, timeout, func() (bool, error) { - job, err := client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{}) - if err != nil { - lastError = err - klog.V(2).Infof("could not get Job %q in the namespace %q, retrying: %v", jobName, ns, err) - return false, nil - } - for _, cond := range job.Status.Conditions { - if cond.Type == batchv1.JobComplete { - return true, nil - } - } - lastError = errors.Errorf("no condition of type %v", batchv1.JobComplete) - klog.V(2).Infof("Job %q in the namespace %q is not yet complete, retrying", jobName, ns) - return false, nil - }) - if err != nil { - return errors.Wrapf(lastError, "Job %q in the namespace %q did not complete in %v", jobName, ns, timeout) - } - - klog.V(2).Infof("Job %q in the namespace %q completed", jobName, ns) - - return nil -} - -func deleteHealthCheckJob(client clientset.Interface, ns, jobName string) error { - klog.V(2).Infof("Deleting Job %q in the namespace %q", jobName, ns) - propagation := metav1.DeletePropagationForeground - if err := client.BatchV1().Jobs(ns).Delete(context.TODO(), jobName, metav1.DeleteOptions{PropagationPolicy: &propagation}); err != nil { - return errors.Wrapf(err, "could not delete Job %q in the namespace %q", jobName, ns) - } - return nil -} - -// controlPlaneNodesReady checks whether all control-plane Nodes in the cluster are in the Running state -func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error { - // list nodes labeled with a "master" node-role - selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ - constants.LabelNodeRoleOldControlPlane: "", - })) - nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: selectorOldControlPlane.String(), - }) - if err != nil { - return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleOldControlPlane) - } - - // list nodes labeled with a "control-plane" node-role - selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ - constants.LabelNodeRoleControlPlane: "", - })) - nodesControlPlane, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: selectorControlPlane.String(), - }) - if err != nil { - return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleControlPlane) - } - - nodes := append(nodesWithOldLabel.Items, nodesControlPlane.Items...) - if len(nodes) == 0 { - return errors.New("failed to find any nodes with a control-plane role") - } - - notReadyControlPlanes := getNotReadyNodes(nodes) - if len(notReadyControlPlanes) != 0 { - return errors.Errorf("there are NotReady control-planes in the cluster: %v", notReadyControlPlanes) - } - return nil -} - -// staticPodManifestHealth makes sure the required static pods are presents -func staticPodManifestHealth(_ clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error { - nonExistentManifests := []string{} - for _, component := range constants.ControlPlaneComponents { - manifestFile := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory()) - if _, err := os.Stat(manifestFile); os.IsNotExist(err) { - nonExistentManifests = append(nonExistentManifests, manifestFile) - } - } - if len(nonExistentManifests) == 0 { - return nil - } - return errors.Errorf("The control plane seems to be Static Pod-hosted, but some of the manifests don't seem to exist on disk. This probably means you're running 'kubeadm upgrade' on a remote machine, which is not supported for a Static Pod-hosted cluster. Manifest files not found: %v", nonExistentManifests) -} - -// IsControlPlaneSelfHosted returns whether the control plane is self hosted or not -func IsControlPlaneSelfHosted(client clientset.Interface) bool { - notReadyDaemonSets, err := getNotReadyDaemonSets(client) - if err != nil { - return false - } - - // If there are no NotReady DaemonSets, we are using selfhosting - return len(notReadyDaemonSets) == 0 -} - -// getNotReadyDaemonSets gets the amount of Ready control plane DaemonSets -func getNotReadyDaemonSets(client clientset.Interface) ([]error, error) { - notReadyDaemonSets := []error{} - for _, component := range constants.ControlPlaneComponents { - dsName := constants.AddSelfHostedPrefix(component) - ds, err := client.AppsV1().DaemonSets(metav1.NamespaceSystem).Get(context.TODO(), dsName, metav1.GetOptions{}) - if err != nil { - return nil, errors.Errorf("couldn't get daemonset %q in the %s namespace", dsName, metav1.NamespaceSystem) - } - - if err := daemonSetHealth(&ds.Status); err != nil { - notReadyDaemonSets = append(notReadyDaemonSets, errors.Wrapf(err, "DaemonSet %q not healthy", dsName)) - } - } - return notReadyDaemonSets, nil -} - -// daemonSetHealth is a helper function for getting the health of a DaemonSet's status -func daemonSetHealth(dsStatus *apps.DaemonSetStatus) error { - if dsStatus.CurrentNumberScheduled != dsStatus.DesiredNumberScheduled { - return errors.Errorf("current number of scheduled Pods ('%d') doesn't match the amount of desired Pods ('%d')", - dsStatus.CurrentNumberScheduled, dsStatus.DesiredNumberScheduled) - } - if dsStatus.NumberAvailable == 0 { - return errors.New("no available Pods for DaemonSet") - } - if dsStatus.NumberReady == 0 { - return errors.New("no ready Pods for DaemonSet") - } - return nil -} - -// getNotReadyNodes returns a string slice of nodes in the cluster that are NotReady -func getNotReadyNodes(nodes []v1.Node) []string { - notReadyNodes := []string{} - for _, node := range nodes { - for _, condition := range node.Status.Conditions { - if condition.Type == v1.NodeReady && condition.Status != v1.ConditionTrue { - notReadyNodes = append(notReadyNodes, node.ObjectMeta.Name) - } - } - } - return notReadyNodes -} diff --git a/cmd/kubeadm/app/phases/upgrade/policy.go b/cmd/kubeadm/app/phases/upgrade/policy.go deleted file mode 100644 index 701f13182bdee..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/policy.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -const ( - // MaximumAllowedMinorVersionUpgradeSkew describes how many minor versions kubeadm can upgrade the control plane version in one go - MaximumAllowedMinorVersionUpgradeSkew = 1 - - // MaximumAllowedMinorVersionDowngradeSkew describes how many minor versions kubeadm can upgrade the control plane version in one go - MaximumAllowedMinorVersionDowngradeSkew = 1 - - // MaximumAllowedMinorVersionKubeletSkew describes how many minor versions the control plane version and the kubelet can skew in a kubeadm cluster - MaximumAllowedMinorVersionKubeletSkew = 1 -) - -// VersionSkewPolicyErrors describes version skew errors that might be seen during the validation process in EnforceVersionPolicies -type VersionSkewPolicyErrors struct { - Mandatory []error - Skippable []error -} - -// EnforceVersionPolicies enforces that the proposed new version is compatible with all the different version skew policies -func EnforceVersionPolicies(versionGetter VersionGetter, newK8sVersionStr string, newK8sVersion *version.Version, allowExperimentalUpgrades, allowRCUpgrades bool) *VersionSkewPolicyErrors { - - skewErrors := &VersionSkewPolicyErrors{ - Mandatory: []error{}, - Skippable: []error{}, - } - - clusterVersionStr, clusterVersion, err := versionGetter.ClusterVersion() - if err != nil { - // This case can't be forced: kubeadm has to be able to lookup cluster version for upgrades to work - skewErrors.Mandatory = append(skewErrors.Mandatory, errors.Wrap(err, "Unable to fetch cluster version")) - return skewErrors - } - fmt.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr) - - kubeadmVersionStr, kubeadmVersion, err := versionGetter.KubeadmVersion() - if err != nil { - // This case can't be forced: kubeadm has to be able to lookup its version for upgrades to work - skewErrors.Mandatory = append(skewErrors.Mandatory, errors.Wrap(err, "Unable to fetch kubeadm version")) - return skewErrors - } - fmt.Printf("[upgrade/versions] kubeadm version: %s\n", kubeadmVersionStr) - - kubeletVersions, err := versionGetter.KubeletVersions() - if err != nil { - // This is a non-critical error; continue although kubeadm couldn't look this up - skewErrors.Skippable = append(skewErrors.Skippable, errors.Wrap(err, "Unable to fetch kubelet version")) - } - - // Make sure the new version is a supported version (higher than the minimum one supported) - if !newK8sVersion.AtLeast(constants.MinimumControlPlaneVersion) { - // This must not happen, kubeadm always supports a minimum version; and we can't go below that - skewErrors.Mandatory = append(skewErrors.Mandatory, errors.Errorf("Specified version to upgrade to %q is equal to or lower than the minimum supported version %q. Please specify a higher version to upgrade to", newK8sVersionStr, clusterVersionStr)) - } - - // kubeadm doesn't support upgrades between two minor versions; e.g. a v1.7 -> v1.9 upgrade is not supported right away - if newK8sVersion.Minor() > clusterVersion.Minor()+MaximumAllowedMinorVersionUpgradeSkew { - tooLargeUpgradeSkewErr := errors.Errorf("Specified version to upgrade to %q is too high; kubeadm can upgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionUpgradeSkew) - // If the version that we're about to upgrade to is a released version, we should fully enforce this policy - // If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag - if len(newK8sVersion.PreRelease()) == 0 { - skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeUpgradeSkewErr) - } else { - skewErrors.Skippable = append(skewErrors.Skippable, tooLargeUpgradeSkewErr) - } - } - - // kubeadm doesn't support downgrades between two minor versions; e.g. a v1.9 -> v1.7 downgrade is not supported right away - if newK8sVersion.Minor() < clusterVersion.Minor()-MaximumAllowedMinorVersionDowngradeSkew { - tooLargeDowngradeSkewErr := errors.Errorf("Specified version to downgrade to %q is too low; kubeadm can downgrade only %d minor version at a time", newK8sVersionStr, MaximumAllowedMinorVersionDowngradeSkew) - // If the version that we're about to downgrade to is a released version, we should fully enforce this policy - // If the version is a CI/dev/experimental version, it's okay to jump two minor version steps, but then require the -f flag - if len(newK8sVersion.PreRelease()) == 0 { - skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeDowngradeSkewErr) - } else { - skewErrors.Skippable = append(skewErrors.Skippable, tooLargeDowngradeSkewErr) - } - } - - // If the kubeadm version is lower than what we want to upgrade to; error - if kubeadmVersion.LessThan(newK8sVersion) { - if newK8sVersion.Minor() > kubeadmVersion.Minor() { - tooLargeKubeadmSkew := errors.Errorf("Specified version to upgrade to %q is at least one minor release higher than the kubeadm minor release (%d > %d). Such an upgrade is not supported", newK8sVersionStr, newK8sVersion.Minor(), kubeadmVersion.Minor()) - // This is unsupported; kubeadm has no idea how it should handle a newer minor release than itself - // If the version is a CI/dev/experimental version though, lower the severity of this check, but then require the -f flag - if len(newK8sVersion.PreRelease()) == 0 { - skewErrors.Mandatory = append(skewErrors.Mandatory, tooLargeKubeadmSkew) - } else { - skewErrors.Skippable = append(skewErrors.Skippable, tooLargeKubeadmSkew) - } - } else { - // Upgrading to a higher patch version than kubeadm is ok if the user specifies --force. Not recommended, but possible. - skewErrors.Skippable = append(skewErrors.Skippable, errors.Errorf("Specified version to upgrade to %q is higher than the kubeadm version %q. Upgrade kubeadm first using the tool you used to install kubeadm", newK8sVersionStr, kubeadmVersionStr)) - } - } - - if kubeadmVersion.Major() > newK8sVersion.Major() || - kubeadmVersion.Minor() > newK8sVersion.Minor() { - skewErrors.Skippable = append(skewErrors.Skippable, errors.Errorf("Kubeadm version %s can only be used to upgrade to Kubernetes version %d.%d", kubeadmVersionStr, kubeadmVersion.Major(), kubeadmVersion.Minor())) - } - - // Detect if the version is unstable and the user didn't allow that - if err = detectUnstableVersionError(newK8sVersion, newK8sVersionStr, allowExperimentalUpgrades, allowRCUpgrades); err != nil { - skewErrors.Skippable = append(skewErrors.Skippable, err) - } - - // Detect if there are too old kubelets in the cluster - // Check for nil here since this is the only case where kubeletVersions can be nil; if KubeletVersions() returned an error - // However, it's okay to skip that check - if kubeletVersions != nil { - if err = detectTooOldKubelets(newK8sVersion, kubeletVersions); err != nil { - skewErrors.Skippable = append(skewErrors.Skippable, err) - } - } - - // If we did not see any errors, return nil - if len(skewErrors.Skippable) == 0 && len(skewErrors.Mandatory) == 0 { - return nil - } - - // Uh oh, we encountered one or more errors, return them - return skewErrors -} - -// detectUnstableVersionError is a helper function for detecting if the unstable version (if specified) is allowed to be used -func detectUnstableVersionError(newK8sVersion *version.Version, newK8sVersionStr string, allowExperimentalUpgrades, allowRCUpgrades bool) error { - // Short-circuit quickly if this is not an unstable version - if len(newK8sVersion.PreRelease()) == 0 { - return nil - } - // If the user has specified that unstable versions are fine, then no error should be returned - if allowExperimentalUpgrades { - return nil - } - // If this is a release candidate and we allow such ones, everything's fine - if strings.HasPrefix(newK8sVersion.PreRelease(), "rc") && allowRCUpgrades { - return nil - } - - return errors.Errorf("Specified version to upgrade to %q is an unstable version and such upgrades weren't allowed via setting the --allow-*-upgrades flags", newK8sVersionStr) -} - -// detectTooOldKubelets errors out if the kubelet versions are so old that an unsupported skew would happen if the cluster was upgraded -func detectTooOldKubelets(newK8sVersion *version.Version, kubeletVersions map[string]uint16) error { - tooOldKubeletVersions := []string{} - for versionStr := range kubeletVersions { - - kubeletVersion, err := version.ParseSemantic(versionStr) - if err != nil { - return errors.Errorf("couldn't parse kubelet version %s", versionStr) - } - - if newK8sVersion.Minor() > kubeletVersion.Minor()+MaximumAllowedMinorVersionKubeletSkew { - tooOldKubeletVersions = append(tooOldKubeletVersions, versionStr) - } - } - if len(tooOldKubeletVersions) == 0 { - return nil - } - - return errors.Errorf("There are kubelets in this cluster that are too old that have these versions %v", tooOldKubeletVersions) -} diff --git a/cmd/kubeadm/app/phases/upgrade/policy_test.go b/cmd/kubeadm/app/phases/upgrade/policy_test.go deleted file mode 100644 index 32a58250e96af..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/policy_test.go +++ /dev/null @@ -1,231 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "testing" - - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestEnforceVersionPolicies(t *testing.T) { - tests := []struct { - name string - vg *fakeVersionGetter - expectedMandatoryErrs int - expectedSkippableErrs int - allowExperimental, allowRCs bool - newK8sVersion string - }{ - { - name: "minor upgrade", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeadmVersion: constants.MinimumControlPlaneVersion.WithPatch(5).String(), - }, - newK8sVersion: constants.MinimumControlPlaneVersion.WithPatch(5).String(), - }, - { - name: "major upgrade", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumControlPlaneVersion.WithPatch(2).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.WithPatch(1).String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), - }, - { - name: "downgrade", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.String(), - kubeadmVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - }, - newK8sVersion: constants.MinimumControlPlaneVersion.WithPatch(2).String(), - }, - { - name: "same version upgrade", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - }, - newK8sVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - }, - { - name: "new version must be higher than v1.12.0", - vg: &fakeVersionGetter{ - clusterVersion: "v1.12.3", - kubeletVersion: "v1.12.3", - kubeadmVersion: "v1.12.3", - }, - newK8sVersion: "v1.11.10", - expectedMandatoryErrs: 1, // version must be higher than v1.12.0 - expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm - }, - { - name: "upgrading two minor versions in one go is not supported", - vg: &fakeVersionGetter{ - clusterVersion: "v1.11.3", - kubeletVersion: "v1.11.3", - kubeadmVersion: constants.CurrentKubernetesVersion.String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), - expectedMandatoryErrs: 1, // can't upgrade two minor versions - expectedSkippableErrs: 1, // kubelet <-> apiserver skew too large - }, - { - name: "downgrading two minor versions in one go is not supported", - vg: &fakeVersionGetter{ - clusterVersion: constants.CurrentKubernetesVersion.WithMinor(constants.CurrentKubernetesVersion.Minor() + 2).String(), - kubeletVersion: constants.CurrentKubernetesVersion.WithMinor(constants.CurrentKubernetesVersion.Minor() + 2).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), - expectedMandatoryErrs: 1, // can't downgrade two minor versions - }, - { - name: "kubeadm version must be higher than the new kube version. However, patch version skews may be forced", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - }, - newK8sVersion: constants.MinimumControlPlaneVersion.WithPatch(5).String(), - expectedSkippableErrs: 1, - }, - { - name: "kubeadm version must be higher than the new kube version. Trying to upgrade k8s to a higher minor version than kubeadm itself should never be supported", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), - expectedMandatoryErrs: 1, - }, - { - name: "the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though.", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: "v1.12.8", - kubeadmVersion: constants.CurrentKubernetesVersion.String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), - expectedSkippableErrs: 1, - }, - { - name: "experimental upgrades supported if the flag is set", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.WithPreRelease("beta.1").String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.WithPreRelease("beta.1").String(), - allowExperimental: true, - }, - { - name: "release candidate upgrades supported if the flag is set", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.WithPreRelease("rc.1").String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.WithPreRelease("rc.1").String(), - allowRCs: true, - }, - { - name: "release candidate upgrades supported if the flag is set", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.WithPreRelease("rc.1").String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.WithPreRelease("rc.1").String(), - allowExperimental: true, - }, - { - name: "the user should not be able to upgrade to an experimental version if they haven't opted into that", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.WithPreRelease("beta.1").String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.WithPreRelease("beta.1").String(), - allowRCs: true, - expectedSkippableErrs: 1, - }, - { - name: "the user should not be able to upgrade to an release candidate version if they haven't opted into that", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.WithPreRelease("rc.1").String(), - }, - newK8sVersion: constants.CurrentKubernetesVersion.WithPreRelease("rc.1").String(), - expectedSkippableErrs: 1, - }, - { - name: "the user can't use a newer minor version of kubeadm to upgrade an older version of kubeadm", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: constants.MinimumKubeletVersion.WithPatch(3).String(), - kubeadmVersion: constants.CurrentKubernetesVersion.String(), - }, - newK8sVersion: constants.MinimumControlPlaneVersion.WithPatch(6).String(), - expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm - }, - { - name: "build release supported at MinimumControlPlaneVersion", - vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.String(), - kubeletVersion: constants.MinimumControlPlaneVersion.String(), - kubeadmVersion: constants.MinimumControlPlaneVersion.WithBuildMetadata("build").String(), - }, - newK8sVersion: constants.MinimumControlPlaneVersion.WithBuildMetadata("build").String(), - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - - newK8sVer, err := version.ParseSemantic(rt.newK8sVersion) - if err != nil { - t.Fatalf("couldn't parse version %s: %v", rt.newK8sVersion, err) - } - - actualSkewErrs := EnforceVersionPolicies(rt.vg, rt.newK8sVersion, newK8sVer, rt.allowExperimental, rt.allowRCs) - if actualSkewErrs == nil { - // No errors were seen. Report unit test failure if we expected to see errors - if rt.expectedMandatoryErrs+rt.expectedSkippableErrs > 0 { - t.Errorf("failed TestEnforceVersionPolicies\n\texpected errors but got none") - } - // Otherwise, just move on with the next test - return - } - - if len(actualSkewErrs.Skippable) != rt.expectedSkippableErrs { - t.Errorf("failed TestEnforceVersionPolicies\n\texpected skippable errors: %d\n\tgot skippable errors: %d\n%#v\n%#v", rt.expectedSkippableErrs, len(actualSkewErrs.Skippable), *rt.vg, actualSkewErrs) - } - if len(actualSkewErrs.Mandatory) != rt.expectedMandatoryErrs { - t.Errorf("failed TestEnforceVersionPolicies\n\texpected mandatory errors: %d\n\tgot mandatory errors: %d\n%#v\n%#v", rt.expectedMandatoryErrs, len(actualSkewErrs.Mandatory), *rt.vg, actualSkewErrs) - } - }) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go deleted file mode 100644 index 1fe329edcb297..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ /dev/null @@ -1,283 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "os" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - errorsutil "k8s.io/apimachinery/pkg/util/errors" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" - nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" - kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" -) - -// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do -// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade -func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error { - errs := []error{} - - // Upload currently used configuration to the cluster - // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases - // depend on centralized information from this source in the future - if err := uploadconfig.UploadConfiguration(cfg, client); err != nil { - errs = append(errs, err) - } - - // Create the new, version-branched kubelet ComponentConfig ConfigMap - if err := kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil { - errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap")) - } - - // Write the new kubelet config down to disk and the env file if needed - if err := writeKubeletConfigFiles(client, cfg, dryRun); err != nil { - errs = append(errs, err) - } - - // Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or - // --cri-socket. - // TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly - if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { - errs = append(errs, errors.Wrap(err, "error uploading crisocket")) - } - - // Create RBAC rules that makes the bootstrap tokens able to get nodes - if err := nodebootstraptoken.AllowBoostrapTokensToGetNodes(client); err != nil { - errs = append(errs, err) - } - - // Create/update RBAC rules that makes the bootstrap tokens able to post CSRs - if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil { - errs = append(errs, err) - } - - // Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically - if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil { - errs = append(errs, err) - } - - // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically - if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil { - errs = append(errs, err) - } - - // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade - // Create the cluster-info ConfigMap with the associated RBAC rules - // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { - // return err - //} - // Create/update RBAC rules that makes the cluster-info ConfigMap reachable - if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil { - errs = append(errs, err) - } - - // If the coredns / kube-dns ConfigMaps are missing, show a warning and assume that the - // DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired. - // - // TODO: remove this once "kubeadm upgrade apply" phases are supported: - // https://github.com/kubernetes/kubeadm/issues/1318 - var missingCoreDNSConfigMap, missingKubeDNSConfigMap bool - if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get( - context.TODO(), - kubeadmconstants.CoreDNSConfigMap, - metav1.GetOptions{}, - ); err != nil && apierrors.IsNotFound(err) { - missingCoreDNSConfigMap = true - } - if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get( - context.TODO(), - kubeadmconstants.KubeDNSConfigMap, - metav1.GetOptions{}, - ); err != nil && apierrors.IsNotFound(err) { - missingKubeDNSConfigMap = true - } - if missingCoreDNSConfigMap && missingKubeDNSConfigMap { - klog.Warningf("the ConfigMaps %q/%q in the namespace %q were not found. "+ - "Assuming that a DNS server was not deployed for this cluster. "+ - "Note that once 'kubeadm upgrade apply' supports phases you "+ - "will have to skip the DNS upgrade manually", - kubeadmconstants.CoreDNSConfigMap, - kubeadmconstants.KubeDNSConfigMap, - metav1.NamespaceSystem) - } else { - // Upgrade CoreDNS/kube-dns - if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client); err != nil { - errs = append(errs, err) - } - // Remove the old DNS deployment if a new DNS service is now used (kube-dns to CoreDNS or vice versa) - if err := removeOldDNSDeploymentIfAnotherDNSIsUsed(&cfg.ClusterConfiguration, client, dryRun); err != nil { - errs = append(errs, err) - } - } - - // If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy - // was skipped during "kubeadm init", and that its redeployment on upgrade is not desired. - // - // TODO: remove this once "kubeadm upgrade apply" phases are supported: - // https://github.com/kubernetes/kubeadm/issues/1318 - if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get( - context.TODO(), - kubeadmconstants.KubeProxyConfigMap, - metav1.GetOptions{}, - ); err != nil && apierrors.IsNotFound(err) { - klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+ - "Assuming that kube-proxy was not deployed for this cluster. "+ - "Note that once 'kubeadm upgrade apply' supports phases you "+ - "will have to skip the kube-proxy upgrade manually", - kubeadmconstants.KubeProxyConfigMap, - metav1.NamespaceSystem) - } else { - // Upgrade kube-proxy - if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client); err != nil { - errs = append(errs, err) - } - } - - return errorsutil.NewAggregate(errs) -} - -func removeOldDNSDeploymentIfAnotherDNSIsUsed(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, dryRun bool) error { - return apiclient.TryRunCommand(func() error { - installedDeploymentName := kubeadmconstants.KubeDNSDeploymentName - deploymentToDelete := kubeadmconstants.CoreDNSDeploymentName - - if cfg.DNS.Type == kubeadmapi.CoreDNS { - installedDeploymentName = kubeadmconstants.CoreDNSDeploymentName - deploymentToDelete = kubeadmconstants.KubeDNSDeploymentName - } - - nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - FieldSelector: fields.Set{"spec.unschedulable": "false"}.AsSelector().String(), - }) - if err != nil { - return err - } - - // If we're dry-running or there are no scheduable nodes available, we don't need to wait for the new DNS addon to become ready - if !dryRun && len(nodes.Items) != 0 { - dnsDeployment, err := client.AppsV1().Deployments(metav1.NamespaceSystem).Get(context.TODO(), installedDeploymentName, metav1.GetOptions{}) - if err != nil { - return err - } - if dnsDeployment.Status.ReadyReplicas == 0 { - return errors.New("the DNS deployment isn't ready yet") - } - } - - // We don't want to wait for the DNS deployment above to become ready when dryrunning (as it never will) - // but here we should execute the DELETE command against the dryrun clientset, as it will only be logged - err = apiclient.DeleteDeploymentForeground(client, metav1.NamespaceSystem, deploymentToDelete) - if err != nil && !apierrors.IsNotFound(err) { - return err - } - return nil - }, 10) -} - -func writeKubeletConfigFiles(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, dryRun bool) error { - kubeletDir, err := GetKubeletDir(dryRun) - if err != nil { - // The error here should never occur in reality, would only be thrown if /tmp doesn't exist on the machine. - return err - } - errs := []error{} - // Write the configuration for the kubelet down to disk so the upgraded kubelet can start with fresh config - if err := kubeletphase.WriteConfigToDisk(&cfg.ClusterConfiguration, kubeletDir); err != nil { - errs = append(errs, errors.Wrap(err, "error writing kubelet configuration to file")) - } - - if dryRun { // Print what contents would be written - dryrunutil.PrintDryRunFile(kubeadmconstants.KubeletConfigurationFileName, kubeletDir, kubeadmconstants.KubeletRunDirectory, os.Stdout) - } - return errorsutil.NewAggregate(errs) -} - -// GetKubeletDir gets the kubelet directory based on whether the user is dry-running this command or not. -func GetKubeletDir(dryRun bool) (string, error) { - if dryRun { - return kubeadmconstants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun") - } - return kubeadmconstants.KubeletRunDirectory, nil -} - -// moveFiles moves files from one directory to another. -func moveFiles(files map[string]string) error { - filesToRecover := map[string]string{} - for from, to := range files { - if err := os.Rename(from, to); err != nil { - return rollbackFiles(filesToRecover, err) - } - filesToRecover[to] = from - } - return nil -} - -// rollbackFiles moves the files back to the original directory. -func rollbackFiles(files map[string]string, originalErr error) error { - errs := []error{originalErr} - for from, to := range files { - if err := os.Rename(from, to); err != nil { - errs = append(errs, err) - } - } - return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs)) -} - -// LabelOldControlPlaneNodes finds all nodes with the legacy node-role label and also applies -// the "control-plane" node-role label to them. -// TODO: https://github.com/kubernetes/kubeadm/issues/2200 -func LabelOldControlPlaneNodes(client clientset.Interface) error { - selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ - kubeadmconstants.LabelNodeRoleOldControlPlane: "", - })) - nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: selectorOldControlPlane.String(), - }) - if err != nil { - return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleOldControlPlane) - } - - for _, n := range nodesWithOldLabel.Items { - if _, hasNewLabel := n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane]; hasNewLabel { - continue - } - err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { - n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane] = "" - }) - if err != nil { - return err - } - } - return nil -} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go b/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go deleted file mode 100644 index 04ede0a0e3ea7..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade_test.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func TestMoveFiles(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - os.Chmod(tmpdir, 0766) - - certPath := filepath.Join(tmpdir, constants.APIServerCertName) - certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) - if err != nil { - t.Fatalf("Failed to create cert file %s: %v", certPath, err) - } - defer certFile.Close() - - keyPath := filepath.Join(tmpdir, constants.APIServerKeyName) - keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) - if err != nil { - t.Fatalf("Failed to create key file %s: %v", keyPath, err) - } - defer keyFile.Close() - - subDir := filepath.Join(tmpdir, "expired") - if err := os.Mkdir(subDir, 0766); err != nil { - t.Fatalf("Failed to create backup directory %s: %v", subDir, err) - } - - filesToMove := map[string]string{ - filepath.Join(tmpdir, constants.APIServerCertName): filepath.Join(subDir, constants.APIServerCertName), - filepath.Join(tmpdir, constants.APIServerKeyName): filepath.Join(subDir, constants.APIServerKeyName), - } - - if err := moveFiles(filesToMove); err != nil { - t.Fatalf("Failed to move files %v: %v", filesToMove, err) - } -} - -func TestRollbackFiles(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - os.Chmod(tmpdir, 0766) - - subDir := filepath.Join(tmpdir, "expired") - if err := os.Mkdir(subDir, 0766); err != nil { - t.Fatalf("Failed to create backup directory %s: %v", subDir, err) - } - - certPath := filepath.Join(subDir, constants.APIServerCertName) - certFile, err := os.OpenFile(certPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) - if err != nil { - t.Fatalf("Failed to create cert file %s: %v", certPath, err) - } - defer certFile.Close() - - keyPath := filepath.Join(subDir, constants.APIServerKeyName) - keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) - if err != nil { - t.Fatalf("Failed to create key file %s: %v", keyPath, err) - } - defer keyFile.Close() - - filesToRollBack := map[string]string{ - filepath.Join(subDir, constants.APIServerCertName): filepath.Join(tmpdir, constants.APIServerCertName), - filepath.Join(subDir, constants.APIServerKeyName): filepath.Join(tmpdir, constants.APIServerKeyName), - } - - errString := "there are files need roll back" - originalErr := errors.New(errString) - err = rollbackFiles(filesToRollBack, originalErr) - if err == nil { - t.Fatalf("Expected error contains %q, got nil", errString) - } - if !strings.Contains(err.Error(), errString) { - t.Fatalf("Expected error contains %q, got %v", errString, err) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/preflight.go b/cmd/kubeadm/app/phases/upgrade/preflight.go deleted file mode 100644 index 536c60ab64b27..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/preflight.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "fmt" - "os" - - "github.com/coredns/corefile-migration/migration" - "github.com/pkg/errors" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" -) - -// CoreDNSCheck validates installed kubelet version -type CoreDNSCheck struct { - name string - client clientset.Interface - f func(clientset.Interface) error -} - -// Name is part of the preflight.Checker interface -func (c CoreDNSCheck) Name() string { - return c.name -} - -// Check is part of the preflight.Checker interface -func (c CoreDNSCheck) Check() (warnings, errors []error) { - if err := c.f(c.client); err != nil { - return nil, []error{err} - } - return nil, nil -} - -// RunCoreDNSMigrationCheck initializes checks related to CoreDNS migration. -func RunCoreDNSMigrationCheck(client clientset.Interface, ignorePreflightErrors sets.String, dnsType kubeadmapi.DNSAddOnType) error { - if dnsType != kubeadmapi.CoreDNS { - return nil - } - migrationChecks := []preflight.Checker{ - &CoreDNSCheck{ - name: "CoreDNSUnsupportedPlugins", - client: client, - f: checkUnsupportedPlugins, - }, - &CoreDNSCheck{ - name: "CoreDNSMigration", - client: client, - f: checkMigration, - }, - &CoreDNSCheck{ - name: "kubeDNSTranslation", - client: client, - f: checkKubeDNSConfigMap, - }, - } - - return preflight.RunChecks(migrationChecks, os.Stderr, ignorePreflightErrors) -} - -// checkUnsupportedPlugins checks if there are any plugins included in the current configuration -// that are unsupported for migration. -func checkUnsupportedPlugins(client clientset.Interface) error { - klog.V(1).Infoln("validating if there are any unsupported CoreDNS plugins in the Corefile") - _, corefile, currentInstalledCoreDNSversion, err := dns.GetCoreDNSInfo(client) - if err != nil { - return err - } - unsupportedCoreDNS, err := migration.Unsupported(currentInstalledCoreDNSversion, currentInstalledCoreDNSversion, corefile) - if err != nil { - return err - } - if len(unsupportedCoreDNS) != 0 { - var UnsupportedPlugins []string - for _, unsup := range unsupportedCoreDNS { - UnsupportedPlugins = append(UnsupportedPlugins, unsup.ToString()) - } - fmt.Println("[preflight] The corefile contains plugins that kubeadm/CoreDNS does not know how to migrate. " + - "Each plugin listed should be manually verified for compatibility with the newer version of CoreDNS. " + - "Once ready, the upgrade can be initiated by skipping the preflight check. During the upgrade, " + - "kubeadm will migrate the configuration while leaving the listed plugin configs untouched, " + - "but cannot guarantee that they will work with the newer version of CoreDNS.") - return errors.Errorf("CoreDNS cannot migrate the following plugins:\n%s", UnsupportedPlugins) - } - return nil -} - -// checkMigration validates if migration of the current CoreDNS ConfigMap is possible. -func checkMigration(client clientset.Interface) error { - klog.V(1).Infoln("validating if migration can be done for the current CoreDNS release.") - _, corefile, currentInstalledCoreDNSversion, err := dns.GetCoreDNSInfo(client) - if err != nil { - return err - } - - _, err = migration.Migrate(currentInstalledCoreDNSversion, kubeadmconstants.CoreDNSVersion, corefile, false) - if err != nil { - return errors.Wrap(err, "CoreDNS will not be upgraded") - } - return nil -} - -// checkKubeDNSConfigMap checks if the translation of kube-dns to CoreDNS ConfigMap is supported -func checkKubeDNSConfigMap(client clientset.Interface) error { - kubeDNSConfigMap, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.KubeDNSConfigMap, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return err - } - - if _, ok := kubeDNSConfigMap.Data["federations"]; ok { - klog.V(1).Infoln("CoreDNS no longer supports Federation and " + - "hence will not translate the federation data from kube-dns to CoreDNS ConfigMap") - return errors.New("kube-dns Federation data will not be translated") - } - return nil -} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods.go b/cmd/kubeadm/app/phases/upgrade/staticpods.go deleted file mode 100644 index 26b43b850c8cc..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/staticpods.go +++ /dev/null @@ -1,647 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "github.com/pkg/errors" - - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" - "k8s.io/kubernetes/cmd/kubeadm/app/util/image" - "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" -) - -const ( - // UpgradeManifestTimeout is timeout of upgrading the static pod manifest - UpgradeManifestTimeout = 5 * time.Minute -) - -// StaticPodPathManager is responsible for tracking the directories used in the static pod upgrade transition -type StaticPodPathManager interface { - // MoveFile should move a file from oldPath to newPath - MoveFile(oldPath, newPath string) error - // KubernetesDir is the directory Kubernetes owns for storing various configuration files - KubernetesDir() string - // PatchesDir should point to the folder where patches for components are stored - PatchesDir() string - // RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet - RealManifestPath(component string) string - // RealManifestDir should point to the static pod manifest directory used by the kubelet - RealManifestDir() string - // TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade - TempManifestPath(component string) string - // TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade - TempManifestDir() string - // BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition - BackupManifestPath(component string) string - // BackupManifestDir should point to the backup directory used for backuping manifests during the transition - BackupManifestDir() string - // BackupEtcdDir should point to the backup directory used for backuping manifests during the transition - BackupEtcdDir() string - // CleanupDirs cleans up all temporary directories - CleanupDirs() error -} - -// KubeStaticPodPathManager is a real implementation of StaticPodPathManager that is used when upgrading a static pod cluster -type KubeStaticPodPathManager struct { - kubernetesDir string - patchesDir string - realManifestDir string - tempManifestDir string - backupManifestDir string - backupEtcdDir string - - keepManifestDir bool - keepEtcdDir bool -} - -// NewKubeStaticPodPathManager creates a new instance of KubeStaticPodPathManager -func NewKubeStaticPodPathManager(kubernetesDir, patchesDir, tempDir, backupDir, backupEtcdDir string, keepManifestDir, keepEtcdDir bool) StaticPodPathManager { - return &KubeStaticPodPathManager{ - kubernetesDir: kubernetesDir, - patchesDir: patchesDir, - realManifestDir: filepath.Join(kubernetesDir, constants.ManifestsSubDirName), - tempManifestDir: tempDir, - backupManifestDir: backupDir, - backupEtcdDir: backupEtcdDir, - keepManifestDir: keepManifestDir, - keepEtcdDir: keepEtcdDir, - } -} - -// NewKubeStaticPodPathManagerUsingTempDirs creates a new instance of KubeStaticPodPathManager with temporary directories backing it -func NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, patchesDir string, saveManifestsDir, saveEtcdDir bool) (StaticPodPathManager, error) { - - upgradedManifestsDir, err := constants.CreateTempDirForKubeadm(kubernetesDir, "kubeadm-upgraded-manifests") - if err != nil { - return nil, err - } - backupManifestsDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-manifests") - if err != nil { - return nil, err - } - backupEtcdDir, err := constants.CreateTimestampDirForKubeadm(kubernetesDir, "kubeadm-backup-etcd") - if err != nil { - return nil, err - } - - return NewKubeStaticPodPathManager(kubernetesDir, patchesDir, upgradedManifestsDir, backupManifestsDir, backupEtcdDir, saveManifestsDir, saveEtcdDir), nil -} - -// MoveFile should move a file from oldPath to newPath -func (spm *KubeStaticPodPathManager) MoveFile(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) -} - -// KubernetesDir should point to the directory Kubernetes owns for storing various configuration files -func (spm *KubeStaticPodPathManager) KubernetesDir() string { - return spm.kubernetesDir -} - -// PatchesDir should point to the folder where patches for components are stored -func (spm *KubeStaticPodPathManager) PatchesDir() string { - return spm.patchesDir -} - -// RealManifestPath gets the file path for the component in the "real" static pod manifest directory used by the kubelet -func (spm *KubeStaticPodPathManager) RealManifestPath(component string) string { - return constants.GetStaticPodFilepath(component, spm.realManifestDir) -} - -// RealManifestDir should point to the static pod manifest directory used by the kubelet -func (spm *KubeStaticPodPathManager) RealManifestDir() string { - return spm.realManifestDir -} - -// TempManifestPath gets the file path for the component in the temporary directory created for generating new manifests for the upgrade -func (spm *KubeStaticPodPathManager) TempManifestPath(component string) string { - return constants.GetStaticPodFilepath(component, spm.tempManifestDir) -} - -// TempManifestDir should point to the temporary directory created for generating new manifests for the upgrade -func (spm *KubeStaticPodPathManager) TempManifestDir() string { - return spm.tempManifestDir -} - -// BackupManifestPath gets the file path for the component in the backup directory used for backuping manifests during the transition -func (spm *KubeStaticPodPathManager) BackupManifestPath(component string) string { - return constants.GetStaticPodFilepath(component, spm.backupManifestDir) -} - -// BackupManifestDir should point to the backup directory used for backuping manifests during the transition -func (spm *KubeStaticPodPathManager) BackupManifestDir() string { - return spm.backupManifestDir -} - -// BackupEtcdDir should point to the backup directory used for backuping manifests during the transition -func (spm *KubeStaticPodPathManager) BackupEtcdDir() string { - return spm.backupEtcdDir -} - -// CleanupDirs cleans up all temporary directories except those the user has requested to keep around -func (spm *KubeStaticPodPathManager) CleanupDirs() error { - var errlist []error - if err := os.RemoveAll(spm.TempManifestDir()); err != nil { - errlist = append(errlist, err) - } - if !spm.keepManifestDir { - if err := os.RemoveAll(spm.BackupManifestDir()); err != nil { - errlist = append(errlist, err) - } - } - - if !spm.keepEtcdDir { - if err := os.RemoveAll(spm.BackupEtcdDir()); err != nil { - errlist = append(errlist, err) - } - } - - return utilerrors.NewAggregate(errlist) -} - -func upgradeComponent(component string, certsRenewMgr *renewal.Manager, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, beforePodHash string, recoverManifests map[string]string) error { - // Special treatment is required for etcd case, when rollbackOldManifests should roll back etcd - // manifests only for the case when component is Etcd - recoverEtcd := false - if component == constants.Etcd { - recoverEtcd = true - } - - fmt.Printf("[upgrade/staticpods] Preparing for %q upgrade\n", component) - - // The old manifest is here; in the /etc/kubernetes/manifests/ - currentManifestPath := pathMgr.RealManifestPath(component) - // The new, upgraded manifest will be written here - newManifestPath := pathMgr.TempManifestPath(component) - // The old manifest will be moved here; into a subfolder of the temporary directory - // If a rollback is needed, these manifests will be put back to where they where initially - backupManifestPath := pathMgr.BackupManifestPath(component) - - // Store the backup path in the recover list. If something goes wrong now, this component will be rolled back. - recoverManifests[component] = backupManifestPath - - // Skip upgrade if current and new manifests are equal - equal, err := staticpod.ManifestFilesAreEqual(currentManifestPath, newManifestPath) - if err != nil { - return err - } - if equal { - fmt.Printf("[upgrade/staticpods] Current and new manifests of %s are equal, skipping upgrade\n", component) - return nil - } - - // if certificate renewal should be performed - if certsRenewMgr != nil { - // renew all the certificates used by the current component - if err := renewCertsByComponent(cfg, component, certsRenewMgr); err != nil { - return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to renew certificates for component %q", component), pathMgr, recoverEtcd) - } - } - - // Move the old manifest into the old-manifests directory - if err := pathMgr.MoveFile(currentManifestPath, backupManifestPath); err != nil { - return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd) - } - - // Move the new manifest into the manifests directory - if err := pathMgr.MoveFile(newManifestPath, currentManifestPath); err != nil { - return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd) - } - - fmt.Printf("[upgrade/staticpods] Moved new manifest to %q and backed up old manifest to %q\n", currentManifestPath, backupManifestPath) - - fmt.Println("[upgrade/staticpods] Waiting for the kubelet to restart the component") - fmt.Printf("[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout %v)\n", UpgradeManifestTimeout) - - // Wait for the mirror Pod hash to change; otherwise we'll run into race conditions here when the kubelet hasn't had time to - // notice the removal of the Static Pod, leading to a false positive below where we check that the API endpoint is healthy - // If we don't do this, there is a case where we remove the Static Pod manifest, kubelet is slow to react, kubeadm checks the - // API endpoint below of the OLD Static Pod component and proceeds quickly enough, which might lead to unexpected results. - if err := waiter.WaitForStaticPodHashChange(cfg.NodeRegistration.Name, component, beforePodHash); err != nil { - return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd) - } - - // Wait for the static pod component to come up and register itself as a mirror pod - if err := waiter.WaitForPodsWithLabel("component=" + component); err != nil { - return rollbackOldManifests(recoverManifests, err, pathMgr, recoverEtcd) - } - - fmt.Printf("[upgrade/staticpods] Component %q upgraded successfully!\n", component) - - return nil -} - -// performEtcdStaticPodUpgrade performs upgrade of etcd, it returns bool which indicates fatal error or not and the actual error. -func performEtcdStaticPodUpgrade(certsRenewMgr *renewal.Manager, client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, recoverManifests map[string]string, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) (bool, error) { - // Add etcd static pod spec only if external etcd is not configured - if cfg.Etcd.External != nil { - return false, errors.New("external etcd detected, won't try to change any etcd state") - } - - // Checking health state of etcd before proceeding with the upgrade - err := oldEtcdClient.CheckClusterHealth() - if err != nil { - return true, errors.Wrap(err, "etcd cluster is not healthy") - } - - // Backing up etcd data store - backupEtcdDir := pathMgr.BackupEtcdDir() - runningEtcdDir := cfg.Etcd.Local.DataDir - if err := kubeadmutil.CopyDir(runningEtcdDir, backupEtcdDir); err != nil { - return true, errors.Wrap(err, "failed to back up etcd data") - } - - // Get the desired etcd version. That's either the one specified by the user in cfg.Etcd.Local.ImageTag - // or the kubeadm preferred one for the desired Kubernetes version - var desiredEtcdVersion *version.Version - if cfg.Etcd.Local.ImageTag != "" { - desiredEtcdVersion, err = version.ParseSemantic(cfg.Etcd.Local.ImageTag) - if err != nil { - return true, errors.Wrapf(err, "failed to parse tag %q as a semantic version", cfg.Etcd.Local.ImageTag) - } - } else { - // Need to check currently used version and version from constants, if differs then upgrade - var warning error - desiredEtcdVersion, warning, err = constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, cfg.KubernetesVersion) - if err != nil { - return true, errors.Wrap(err, "failed to retrieve an etcd version for the target Kubernetes version") - } - if warning != nil { - klog.Warningf("[upgrade/etcd] %v", warning) - } - } - - // Get the etcd version of the local/stacked etcd member running on the current machine - currentEtcdVersionStr, err := GetEtcdImageTagFromStaticPod(pathMgr.RealManifestDir()) - if err != nil { - return true, errors.Wrap(err, "failed to retrieve the current etcd version") - } - - cmpResult, err := desiredEtcdVersion.Compare(currentEtcdVersionStr) - if err != nil { - return true, errors.Wrapf(err, "failed comparing the current etcd version %q to the desired one %q", currentEtcdVersionStr, desiredEtcdVersion) - } - if cmpResult < 0 { - return false, errors.Errorf("the desired etcd version %q is older than the currently installed %q. Skipping etcd upgrade", desiredEtcdVersion, currentEtcdVersionStr) - } - - beforeEtcdPodHash, err := waiter.WaitForStaticPodSingleHash(cfg.NodeRegistration.Name, constants.Etcd) - if err != nil { - return true, errors.Wrap(err, "failed to get etcd pod's hash") - } - - // Write the updated etcd static Pod manifest into the temporary directory, at this point no etcd change - // has occurred in any aspects. - if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.TempManifestDir(), pathMgr.PatchesDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil { - return true, errors.Wrap(err, "error creating local etcd static pod manifest file") - } - - retries := 10 - retryInterval := 15 * time.Second - - // Perform etcd upgrade using common to all control plane components function - if err := upgradeComponent(constants.Etcd, certsRenewMgr, waiter, pathMgr, cfg, beforeEtcdPodHash, recoverManifests); err != nil { - fmt.Printf("[upgrade/etcd] Failed to upgrade etcd: %v\n", err) - // Since upgrade component failed, the old etcd manifest has either been restored or was never touched - // Now we need to check the health of etcd cluster if it is up with old manifest - fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available") - if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil { - fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err) - - // At this point we know that etcd cluster is dead and it is safe to copy backup datastore and to rollback old etcd manifest - fmt.Println("[upgrade/etcd] Rolling back etcd data") - if err := rollbackEtcdData(cfg, pathMgr); err != nil { - // Even copying back datastore failed, no options for recovery left, bailing out - return true, errors.Errorf("fatal error rolling back local etcd cluster datadir: %v, the backup of etcd database is stored here:(%s)", err, backupEtcdDir) - } - fmt.Println("[upgrade/etcd] Etcd data rollback successful") - - // Now that we've rolled back the data, let's check if the cluster comes up - fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available") - if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil { - fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err) - // Nothing else left to try to recover etcd cluster - return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster manifest, the backup of etcd database is stored here:(%s)", backupEtcdDir) - } - - // We've recovered to the previous etcd from this case - } - fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available") - - // Since etcd cluster came back up with the old manifest - return true, errors.Wrap(err, "fatal error when trying to upgrade the etcd cluster, rolled the state back to pre-upgrade state") - } - - // Initialize the new etcd client if it wasn't pre-initialized - if newEtcdClient == nil { - etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) - if err != nil { - return true, errors.Wrap(err, "fatal error creating etcd client") - } - newEtcdClient = etcdClient - } - - // Checking health state of etcd after the upgrade - fmt.Println("[upgrade/etcd] Waiting for etcd to become available") - if _, err = newEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil { - fmt.Printf("[upgrade/etcd] Failed to healthcheck etcd: %v\n", err) - // Despite the fact that upgradeComponent was successful, there is something wrong with the etcd cluster - // First step is to restore back up of datastore - fmt.Println("[upgrade/etcd] Rolling back etcd data") - if err := rollbackEtcdData(cfg, pathMgr); err != nil { - // Even copying back datastore failed, no options for recovery left, bailing out - return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster datadir, the backup of etcd database is stored here:(%s)", backupEtcdDir) - } - fmt.Println("[upgrade/etcd] Etcd data rollback successful") - - // Old datastore has been copied, rolling back old manifests - fmt.Println("[upgrade/etcd] Rolling back etcd manifest") - rollbackOldManifests(recoverManifests, err, pathMgr, true) - // rollbackOldManifests() always returns an error -- ignore it and continue - - // Assuming rollback of the old etcd manifest was successful, check the status of etcd cluster again - fmt.Println("[upgrade/etcd] Waiting for previous etcd to become available") - if _, err := oldEtcdClient.WaitForClusterAvailable(retries, retryInterval); err != nil { - fmt.Printf("[upgrade/etcd] Failed to healthcheck previous etcd: %v\n", err) - // Nothing else left to try to recover etcd cluster - return true, errors.Wrapf(err, "fatal error rolling back local etcd cluster manifest, the backup of etcd database is stored here:(%s)", backupEtcdDir) - } - fmt.Println("[upgrade/etcd] Etcd was rolled back and is now available") - - // We've successfully rolled back etcd, and now return an error describing that the upgrade failed - return true, errors.Wrap(err, "fatal error upgrading local etcd cluster, rolled the state back to pre-upgrade state") - } - - return false, nil -} - -// StaticPodControlPlane upgrades a static pod-hosted control plane -func StaticPodControlPlane(client clientset.Interface, waiter apiclient.Waiter, pathMgr StaticPodPathManager, cfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, oldEtcdClient, newEtcdClient etcdutil.ClusterInterrogator) error { - recoverManifests := map[string]string{} - var isExternalEtcd bool - - beforePodHashMap, err := waiter.WaitForStaticPodControlPlaneHashes(cfg.NodeRegistration.Name) - if err != nil { - return err - } - - if oldEtcdClient == nil { - if cfg.Etcd.External != nil { - // External etcd - isExternalEtcd = true - etcdClient, err := etcdutil.New( - cfg.Etcd.External.Endpoints, - cfg.Etcd.External.CAFile, - cfg.Etcd.External.CertFile, - cfg.Etcd.External.KeyFile, - ) - if err != nil { - return errors.Wrap(err, "failed to create etcd client for external etcd") - } - oldEtcdClient = etcdClient - // Since etcd is managed externally, the new etcd client will be the same as the old client - if newEtcdClient == nil { - newEtcdClient = etcdClient - } - } else { - // etcd Static Pod - etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir) - if err != nil { - return errors.Wrap(err, "failed to create etcd client") - } - oldEtcdClient = etcdClient - } - } - - var certsRenewMgr *renewal.Manager - if renewCerts { - certsRenewMgr, err = renewal.NewManager(&cfg.ClusterConfiguration, pathMgr.KubernetesDir()) - if err != nil { - return errors.Wrap(err, "failed to create the certificate renewal manager") - } - } - - // etcd upgrade is done prior to other control plane components - if !isExternalEtcd && etcdUpgrade { - // set the TLS upgrade flag for all components - fmt.Printf("[upgrade/etcd] Upgrading to TLS for %s\n", constants.Etcd) - - // Perform etcd upgrade using common to all control plane components function - fatal, err := performEtcdStaticPodUpgrade(certsRenewMgr, client, waiter, pathMgr, cfg, recoverManifests, oldEtcdClient, newEtcdClient) - if err != nil { - if fatal { - return err - } - fmt.Printf("[upgrade/etcd] Non fatal issue encountered during upgrade: %v\n", err) - } - } - - // Write the updated static Pod manifests into the temporary directory - fmt.Printf("[upgrade/staticpods] Writing new Static Pod manifests to %q\n", pathMgr.TempManifestDir()) - err = controlplane.CreateInitStaticPodManifestFiles(pathMgr.TempManifestDir(), pathMgr.PatchesDir(), cfg) - if err != nil { - return errors.Wrap(err, "error creating init static pod manifest files") - } - - for _, component := range constants.ControlPlaneComponents { - if err = upgradeComponent(component, certsRenewMgr, waiter, pathMgr, cfg, beforePodHashMap[component], recoverManifests); err != nil { - return err - } - } - - if renewCerts { - // renew the certificate embedded in the admin.conf file - renewed, err := certsRenewMgr.RenewUsingLocalCA(constants.AdminKubeConfigFileName) - if err != nil { - return rollbackOldManifests(recoverManifests, errors.Wrapf(err, "failed to upgrade the %s certificates", constants.AdminKubeConfigFileName), pathMgr, false) - } - - if !renewed { - // if not error, but not renewed because of external CA detected, inform the user - fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", constants.AdminKubeConfigFileName) - } - } - - // Remove the temporary directories used on a best-effort (don't fail if the calls error out) - // The calls are set here by design; we should _not_ use "defer" above as that would remove the directories - // even in the "fail and rollback" case, where we want the directories preserved for the user. - return pathMgr.CleanupDirs() -} - -// rollbackOldManifests rolls back the backed-up manifests if something went wrong. -// It always returns an error to the caller. -func rollbackOldManifests(oldManifests map[string]string, origErr error, pathMgr StaticPodPathManager, restoreEtcd bool) error { - errs := []error{origErr} - for component, backupPath := range oldManifests { - // Will restore etcd manifest only if it was explicitly requested by setting restoreEtcd to True - if component == constants.Etcd && !restoreEtcd { - continue - } - // Where we should put back the backed up manifest - realManifestPath := pathMgr.RealManifestPath(component) - - // Move the backup manifest back into the manifests directory - err := pathMgr.MoveFile(backupPath, realManifestPath) - if err != nil { - errs = append(errs, err) - } - } - // Let the user know there were problems, but we tried to recover - return errors.Wrap(utilerrors.NewAggregate(errs), - "couldn't upgrade control plane. kubeadm has tried to recover everything into the earlier state. Errors faced") -} - -// rollbackEtcdData rolls back the content of etcd folder if something went wrong. -// When the folder contents are successfully rolled back, nil is returned, otherwise an error is returned. -func rollbackEtcdData(cfg *kubeadmapi.InitConfiguration, pathMgr StaticPodPathManager) error { - backupEtcdDir := pathMgr.BackupEtcdDir() - runningEtcdDir := cfg.Etcd.Local.DataDir - - if err := kubeadmutil.CopyDir(backupEtcdDir, runningEtcdDir); err != nil { - // Let the user know there we're problems, but we tried to reçover - return errors.Wrapf(err, "couldn't recover etcd database with error, the location of etcd backup: %s ", backupEtcdDir) - } - - return nil -} - -// renewCertsByComponent takes charge of renewing certificates used by a specific component before -// the static pod of the component is upgraded -func renewCertsByComponent(cfg *kubeadmapi.InitConfiguration, component string, certsRenewMgr *renewal.Manager) error { - var certificates []string - - // if etcd, only in case of local etcd, renew server, peer and health check certificate - if component == constants.Etcd { - if cfg.Etcd.Local != nil { - certificates = []string{ - certsphase.KubeadmCertEtcdServer().Name, - certsphase.KubeadmCertEtcdPeer().Name, - certsphase.KubeadmCertEtcdHealthcheck().Name, - } - } - } - - // if apiserver, renew apiserver serving certificate, kubelet and front-proxy client certificate. - //if local etcd, renew also the etcd client certificate - if component == constants.KubeAPIServer { - certificates = []string{ - certsphase.KubeadmCertAPIServer().Name, - certsphase.KubeadmCertKubeletClient().Name, - certsphase.KubeadmCertFrontProxyClient().Name, - } - if cfg.Etcd.Local != nil { - certificates = append(certificates, certsphase.KubeadmCertEtcdAPIClient().Name) - } - } - - // if controller-manager, renew the certificate embedded in the controller-manager kubeConfig file - if component == constants.KubeControllerManager { - certificates = []string{ - constants.ControllerManagerKubeConfigFileName, - } - } - - // if scheduler, renew the certificate embedded in the scheduler kubeConfig file - if component == constants.KubeScheduler { - certificates = []string{ - constants.SchedulerKubeConfigFileName, - } - } - - // renew the selected components - for _, cert := range certificates { - fmt.Printf("[upgrade/staticpods] Renewing %s certificate\n", cert) - renewed, err := certsRenewMgr.RenewUsingLocalCA(cert) - if err != nil { - return err - } - if !renewed { - // if not error, but not renewed because of external CA detected, inform the user - fmt.Printf("[upgrade/staticpods] External CA detected, %s certificate can't be renewed\n", cert) - } - } - - return nil -} - -// GetPathManagerForUpgrade returns a path manager properly configured for the given InitConfiguration. -func GetPathManagerForUpgrade(kubernetesDir, patchesDir string, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade bool) (StaticPodPathManager, error) { - isExternalEtcd := internalcfg.Etcd.External != nil - return NewKubeStaticPodPathManagerUsingTempDirs(kubernetesDir, patchesDir, true, etcdUpgrade && !isExternalEtcd) -} - -// PerformStaticPodUpgrade performs the upgrade of the control plane components for a static pod hosted cluster -func PerformStaticPodUpgrade(client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration, etcdUpgrade, renewCerts bool, patchesDir string) error { - pathManager, err := GetPathManagerForUpgrade(constants.KubernetesDir, patchesDir, internalcfg, etcdUpgrade) - if err != nil { - return err - } - - // The arguments oldEtcdClient and newEtdClient, are uninitialized because passing in the clients allow for mocking the client during testing - return StaticPodControlPlane(client, waiter, pathManager, internalcfg, etcdUpgrade, renewCerts, nil, nil) -} - -// DryRunStaticPodUpgrade fakes an upgrade of the control plane -func DryRunStaticPodUpgrade(patchesDir string, internalcfg *kubeadmapi.InitConfiguration) error { - - dryRunManifestDir, err := constants.CreateTempDirForKubeadm("", "kubeadm-upgrade-dryrun") - if err != nil { - return err - } - defer os.RemoveAll(dryRunManifestDir) - if err := controlplane.CreateInitStaticPodManifestFiles(dryRunManifestDir, patchesDir, internalcfg); err != nil { - return err - } - - // Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests - files := []dryrunutil.FileToPrint{} - for _, component := range constants.ControlPlaneComponents { - realPath := constants.GetStaticPodFilepath(component, dryRunManifestDir) - outputPath := constants.GetStaticPodFilepath(component, constants.GetStaticPodDirectory()) - files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath)) - } - - return dryrunutil.PrintDryRunFiles(files, os.Stdout) -} - -// GetEtcdImageTagFromStaticPod returns the image tag of the local etcd static pod -func GetEtcdImageTagFromStaticPod(manifestDir string) (string, error) { - realPath := constants.GetStaticPodFilepath(constants.Etcd, manifestDir) - pod, err := staticpod.ReadStaticPodFromDisk(realPath) - if err != nil { - return "", err - } - - return image.TagFromImage(pod.Spec.Containers[0].Image), nil -} diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go deleted file mode 100644 index 15c28c227625e..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ /dev/null @@ -1,1017 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "crypto/sha256" - "crypto/x509" - "fmt" - "io/ioutil" - "math/big" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/pkg/errors" - "go.etcd.io/etcd/pkg/transport" - - "k8s.io/client-go/tools/clientcmd" - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/renewal" - controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane" - etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd" - kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -const ( - waitForHashes = "wait-for-hashes" - waitForHashChange = "wait-for-hash-change" - waitForPodsWithLabel = "wait-for-pods-with-label" - - testConfiguration = ` -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -nodeRegistration: - name: foo - criSocket: "" -localAPIEndpoint: - advertiseAddress: 192.168.2.2 - bindPort: 6443 -bootstrapTokens: -- token: ce3aa5.5ec8455bb76b379f - ttl: 24h ---- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration - -apiServer: - certSANs: null - extraArgs: null -certificatesDir: %s -etcd: - local: - dataDir: %s - image: "" -imageRepository: k8s.gcr.io -kubernetesVersion: %s -networking: - dnsDomain: cluster.local - podSubnet: "" - serviceSubnet: 10.96.0.0/12 -useHyperKubeImage: false -` -) - -// fakeWaiter is a fake apiclient.Waiter that returns errors it was initialized with -type fakeWaiter struct { - errsToReturn map[string]error -} - -func NewFakeStaticPodWaiter(errsToReturn map[string]error) apiclient.Waiter { - return &fakeWaiter{ - errsToReturn: errsToReturn, - } -} - -// WaitForAPI just returns a dummy nil, to indicate that the program should just proceed -func (w *fakeWaiter) WaitForAPI() error { - return nil -} - -// WaitForPodsWithLabel just returns an error if set from errsToReturn -func (w *fakeWaiter) WaitForPodsWithLabel(kvLabel string) error { - return w.errsToReturn[waitForPodsWithLabel] -} - -// WaitForPodToDisappear just returns a dummy nil, to indicate that the program should just proceed -func (w *fakeWaiter) WaitForPodToDisappear(podName string) error { - return nil -} - -// SetTimeout is a no-op; we don't use it in this implementation -func (w *fakeWaiter) SetTimeout(_ time.Duration) {} - -// WaitForStaticPodControlPlaneHashes returns an error if set from errsToReturn -func (w *fakeWaiter) WaitForStaticPodControlPlaneHashes(_ string) (map[string]string, error) { - return map[string]string{}, w.errsToReturn[waitForHashes] -} - -// WaitForStaticPodSingleHash returns an error if set from errsToReturn -func (w *fakeWaiter) WaitForStaticPodSingleHash(_ string, _ string) (string, error) { - return "", w.errsToReturn[waitForHashes] -} - -// WaitForStaticPodHashChange returns an error if set from errsToReturn -func (w *fakeWaiter) WaitForStaticPodHashChange(_, _, _ string) error { - return w.errsToReturn[waitForHashChange] -} - -// WaitForHealthyKubelet returns a dummy nil just to implement the interface -func (w *fakeWaiter) WaitForHealthyKubelet(_ time.Duration, _ string) error { - return nil -} - -// WaitForKubeletAndFunc is a wrapper for WaitForHealthyKubelet that also blocks for a function -func (w *fakeWaiter) WaitForKubeletAndFunc(f func() error) error { - return nil -} - -type fakeStaticPodPathManager struct { - kubernetesDir string - patchesDir string - realManifestDir string - tempManifestDir string - backupManifestDir string - backupEtcdDir string - MoveFileFunc func(string, string) error -} - -func NewFakeStaticPodPathManager(moveFileFunc func(string, string) error) (StaticPodPathManager, error) { - kubernetesDir, err := ioutil.TempDir("", "kubeadm-pathmanager-") - if err != nil { - return nil, errors.Wrapf(err, "couldn't create a temporary directory for the upgrade") - } - - realManifestDir := filepath.Join(kubernetesDir, constants.ManifestsSubDirName) - if err := os.Mkdir(realManifestDir, 0700); err != nil { - return nil, errors.Wrapf(err, "couldn't create a realManifestDir for the upgrade") - } - - upgradedManifestDir := filepath.Join(kubernetesDir, "upgraded-manifests") - if err := os.Mkdir(upgradedManifestDir, 0700); err != nil { - return nil, errors.Wrapf(err, "couldn't create a upgradedManifestDir for the upgrade") - } - - backupManifestDir := filepath.Join(kubernetesDir, "backup-manifests") - if err := os.Mkdir(backupManifestDir, 0700); err != nil { - return nil, errors.Wrap(err, "couldn't create a backupManifestDir for the upgrade") - } - - backupEtcdDir := filepath.Join(kubernetesDir, "kubeadm-backup-etcd") - if err := os.Mkdir(backupEtcdDir, 0700); err != nil { - return nil, err - } - - return &fakeStaticPodPathManager{ - kubernetesDir: kubernetesDir, - realManifestDir: realManifestDir, - tempManifestDir: upgradedManifestDir, - backupManifestDir: backupManifestDir, - backupEtcdDir: backupEtcdDir, - MoveFileFunc: moveFileFunc, - }, nil -} - -func (spm *fakeStaticPodPathManager) MoveFile(oldPath, newPath string) error { - return spm.MoveFileFunc(oldPath, newPath) -} - -func (spm *fakeStaticPodPathManager) KubernetesDir() string { - return spm.kubernetesDir -} - -func (spm *fakeStaticPodPathManager) PatchesDir() string { - return spm.patchesDir -} - -func (spm *fakeStaticPodPathManager) RealManifestPath(component string) string { - return constants.GetStaticPodFilepath(component, spm.realManifestDir) -} -func (spm *fakeStaticPodPathManager) RealManifestDir() string { - return spm.realManifestDir -} - -func (spm *fakeStaticPodPathManager) TempManifestPath(component string) string { - return constants.GetStaticPodFilepath(component, spm.tempManifestDir) -} -func (spm *fakeStaticPodPathManager) TempManifestDir() string { - return spm.tempManifestDir -} - -func (spm *fakeStaticPodPathManager) BackupManifestPath(component string) string { - return constants.GetStaticPodFilepath(component, spm.backupManifestDir) -} -func (spm *fakeStaticPodPathManager) BackupManifestDir() string { - return spm.backupManifestDir -} - -func (spm *fakeStaticPodPathManager) BackupEtcdDir() string { - return spm.backupEtcdDir -} - -func (spm *fakeStaticPodPathManager) CleanupDirs() error { - if err := os.RemoveAll(spm.TempManifestDir()); err != nil { - return err - } - if err := os.RemoveAll(spm.BackupManifestDir()); err != nil { - return err - } - return os.RemoveAll(spm.BackupEtcdDir()) -} - -type fakeTLSEtcdClient struct{ TLS bool } - -func (c fakeTLSEtcdClient) WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) { - return true, nil -} - -func (c fakeTLSEtcdClient) CheckClusterHealth() error { - return nil -} - -func (c fakeTLSEtcdClient) Sync() error { return nil } - -func (c fakeTLSEtcdClient) ListMembers() ([]etcdutil.Member, error) { - return []etcdutil.Member{}, nil -} - -func (c fakeTLSEtcdClient) AddMember(name string, peerAddrs string) ([]etcdutil.Member, error) { - return []etcdutil.Member{}, nil -} - -func (c fakeTLSEtcdClient) GetMemberID(peerURL string) (uint64, error) { - return 0, nil -} - -func (c fakeTLSEtcdClient) RemoveMember(id uint64) ([]etcdutil.Member, error) { - return []etcdutil.Member{}, nil -} - -type fakePodManifestEtcdClient struct{ ManifestDir, CertificatesDir string } - -func (c fakePodManifestEtcdClient) WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) { - return true, nil -} - -func (c fakePodManifestEtcdClient) CheckClusterHealth() error { - // Make sure the certificates generated from the upgrade are readable from disk - tlsInfo := transport.TLSInfo{ - CertFile: filepath.Join(c.CertificatesDir, constants.EtcdCACertName), - KeyFile: filepath.Join(c.CertificatesDir, constants.EtcdHealthcheckClientCertName), - TrustedCAFile: filepath.Join(c.CertificatesDir, constants.EtcdHealthcheckClientKeyName), - } - _, err := tlsInfo.ClientConfig() - return err -} - -func (c fakePodManifestEtcdClient) Sync() error { return nil } - -func (c fakePodManifestEtcdClient) ListMembers() ([]etcdutil.Member, error) { - return []etcdutil.Member{}, nil -} - -func (c fakePodManifestEtcdClient) AddMember(name string, peerAddrs string) ([]etcdutil.Member, error) { - return []etcdutil.Member{}, nil -} - -func (c fakePodManifestEtcdClient) GetMemberID(peerURL string) (uint64, error) { - return 0, nil -} - -func (c fakePodManifestEtcdClient) RemoveMember(id uint64) ([]etcdutil.Member, error) { - return []etcdutil.Member{}, nil -} - -func TestStaticPodControlPlane(t *testing.T) { - tests := []struct { - description string - waitErrsToReturn map[string]error - moveFileFunc func(string, string) error - skipKubeConfig string - expectedErr bool - manifestShouldChange bool - }{ - { - description: "error-free case should succeed", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) - }, - expectedErr: false, - manifestShouldChange: true, - }, - { - description: "any wait error should result in a rollback and an abort 1", - waitErrsToReturn: map[string]error{ - waitForHashes: errors.New("boo! failed"), - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) - }, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any wait error should result in a rollback and an abort 2", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: errors.New("boo! failed"), - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) - }, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any wait error should result in a rollback and an abort 3", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: errors.New("boo! failed"), - }, - moveFileFunc: func(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) - }, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any path-moving error should result in a rollback and an abort 1", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - // fail for kube-apiserver move - if strings.Contains(newPath, "kube-apiserver") { - return errors.New("moving the kube-apiserver file failed") - } - return os.Rename(oldPath, newPath) - }, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any path-moving error should result in a rollback and an abort 2", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - // fail for kube-controller-manager move - if strings.Contains(newPath, "kube-controller-manager") { - return errors.New("moving the kube-apiserver file failed") - } - return os.Rename(oldPath, newPath) - }, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any path-moving error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy)", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - // fail for kube-scheduler move - if strings.Contains(newPath, "kube-scheduler") { - return errors.New("moving the kube-apiserver file failed") - } - return os.Rename(oldPath, newPath) - }, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any cert renew error should result in a rollback and an abort; even though this is the last component (kube-apiserver and kube-controller-manager healthy)", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) - }, - skipKubeConfig: constants.SchedulerKubeConfigFileName, - expectedErr: true, - manifestShouldChange: false, - }, - { - description: "any cert renew error should result in a rollback and an abort; even though this is admin.conf (kube-apiserver and kube-controller-manager and kube-scheduler healthy)", - waitErrsToReturn: map[string]error{ - waitForHashes: nil, - waitForHashChange: nil, - waitForPodsWithLabel: nil, - }, - moveFileFunc: func(oldPath, newPath string) error { - return os.Rename(oldPath, newPath) - }, - skipKubeConfig: constants.AdminKubeConfigFileName, - expectedErr: true, - manifestShouldChange: false, - }, - } - - for i := range tests { - rt := tests[i] - t.Run(rt.description, func(t *testing.T) { - t.Parallel() - waiter := NewFakeStaticPodWaiter(rt.waitErrsToReturn) - pathMgr, err := NewFakeStaticPodPathManager(rt.moveFileFunc) - if err != nil { - t.Fatalf("couldn't run NewFakeStaticPodPathManager: %v", err) - } - defer os.RemoveAll(pathMgr.(*fakeStaticPodPathManager).KubernetesDir()) - tmpKubernetesDir := pathMgr.(*fakeStaticPodPathManager).KubernetesDir() - - tempCertsDir, err := ioutil.TempDir("", "kubeadm-certs") - if err != nil { - t.Fatalf("couldn't create temporary certificates directory: %v", err) - } - defer os.RemoveAll(tempCertsDir) - tmpEtcdDataDir, err := ioutil.TempDir("", "kubeadm-etcd-data") - if err != nil { - t.Fatalf("couldn't create temporary etcd data directory: %v", err) - } - defer os.RemoveAll(tmpEtcdDataDir) - - oldcfg, err := getConfig(constants.MinimumControlPlaneVersion.String(), tempCertsDir, tmpEtcdDataDir) - if err != nil { - t.Fatalf("couldn't create config: %v", err) - } - - tree, err := certsphase.GetCertsWithoutEtcd().AsMap().CertTree() - if err != nil { - t.Fatalf("couldn't get cert tree: %v", err) - } - - if err := tree.CreateTree(oldcfg); err != nil { - t.Fatalf("couldn't get create cert tree: %v", err) - } - - for _, kubeConfig := range []string{ - constants.AdminKubeConfigFileName, - constants.SchedulerKubeConfigFileName, - constants.ControllerManagerKubeConfigFileName, - } { - if rt.skipKubeConfig == kubeConfig { - continue - } - if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpKubernetesDir, oldcfg); err != nil { - t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err) - } - } - - // Initialize the directory with v1.7 manifests; should then be upgraded to v1.8 using the method - err = controlplanephase.CreateInitStaticPodManifestFiles(pathMgr.RealManifestDir(), pathMgr.PatchesDir(), oldcfg) - if err != nil { - t.Fatalf("couldn't run CreateInitStaticPodManifestFiles: %v", err) - } - err = etcdphase.CreateLocalEtcdStaticPodManifestFile(pathMgr.RealManifestDir(), pathMgr.PatchesDir(), oldcfg.NodeRegistration.Name, &oldcfg.ClusterConfiguration, &oldcfg.LocalAPIEndpoint) - if err != nil { - t.Fatalf("couldn't run CreateLocalEtcdStaticPodManifestFile: %v", err) - } - // Get a hash of the v1.7 API server manifest to compare later (was the file re-written) - oldHash, err := getAPIServerHash(pathMgr.RealManifestDir()) - if err != nil { - t.Fatalf("couldn't read temp file: %v", err) - } - - newcfg, err := getConfig(constants.CurrentKubernetesVersion.String(), tempCertsDir, tmpEtcdDataDir) - if err != nil { - t.Fatalf("couldn't create config: %v", err) - } - - // create the kubeadm etcd certs - caCert, caKey, err := certsphase.KubeadmCertEtcdCA().CreateAsCA(newcfg) - if err != nil { - t.Fatalf("couldn't create new CA certificate: %v", err) - } - for _, cert := range []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdServer(), - certsphase.KubeadmCertEtcdPeer(), - certsphase.KubeadmCertEtcdHealthcheck(), - certsphase.KubeadmCertEtcdAPIClient(), - } { - if err := cert.CreateFromCA(newcfg, caCert, caKey); err != nil { - t.Fatalf("couldn't create certificate %s: %v", cert.Name, err) - } - } - - actualErr := StaticPodControlPlane( - nil, - waiter, - pathMgr, - newcfg, - true, - true, - fakeTLSEtcdClient{ - TLS: false, - }, - fakePodManifestEtcdClient{ - ManifestDir: pathMgr.RealManifestDir(), - CertificatesDir: newcfg.CertificatesDir, - }, - ) - if (actualErr != nil) != rt.expectedErr { - t.Errorf( - "failed UpgradeStaticPodControlPlane\n%s\n\texpected error: %t\n\tgot: %t\n\tactual error: %v", - rt.description, - rt.expectedErr, - (actualErr != nil), - actualErr, - ) - } - - newHash, err := getAPIServerHash(pathMgr.RealManifestDir()) - if err != nil { - t.Fatalf("couldn't read temp file: %v", err) - } - - if (oldHash != newHash) != rt.manifestShouldChange { - t.Errorf( - "failed StaticPodControlPlane\n%s\n\texpected manifest change: %t\n\tgot: %t\n\tnewHash: %v", - rt.description, - rt.manifestShouldChange, - (oldHash != newHash), - newHash, - ) - } - }) - } -} - -func getAPIServerHash(dir string) (string, error) { - manifestPath := constants.GetStaticPodFilepath(constants.KubeAPIServer, dir) - - fileBytes, err := ioutil.ReadFile(manifestPath) - if err != nil { - return "", err - } - - return fmt.Sprintf("%x", sha256.Sum256(fileBytes)), nil -} - -func getConfig(version, certsDir, etcdDataDir string) (*kubeadmapi.InitConfiguration, error) { - configBytes := []byte(fmt.Sprintf(testConfiguration, certsDir, etcdDataDir, version)) - - // Unmarshal the config - return configutil.BytesToInitConfiguration(configBytes) -} - -func getTempDir(t *testing.T, name string) (string, func()) { - dir, err := ioutil.TempDir(os.TempDir(), name) - if err != nil { - t.Fatalf("couldn't make temporary directory: %v", err) - } - - return dir, func() { - os.RemoveAll(dir) - } -} - -func TestCleanupDirs(t *testing.T) { - tests := []struct { - name string - keepManifest, keepEtcd bool - }{ - { - name: "save manifest backup", - keepManifest: true, - }, - { - name: "save both etcd and manifest", - keepManifest: true, - keepEtcd: true, - }, - { - name: "save nothing", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - realKubernetesDir, cleanup := getTempDir(t, "realKubernetesDir") - defer cleanup() - - tempManifestDir, cleanup := getTempDir(t, "tempManifestDir") - defer cleanup() - - backupManifestDir, cleanup := getTempDir(t, "backupManifestDir") - defer cleanup() - - backupEtcdDir, cleanup := getTempDir(t, "backupEtcdDir") - defer cleanup() - - mgr := NewKubeStaticPodPathManager(realKubernetesDir, "", tempManifestDir, backupManifestDir, backupEtcdDir, test.keepManifest, test.keepEtcd) - err := mgr.CleanupDirs() - if err != nil { - t.Errorf("unexpected error cleaning up: %v", err) - } - - if _, err := os.Stat(tempManifestDir); !os.IsNotExist(err) { - t.Errorf("%q should not have existed", tempManifestDir) - } - _, err = os.Stat(backupManifestDir) - if test.keepManifest { - if err != nil { - t.Errorf("unexpected error getting backup manifest dir") - } - } else { - if !os.IsNotExist(err) { - t.Error("expected backup manifest to not exist") - } - } - - _, err = os.Stat(backupEtcdDir) - if test.keepEtcd { - if err != nil { - t.Errorf("unexpected error getting backup etcd dir") - } - } else { - if !os.IsNotExist(err) { - t.Error("expected backup etcd dir to not exist") - } - } - }) - } -} - -func TestRenewCertsByComponent(t *testing.T) { - caCert, caKey := certstestutil.SetupCertificateAuthority(t) - - tests := []struct { - name string - component string - externalCA bool - externalFrontProxyCA bool - skipCreateEtcdCA bool - shouldErrorOnRenew bool - certsShouldExist []*certsphase.KubeadmCert - certsShouldBeRenewed []*certsphase.KubeadmCert // NB. If empty, it will assume certsShouldBeRenewed == certsShouldExist - kubeConfigShouldExist []string - }{ - { - name: "all CA exist, all certs should be rotated for etcd", - component: constants.Etcd, - certsShouldExist: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdServer(), - certsphase.KubeadmCertEtcdPeer(), - certsphase.KubeadmCertEtcdHealthcheck(), - }, - }, - { - name: "all CA exist, all certs should be rotated for apiserver", - component: constants.KubeAPIServer, - certsShouldExist: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertAPIServer(), - certsphase.KubeadmCertKubeletClient(), - certsphase.KubeadmCertFrontProxyClient(), - }, - }, - { - name: "external CA, renew only certificates not signed by CA for apiserver", - component: constants.KubeAPIServer, - certsShouldExist: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertFrontProxyClient(), - certsphase.KubeadmCertAPIServer(), - certsphase.KubeadmCertKubeletClient(), - }, - certsShouldBeRenewed: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertFrontProxyClient(), - }, - externalCA: true, - }, - { - name: "external front-proxy-CA, renew only certificates not signed by front-proxy-CA for apiserver", - component: constants.KubeAPIServer, - certsShouldExist: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertFrontProxyClient(), - certsphase.KubeadmCertAPIServer(), - certsphase.KubeadmCertKubeletClient(), - }, - certsShouldBeRenewed: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdAPIClient(), - certsphase.KubeadmCertAPIServer(), - certsphase.KubeadmCertKubeletClient(), - }, - externalFrontProxyCA: true, - }, - { - name: "all CA exist, should be rotated for scheduler", - component: constants.KubeScheduler, - kubeConfigShouldExist: []string{ - constants.SchedulerKubeConfigFileName, - }, - }, - { - name: "all CA exist, should be rotated for controller manager", - component: constants.KubeControllerManager, - kubeConfigShouldExist: []string{ - constants.ControllerManagerKubeConfigFileName, - }, - }, - { - name: "missing a cert to renew", - component: constants.Etcd, - shouldErrorOnRenew: true, - certsShouldExist: []*certsphase.KubeadmCert{ - certsphase.KubeadmCertEtcdServer(), - certsphase.KubeadmCertEtcdPeer(), - }, - }, - { - name: "no CA, cannot continue", - component: constants.Etcd, - skipCreateEtcdCA: true, - shouldErrorOnRenew: true, - }, - } - - for i := range tests { - test := tests[i] - t.Run(test.name, func(t *testing.T) { - t.Parallel() - - // Setup up basic requities - tmpDir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpDir) - - cfg := testutil.GetDefaultInternalConfig(t) - cfg.CertificatesDir = tmpDir - - if err := pkiutil.WriteCertAndKey(tmpDir, constants.CACertAndKeyBaseName, caCert, caKey); err != nil { - t.Fatalf("couldn't write out CA: %v", err) - } - if test.externalCA { - os.Remove(filepath.Join(tmpDir, constants.CAKeyName)) - } - if err := pkiutil.WriteCertAndKey(tmpDir, constants.FrontProxyCACertAndKeyBaseName, caCert, caKey); err != nil { - t.Fatalf("couldn't write out front-proxy-CA: %v", err) - } - if test.externalFrontProxyCA { - os.Remove(filepath.Join(tmpDir, constants.FrontProxyCAKeyName)) - } - if !test.skipCreateEtcdCA { - if err := pkiutil.WriteCertAndKey(tmpDir, constants.EtcdCACertAndKeyBaseName, caCert, caKey); err != nil { - t.Fatalf("couldn't write out etcd-CA: %v", err) - } - } - - certMaps := make(map[string]big.Int) - - // Create expected certs and load to recorde the serial numbers - for _, kubeCert := range test.certsShouldExist { - if err := kubeCert.CreateFromCA(cfg, caCert, caKey); err != nil { - t.Fatalf("couldn't create certificate %q: %v", kubeCert.Name, err) - } - - cert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName) - if err != nil { - t.Fatalf("couldn't load certificate %q: %v", kubeCert.Name, err) - } - certMaps[kubeCert.Name] = *cert.SerialNumber - } - - // Create expected kubeconfigs - for _, kubeConfig := range test.kubeConfigShouldExist { - if err := kubeconfigphase.CreateKubeConfigFile(kubeConfig, tmpDir, cfg); err != nil { - t.Fatalf("couldn't create kubeconfig %q: %v", kubeConfig, err) - } - - newCerts, err := getEmbeddedCerts(tmpDir, kubeConfig) - if err != nil { - t.Fatalf("error reading embedded certs from %s: %v", kubeConfig, err) - } - certMaps[kubeConfig] = *newCerts[0].SerialNumber - } - - // Renew everything - rm, err := renewal.NewManager(&cfg.ClusterConfiguration, tmpDir) - if err != nil { - t.Fatalf("Failed to create the certificate renewal manager: %v", err) - } - - err = renewCertsByComponent(cfg, test.component, rm) - if test.shouldErrorOnRenew { - if err == nil { - t.Fatal("expected renewal error, got nothing") - } - // expected error, got error - return - } - if err != nil { - t.Fatalf("couldn't renew certificates: %v", err) - } - - // See if the certificate serial numbers change - for _, kubeCert := range test.certsShouldExist { - newCert, err := pkiutil.TryLoadCertFromDisk(tmpDir, kubeCert.BaseName) - if err != nil { - t.Errorf("couldn't load new certificate %q: %v", kubeCert.Name, err) - continue - } - oldSerial := certMaps[kubeCert.Name] - - shouldBeRenewed := true - if test.certsShouldBeRenewed != nil { - shouldBeRenewed = false - for _, x := range test.certsShouldBeRenewed { - if x.Name == kubeCert.Name { - shouldBeRenewed = true - } - } - } - - if shouldBeRenewed && oldSerial.Cmp(newCert.SerialNumber) == 0 { - t.Errorf("certifitate %v was not reissued when expected", kubeCert.Name) - } - if !shouldBeRenewed && oldSerial.Cmp(newCert.SerialNumber) != 0 { - t.Errorf("certifitate %v was reissued when not expected", kubeCert.Name) - } - } - - // See if the embedded certificate serial numbers change - for _, kubeConfig := range test.kubeConfigShouldExist { - newCerts, err := getEmbeddedCerts(tmpDir, kubeConfig) - if err != nil { - t.Fatalf("error reading embedded certs from %s: %v", kubeConfig, err) - } - oldSerial := certMaps[kubeConfig] - if oldSerial.Cmp(newCerts[0].SerialNumber) == 0 { - t.Errorf("certifitate %v was not reissued", kubeConfig) - } - } - }) - - } -} - -func getEmbeddedCerts(tmpDir, kubeConfig string) ([]*x509.Certificate, error) { - kubeconfigPath := filepath.Join(tmpDir, kubeConfig) - newConfig, err := clientcmd.LoadFromFile(kubeconfigPath) - if err != nil { - return nil, errors.Wrapf(err, "failed to load kubeconfig file %s", kubeconfigPath) - } - - authInfoName := newConfig.Contexts[newConfig.CurrentContext].AuthInfo - authInfo := newConfig.AuthInfos[authInfoName] - - return certutil.ParseCertsPEM(authInfo.ClientCertificateData) -} - -func TestGetPathManagerForUpgrade(t *testing.T) { - - externalEtcd := &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - Endpoints: []string{"10.100.0.1:2379", "10.100.0.2:2379", "10.100.0.3:2379"}, - }, - }, - }, - } - - stackedEtcd := &kubeadmapi.InitConfiguration{} - - tests := []struct { - name string - cfg *kubeadmapi.InitConfiguration - etcdUpgrade bool - shouldDeleteEtcd bool - }{ - { - name: "external etcd but no etcd upgrade", - cfg: externalEtcd, - etcdUpgrade: false, - shouldDeleteEtcd: true, - }, - { - name: "external etcd with etcd upgrade", - cfg: externalEtcd, - etcdUpgrade: true, - shouldDeleteEtcd: true, - }, - { - name: "stacked etcd but no etcd upgrade", - cfg: stackedEtcd, - etcdUpgrade: false, - shouldDeleteEtcd: true, - }, - { - name: "stacked etcd with etcd upgrade", - cfg: stackedEtcd, - etcdUpgrade: true, - shouldDeleteEtcd: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // Use a temporary directory - tmpdir, err := ioutil.TempDir("", "TestGetPathManagerForUpgrade") - if err != nil { - t.Fatalf("unexpected error making temporary directory: %v", err) - } - defer func() { - os.RemoveAll(tmpdir) - }() - - pathmgr, err := GetPathManagerForUpgrade(tmpdir, "", test.cfg, test.etcdUpgrade) - if err != nil { - t.Fatalf("unexpected error creating path manager: %v", err) - } - - if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) { - t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err) - } - - if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) { - t.Errorf("expected etcd dir %s to exist, but it did not (%v)", pathmgr.BackupEtcdDir(), err) - } - - if err := pathmgr.CleanupDirs(); err != nil { - t.Fatalf("unexpected error cleaning up directories: %v", err) - } - - if _, err := os.Stat(pathmgr.BackupManifestDir()); os.IsNotExist(err) { - t.Errorf("expected manifest dir %s to exist, but it did not (%v)", pathmgr.BackupManifestDir(), err) - } - - if test.shouldDeleteEtcd { - if _, err := os.Stat(pathmgr.BackupEtcdDir()); !os.IsNotExist(err) { - t.Errorf("expected etcd dir %s not to exist, but it did (%v)", pathmgr.BackupEtcdDir(), err) - } - } else { - if _, err := os.Stat(pathmgr.BackupEtcdDir()); os.IsNotExist(err) { - t.Errorf("expected etcd dir %s to exist, but it did not", pathmgr.BackupEtcdDir()) - } - } - }) - } - -} - -func TestGetEtcdImageTagFromStaticPod(t *testing.T) { - const expectedEtcdVersion = "3.1.12" - const etcdStaticPod = `apiVersion: v1 -kind: Pod -metadata: - labels: - component: etcd - tier: control-plane - name: etcd - namespace: kube-system -spec: - containers: - - name: etcd - image: k8s.gcr.io/etcd:` + expectedEtcdVersion - - manifestsDir, err := ioutil.TempDir("", "GetEtcdImageTagFromStaticPod-test-manifests") - if err != nil { - t.Fatalf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(manifestsDir) - - if err = ioutil.WriteFile(constants.GetStaticPodFilepath(constants.Etcd, manifestsDir), []byte(etcdStaticPod), 0644); err != nil { - t.Fatalf("Unable to create test static pod manifest: %v", err) - } - - got, err := GetEtcdImageTagFromStaticPod(manifestsDir) - if err != nil { - t.Errorf("unexpected error: %v", err) - } else if got != expectedEtcdVersion { - t.Errorf("unexpected result:\n\tgot: %q\n\texpected: %q", got, expectedEtcdVersion) - } -} diff --git a/cmd/kubeadm/app/phases/upgrade/versiongetter.go b/cmd/kubeadm/app/phases/upgrade/versiongetter.go deleted file mode 100644 index cd38f05ce88cc..0000000000000 --- a/cmd/kubeadm/app/phases/upgrade/versiongetter.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "fmt" - "github.com/pkg/errors" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - versionutil "k8s.io/apimachinery/pkg/util/version" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/component-base/version" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -// VersionGetter defines an interface for fetching different versions. -// Easy to implement a fake variant of this interface for unit testing -type VersionGetter interface { - // ClusterVersion should return the version of the cluster i.e. the API Server version - ClusterVersion() (string, *versionutil.Version, error) - // KubeadmVersion should return the version of the kubeadm CLI - KubeadmVersion() (string, *versionutil.Version, error) - // VersionFromCILabel should resolve CI labels like `latest`, `stable`, `stable-1.8`, etc. to real versions - VersionFromCILabel(string, string) (string, *versionutil.Version, error) - // KubeletVersions should return a map with a version and a number that describes how many kubelets there are for that version - KubeletVersions() (map[string]uint16, error) -} - -// KubeVersionGetter handles the version-fetching mechanism from external sources -type KubeVersionGetter struct { - client clientset.Interface -} - -// NewKubeVersionGetter returns a new instance of KubeVersionGetter -func NewKubeVersionGetter(client clientset.Interface) VersionGetter { - return &KubeVersionGetter{ - client: client, - } -} - -// ClusterVersion gets API server version -func (g *KubeVersionGetter) ClusterVersion() (string, *versionutil.Version, error) { - clusterVersionInfo, err := g.client.Discovery().ServerVersion() - if err != nil { - return "", nil, errors.Wrap(err, "Couldn't fetch cluster version from the API Server") - } - - clusterVersion, err := versionutil.ParseSemantic(clusterVersionInfo.String()) - if err != nil { - return "", nil, errors.Wrap(err, "Couldn't parse cluster version") - } - return clusterVersionInfo.String(), clusterVersion, nil -} - -// KubeadmVersion gets kubeadm version -func (g *KubeVersionGetter) KubeadmVersion() (string, *versionutil.Version, error) { - kubeadmVersionInfo := version.Get() - - kubeadmVersion, err := versionutil.ParseSemantic(kubeadmVersionInfo.String()) - if err != nil { - return "", nil, errors.Wrap(err, "Couldn't parse kubeadm version") - } - return kubeadmVersionInfo.String(), kubeadmVersion, nil -} - -// VersionFromCILabel resolves a version label like "latest" or "stable" to an actual version using the public Kubernetes CI uploads -func (g *KubeVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) { - versionStr, err := kubeadmutil.KubernetesReleaseVersion(ciVersionLabel) - if err != nil { - return "", nil, errors.Wrapf(err, "Couldn't fetch latest %s from the internet", description) - } - - ver, err := versionutil.ParseSemantic(versionStr) - if err != nil { - return "", nil, errors.Wrapf(err, "Couldn't parse latest %s", description) - } - return versionStr, ver, nil -} - -// KubeletVersions gets the versions of the kubelets in the cluster -func (g *KubeVersionGetter) KubeletVersions() (map[string]uint16, error) { - nodes, err := g.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, errors.New("couldn't list all nodes in cluster") - } - return computeKubeletVersions(nodes.Items), nil -} - -// computeKubeletVersions returns a string-int map that describes how many nodes are of a specific version -func computeKubeletVersions(nodes []v1.Node) map[string]uint16 { - kubeletVersions := map[string]uint16{} - for _, node := range nodes { - kver := node.Status.NodeInfo.KubeletVersion - if _, found := kubeletVersions[kver]; !found { - kubeletVersions[kver] = 1 - continue - } - kubeletVersions[kver]++ - } - return kubeletVersions -} - -// OfflineVersionGetter will use the version provided or -type OfflineVersionGetter struct { - VersionGetter - version string -} - -// NewOfflineVersionGetter wraps a VersionGetter and skips online communication if default information is supplied. -// Version can be "" and the behavior will be identical to the versionGetter passed in. -func NewOfflineVersionGetter(versionGetter VersionGetter, version string) VersionGetter { - return &OfflineVersionGetter{ - VersionGetter: versionGetter, - version: version, - } -} - -// VersionFromCILabel will return the version that was passed into the struct -func (o *OfflineVersionGetter) VersionFromCILabel(ciVersionLabel, description string) (string, *versionutil.Version, error) { - if o.version == "" { - versionStr, version, err := o.VersionGetter.VersionFromCILabel(ciVersionLabel, description) - if err == nil { - fmt.Printf("[upgrade/versions] Latest %s: %s\n", description, versionStr) - } - return versionStr, version, err - } - ver, err := versionutil.ParseSemantic(o.version) - if err != nil { - return "", nil, errors.Wrapf(err, "Couldn't parse version %s", description) - } - return o.version, ver, nil -} diff --git a/cmd/kubeadm/app/phases/uploadconfig/BUILD b/cmd/kubeadm/app/phases/uploadconfig/BUILD deleted file mode 100644 index da1156c9c7851..0000000000000 --- a/cmd/kubeadm/app/phases/uploadconfig/BUILD +++ /dev/null @@ -1,55 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["uploadconfig.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["uploadconfig_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go deleted file mode 100644 index 000ce9da3641f..0000000000000 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig.go +++ /dev/null @@ -1,185 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package uploadconfig - -import ( - "fmt" - - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -const ( - // NodesKubeadmConfigClusterRoleName sets the name for the ClusterRole that allows - // the bootstrap tokens to access the kubeadm-config ConfigMap during the node bootstrap/discovery - // or during upgrade nodes - NodesKubeadmConfigClusterRoleName = "kubeadm:nodes-kubeadm-config" -) - -// ResetClusterStatusForNode removes the APIEndpoint of a given control-plane node -// from the ClusterStatus and updates the kubeadm ConfigMap -func ResetClusterStatusForNode(nodeName string, client clientset.Interface) error { - fmt.Printf("[reset] Removing info for node %q from the ConfigMap %q in the %q Namespace\n", - nodeName, kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - - return apiclient.MutateConfigMap(client, metav1.ObjectMeta{ - Name: kubeadmconstants.KubeadmConfigConfigMap, - Namespace: metav1.NamespaceSystem, - }, func(cm *v1.ConfigMap) error { - return mutateClusterStatus(cm, func(cs *kubeadmapi.ClusterStatus) error { - // Handle a nil APIEndpoints map. Should only happen if someone manually - // interacted with the ConfigMap. - if cs.APIEndpoints == nil { - return errors.Errorf("APIEndpoints from ConfigMap %q in the %q Namespace is nil", - kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - } - klog.V(2).Infof("Removing APIEndpoint for Node %q", nodeName) - delete(cs.APIEndpoints, nodeName) - return nil - }) - }) -} - -// UploadConfiguration saves the InitConfiguration used for later reference (when upgrading for instance) -func UploadConfiguration(cfg *kubeadmapi.InitConfiguration, client clientset.Interface) error { - fmt.Printf("[upload-config] Storing the configuration used in ConfigMap %q in the %q Namespace\n", kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - - // Prepare the ClusterConfiguration for upload - // The components store their config in their own ConfigMaps, then reset the .ComponentConfig struct; - // We don't want to mutate the cfg itself, so create a copy of it using .DeepCopy of it first - clusterConfigurationToUpload := cfg.ClusterConfiguration.DeepCopy() - clusterConfigurationToUpload.ComponentConfigs = kubeadmapi.ComponentConfigMap{} - - // Marshal the ClusterConfiguration into YAML - clusterConfigurationYaml, err := configutil.MarshalKubeadmConfigObject(clusterConfigurationToUpload) - if err != nil { - return err - } - - // Prepare the ClusterStatus for upload - clusterStatus := &kubeadmapi.ClusterStatus{ - APIEndpoints: map[string]kubeadmapi.APIEndpoint{ - cfg.NodeRegistration.Name: cfg.LocalAPIEndpoint, - }, - } - // Marshal the ClusterStatus into YAML - clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus) - if err != nil { - return err - } - - err = apiclient.CreateOrMutateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: kubeadmconstants.KubeadmConfigConfigMap, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - kubeadmconstants.ClusterConfigurationConfigMapKey: string(clusterConfigurationYaml), - kubeadmconstants.ClusterStatusConfigMapKey: string(clusterStatusYaml), - }, - }, func(cm *v1.ConfigMap) error { - // Upgrade will call to UploadConfiguration with a modified KubernetesVersion reflecting the new - // Kubernetes version. In that case, the mutation path will take place. - cm.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] = string(clusterConfigurationYaml) - // Mutate the ClusterStatus now - return mutateClusterStatus(cm, func(cs *kubeadmapi.ClusterStatus) error { - // Handle a nil APIEndpoints map. Should only happen if someone manually - // interacted with the ConfigMap. - if cs.APIEndpoints == nil { - return errors.Errorf("APIEndpoints from ConfigMap %q in the %q Namespace is nil", - kubeadmconstants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - } - cs.APIEndpoints[cfg.NodeRegistration.Name] = cfg.LocalAPIEndpoint - return nil - }) - }) - if err != nil { - return err - } - - // Ensure that the NodesKubeadmConfigClusterRoleName exists - err = apiclient.CreateOrUpdateRole(client, &rbac.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: NodesKubeadmConfigClusterRoleName, - Namespace: metav1.NamespaceSystem, - }, - Rules: []rbac.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{kubeadmconstants.KubeadmConfigConfigMap}, - }, - }, - }) - if err != nil { - return err - } - - // Binds the NodesKubeadmConfigClusterRoleName to all the bootstrap tokens - // that are members of the system:bootstrappers:kubeadm:default-node-token group - // and to all nodes - return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: NodesKubeadmConfigClusterRoleName, - Namespace: metav1.NamespaceSystem, - }, - RoleRef: rbac.RoleRef{ - APIGroup: rbac.GroupName, - Kind: "Role", - Name: NodesKubeadmConfigClusterRoleName, - }, - Subjects: []rbac.Subject{ - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodeBootstrapTokenAuthGroup, - }, - { - Kind: rbac.GroupKind, - Name: kubeadmconstants.NodesGroup, - }, - }, - }) -} - -func mutateClusterStatus(cm *v1.ConfigMap, mutator func(*kubeadmapi.ClusterStatus) error) error { - // Obtain the existing ClusterStatus object - clusterStatus, err := configutil.UnmarshalClusterStatus(cm.Data) - if err != nil { - return err - } - // Mutate the ClusterStatus - if err := mutator(clusterStatus); err != nil { - return err - } - // Marshal the ClusterStatus back into YAML - clusterStatusYaml, err := configutil.MarshalKubeadmConfigObject(clusterStatus) - if err != nil { - return err - } - // Write the marshaled mutated cluster status back to the ConfigMap - cm.Data[kubeadmconstants.ClusterStatusConfigMapKey] = string(clusterStatusYaml) - return nil -} diff --git a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go b/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go deleted file mode 100644 index f54c0dc9b73f4..0000000000000 --- a/cmd/kubeadm/app/phases/uploadconfig/uploadconfig_test.go +++ /dev/null @@ -1,172 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package uploadconfig - -import ( - "context" - "reflect" - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientsetfake "k8s.io/client-go/kubernetes/fake" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -func TestUploadConfiguration(t *testing.T) { - tests := []struct { - name string - updateExisting bool - verifyResult bool - }{ - { - name: "basic validation with correct key", - verifyResult: true, - }, - { - name: "update existing should report no error", - updateExisting: true, - verifyResult: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t2 *testing.T) { - initialcfg := &kubeadmapiv1beta2.InitConfiguration{ - LocalAPIEndpoint: kubeadmapiv1beta2.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - }, - BootstrapTokens: []kubeadmapiv1beta2.BootstrapToken{ - { - Token: &kubeadmapiv1beta2.BootstrapTokenString{ - ID: "abcdef", - Secret: "abcdef0123456789", - }, - }, - }, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - Name: "node-foo", - CRISocket: "/var/run/custom-cri.sock", - }, - } - clustercfg := &kubeadmapiv1beta2.ClusterConfiguration{ - KubernetesVersion: kubeadmconstants.MinimumControlPlaneVersion.WithPatch(10).String(), - } - cfg, err := configutil.DefaultedInitConfiguration(initialcfg, clustercfg) - - if err != nil { - t2.Fatalf("UploadConfiguration() error = %v", err) - } - - cfg.ComponentConfigs = kubeadmapi.ComponentConfigMap{} - - status := &kubeadmapi.ClusterStatus{ - APIEndpoints: map[string]kubeadmapi.APIEndpoint{ - "node-foo": cfg.LocalAPIEndpoint, - }, - } - - client := clientsetfake.NewSimpleClientset() - // For idempotent test, we check the result of the second call. - if err := UploadConfiguration(cfg, client); err != nil { - t2.Fatalf("UploadConfiguration() error = %v", err) - } - if tt.updateExisting { - if err := UploadConfiguration(cfg, client); err != nil { - t2.Fatalf("UploadConfiguration() error = %v", err) - } - } - if tt.verifyResult { - controlPlaneCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), kubeadmconstants.KubeadmConfigConfigMap, metav1.GetOptions{}) - if err != nil { - t2.Fatalf("Fail to query ConfigMap error = %v", err) - } - configData := controlPlaneCfg.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] - if configData == "" { - t2.Fatal("Fail to find ClusterConfigurationConfigMapKey key") - } - - decodedCfg := &kubeadmapi.ClusterConfiguration{} - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(configData), decodedCfg); err != nil { - t2.Fatalf("unable to decode config from bytes: %v", err) - } - - if len(decodedCfg.ComponentConfigs) != 0 { - t2.Errorf("unexpected component configs in decodedCfg: %d", len(decodedCfg.ComponentConfigs)) - } - - // Force initialize with an empty map so that reflect.DeepEqual works - decodedCfg.ComponentConfigs = kubeadmapi.ComponentConfigMap{} - - if !reflect.DeepEqual(decodedCfg, &cfg.ClusterConfiguration) { - t2.Errorf("the initial and decoded ClusterConfiguration didn't match:\n%t\n===\n%t", decodedCfg.ComponentConfigs == nil, cfg.ComponentConfigs == nil) - } - - statusData := controlPlaneCfg.Data[kubeadmconstants.ClusterStatusConfigMapKey] - if statusData == "" { - t2.Fatal("failed to find ClusterStatusConfigMapKey key") - } - - decodedStatus := &kubeadmapi.ClusterStatus{} - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(statusData), decodedStatus); err != nil { - t2.Fatalf("unable to decode status from bytes: %v", err) - } - - if !reflect.DeepEqual(decodedStatus, status) { - t2.Error("the initial and decoded ClusterStatus didn't match") - } - } - }) - } -} - -func TestMutateClusterStatus(t *testing.T) { - cm := &v1.ConfigMap{ - Data: map[string]string{ - kubeadmconstants.ClusterStatusConfigMapKey: "", - }, - } - - endpoints := map[string]kubeadmapi.APIEndpoint{ - "some-node": { - AdvertiseAddress: "127.0.0.1", - BindPort: 6443, - }, - } - - err := mutateClusterStatus(cm, func(cs *kubeadmapi.ClusterStatus) error { - cs.APIEndpoints = endpoints - return nil - }) - if err != nil { - t.Fatalf("could not mutate cluster status: %v", err) - } - - // Try to unmarshal the cluster status back and compare with the original mutated structure - cs, err := configutil.UnmarshalClusterStatus(cm.Data) - if err != nil { - t.Fatalf("could not unmarshal cluster status: %v", err) - } - - if !reflect.DeepEqual(cs.APIEndpoints, endpoints) { - t.Fatalf("mutation of cluster status failed: %v", err) - } -} diff --git a/cmd/kubeadm/app/preflight/BUILD b/cmd/kubeadm/app/preflight/BUILD deleted file mode 100644 index 49ec4e30f89e3..0000000000000 --- a/cmd/kubeadm/app/preflight/BUILD +++ /dev/null @@ -1,77 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "checks.go", - "checks_darwin.go", - "checks_linux.go", - "checks_unix.go", - "checks_windows.go", - "utils.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/preflight", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/images:go_default_library", - "//cmd/kubeadm/app/util/initsystem:go_default_library", - "//cmd/kubeadm/app/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//vendor/github.com/PuerkitoBio/purell:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/system-validators/validators:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:android": [ - "//cmd/kubeadm/app/util:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//cmd/kubeadm/app/util:go_default_library", - ], - "//conditions:default": [], - }), -) - -go_test( - name = "go_default_test", - srcs = [ - "checks_test.go", - "utils_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go deleted file mode 100644 index cd6561897925e..0000000000000 --- a/cmd/kubeadm/app/preflight/checks.go +++ /dev/null @@ -1,1110 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "bufio" - "bytes" - "crypto/tls" - "crypto/x509" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "net/url" - "os" - "path/filepath" - "runtime" - "strings" - "time" - - "github.com/PuerkitoBio/purell" - "github.com/pkg/errors" - netutil "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/sets" - versionutil "k8s.io/apimachinery/pkg/util/version" - kubeadmversion "k8s.io/component-base/version" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/images" - "k8s.io/kubernetes/cmd/kubeadm/app/util/initsystem" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" - system "k8s.io/system-validators/validators" - utilsexec "k8s.io/utils/exec" - utilsnet "k8s.io/utils/net" -) - -const ( - bridgenf = "/proc/sys/net/bridge/bridge-nf-call-iptables" - bridgenf6 = "/proc/sys/net/bridge/bridge-nf-call-ip6tables" - ipv4Forward = "/proc/sys/net/ipv4/ip_forward" - ipv6DefaultForwarding = "/proc/sys/net/ipv6/conf/default/forwarding" - externalEtcdRequestTimeout = time.Duration(10 * time.Second) - externalEtcdRequestRetries = 3 - externalEtcdRequestInterval = time.Duration(5 * time.Second) -) - -var ( - minExternalEtcdVersion = versionutil.MustParseSemantic(kubeadmconstants.MinExternalEtcdVersion) -) - -// Error defines struct for communicating error messages generated by preflight checks -type Error struct { - Msg string -} - -// Error implements the standard error interface -func (e *Error) Error() string { - return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`") -} - -// Preflight identifies this error as a preflight error -func (e *Error) Preflight() bool { - return true -} - -// Checker validates the state of the system to ensure kubeadm will be -// successful as often as possible. -type Checker interface { - Check() (warnings, errorList []error) - Name() string -} - -// ContainerRuntimeCheck verifies the container runtime. -type ContainerRuntimeCheck struct { - runtime utilruntime.ContainerRuntime -} - -// Name returns label for RuntimeCheck. -func (ContainerRuntimeCheck) Name() string { - return "CRI" -} - -// Check validates the container runtime -func (crc ContainerRuntimeCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating the container runtime") - if err := crc.runtime.IsRunning(); err != nil { - errorList = append(errorList, err) - } - return warnings, errorList -} - -// ServiceCheck verifies that the given service is enabled and active. If we do not -// detect a supported init system however, all checks are skipped and a warning is -// returned. -type ServiceCheck struct { - Service string - CheckIfActive bool - Label string -} - -// Name returns label for ServiceCheck. If not provided, will return based on the service parameter -func (sc ServiceCheck) Name() string { - if sc.Label != "" { - return sc.Label - } - return fmt.Sprintf("Service-%s", strings.Title(sc.Service)) -} - -// Check validates if the service is enabled and active. -func (sc ServiceCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating if the %q service is enabled and active", sc.Service) - initSystem, err := initsystem.GetInitSystem() - if err != nil { - return []error{err}, nil - } - - if !initSystem.ServiceExists(sc.Service) { - return []error{errors.Errorf("%s service does not exist", sc.Service)}, nil - } - - if !initSystem.ServiceIsEnabled(sc.Service) { - warnings = append(warnings, - errors.Errorf("%s service is not enabled, please run '%s'", - sc.Service, initSystem.EnableCommand(sc.Service))) - } - - if sc.CheckIfActive && !initSystem.ServiceIsActive(sc.Service) { - errorList = append(errorList, - errors.Errorf("%s service is not active, please run 'systemctl start %s.service'", - sc.Service, sc.Service)) - } - - return warnings, errorList -} - -// FirewalldCheck checks if firewalld is enabled or active. If it is, warn the user that there may be problems -// if no actions are taken. -type FirewalldCheck struct { - ports []int -} - -// Name returns label for FirewalldCheck. -func (FirewalldCheck) Name() string { - return "Firewalld" -} - -// Check validates if the firewall is enabled and active. -func (fc FirewalldCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating if the firewall is enabled and active") - initSystem, err := initsystem.GetInitSystem() - if err != nil { - return []error{err}, nil - } - - if !initSystem.ServiceExists("firewalld") { - return nil, nil - } - - if initSystem.ServiceIsActive("firewalld") { - err := errors.Errorf("firewalld is active, please ensure ports %v are open or your cluster may not function correctly", - fc.ports) - return []error{err}, nil - } - - return nil, nil -} - -// PortOpenCheck ensures the given port is available for use. -type PortOpenCheck struct { - port int - label string -} - -// Name returns name for PortOpenCheck. If not known, will return "PortXXXX" based on port number -func (poc PortOpenCheck) Name() string { - if poc.label != "" { - return poc.label - } - return fmt.Sprintf("Port-%d", poc.port) -} - -// Check validates if the particular port is available. -func (poc PortOpenCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating availability of port %d", poc.port) - - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", poc.port)) - if err != nil { - errorList = []error{errors.Errorf("Port %d is in use", poc.port)} - } - if ln != nil { - if err = ln.Close(); err != nil { - warnings = append(warnings, - errors.Errorf("when closing port %d, encountered %v", poc.port, err)) - } - } - - return warnings, errorList -} - -// IsPrivilegedUserCheck verifies user is privileged (linux - root, windows - Administrator) -type IsPrivilegedUserCheck struct{} - -// Name returns name for IsPrivilegedUserCheck -func (IsPrivilegedUserCheck) Name() string { - return "IsPrivilegedUser" -} - -// IsDockerSystemdCheck verifies if Docker is setup to use systemd as the cgroup driver. -type IsDockerSystemdCheck struct{} - -// Name returns name for IsDockerSystemdCheck -func (IsDockerSystemdCheck) Name() string { - return "IsDockerSystemdCheck" -} - -// DirAvailableCheck checks if the given directory either does not exist, or is empty. -type DirAvailableCheck struct { - Path string - Label string -} - -// Name returns label for individual DirAvailableChecks. If not known, will return based on path. -func (dac DirAvailableCheck) Name() string { - if dac.Label != "" { - return dac.Label - } - return fmt.Sprintf("DirAvailable-%s", strings.Replace(dac.Path, "/", "-", -1)) -} - -// Check validates if a directory does not exist or empty. -func (dac DirAvailableCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence and emptiness of directory %s", dac.Path) - - // If it doesn't exist we are good: - if _, err := os.Stat(dac.Path); os.IsNotExist(err) { - return nil, nil - } - - f, err := os.Open(dac.Path) - if err != nil { - return nil, []error{errors.Wrapf(err, "unable to check if %s is empty", dac.Path)} - } - defer f.Close() - - _, err = f.Readdirnames(1) - if err != io.EOF { - return nil, []error{errors.Errorf("%s is not empty", dac.Path)} - } - - return nil, nil -} - -// FileAvailableCheck checks that the given file does not already exist. -type FileAvailableCheck struct { - Path string - Label string -} - -// Name returns label for individual FileAvailableChecks. If not known, will return based on path. -func (fac FileAvailableCheck) Name() string { - if fac.Label != "" { - return fac.Label - } - return fmt.Sprintf("FileAvailable-%s", strings.Replace(fac.Path, "/", "-", -1)) -} - -// Check validates if the given file does not already exist. -func (fac FileAvailableCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence of file %s", fac.Path) - - if _, err := os.Stat(fac.Path); err == nil { - return nil, []error{errors.Errorf("%s already exists", fac.Path)} - } - return nil, nil -} - -// FileExistingCheck checks that the given file does not already exist. -type FileExistingCheck struct { - Path string - Label string -} - -// Name returns label for individual FileExistingChecks. If not known, will return based on path. -func (fac FileExistingCheck) Name() string { - if fac.Label != "" { - return fac.Label - } - return fmt.Sprintf("FileExisting-%s", strings.Replace(fac.Path, "/", "-", -1)) -} - -// Check validates if the given file already exists. -func (fac FileExistingCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence of file %s", fac.Path) - - if _, err := os.Stat(fac.Path); err != nil { - return nil, []error{errors.Errorf("%s doesn't exist", fac.Path)} - } - return nil, nil -} - -// FileContentCheck checks that the given file contains the string Content. -type FileContentCheck struct { - Path string - Content []byte - Label string -} - -// Name returns label for individual FileContentChecks. If not known, will return based on path. -func (fcc FileContentCheck) Name() string { - if fcc.Label != "" { - return fcc.Label - } - return fmt.Sprintf("FileContent-%s", strings.Replace(fcc.Path, "/", "-", -1)) -} - -// Check validates if the given file contains the given content. -func (fcc FileContentCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the contents of file %s", fcc.Path) - f, err := os.Open(fcc.Path) - if err != nil { - return nil, []error{errors.Errorf("%s does not exist", fcc.Path)} - } - - lr := io.LimitReader(f, int64(len(fcc.Content))) - defer f.Close() - - buf := &bytes.Buffer{} - _, err = io.Copy(buf, lr) - if err != nil { - return nil, []error{errors.Errorf("%s could not be read", fcc.Path)} - } - - if !bytes.Equal(buf.Bytes(), fcc.Content) { - return nil, []error{errors.Errorf("%s contents are not set to %s", fcc.Path, fcc.Content)} - } - return nil, []error{} - -} - -// InPathCheck checks if the given executable is present in $PATH -type InPathCheck struct { - executable string - mandatory bool - exec utilsexec.Interface - label string - suggestion string -} - -// Name returns label for individual InPathCheck. If not known, will return based on path. -func (ipc InPathCheck) Name() string { - if ipc.label != "" { - return ipc.label - } - return fmt.Sprintf("FileExisting-%s", strings.Replace(ipc.executable, "/", "-", -1)) -} - -// Check validates if the given executable is present in the path. -func (ipc InPathCheck) Check() (warnings, errs []error) { - klog.V(1).Infof("validating the presence of executable %s", ipc.executable) - _, err := ipc.exec.LookPath(ipc.executable) - if err != nil { - if ipc.mandatory { - // Return as an error: - return nil, []error{errors.Errorf("%s not found in system path", ipc.executable)} - } - // Return as a warning: - warningMessage := fmt.Sprintf("%s not found in system path", ipc.executable) - if ipc.suggestion != "" { - warningMessage += fmt.Sprintf("\nSuggestion: %s", ipc.suggestion) - } - return []error{errors.New(warningMessage)}, nil - } - return nil, nil -} - -// HostnameCheck checks if hostname match dns sub domain regex. -// If hostname doesn't match this regex, kubelet will not launch static pods like kube-apiserver/kube-controller-manager and so on. -type HostnameCheck struct { - nodeName string -} - -// Name will return Hostname as name for HostnameCheck -func (HostnameCheck) Name() string { - return "Hostname" -} - -// Check validates if hostname match dns sub domain regex. -func (hc HostnameCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("checking whether the given node name is reachable using net.LookupHost") - - addr, err := net.LookupHost(hc.nodeName) - if addr == nil { - warnings = append(warnings, errors.Errorf("hostname \"%s\" could not be reached", hc.nodeName)) - } - if err != nil { - warnings = append(warnings, errors.Wrapf(err, "hostname \"%s\"", hc.nodeName)) - } - return warnings, errorList -} - -// HTTPProxyCheck checks if https connection to specific host is going -// to be done directly or over proxy. If proxy detected, it will return warning. -type HTTPProxyCheck struct { - Proto string - Host string -} - -// Name returns HTTPProxy as name for HTTPProxyCheck -func (hst HTTPProxyCheck) Name() string { - return "HTTPProxy" -} - -// Check validates http connectivity type, direct or via proxy. -func (hst HTTPProxyCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating if the connectivity type is via proxy or direct") - u := &url.URL{Scheme: hst.Proto, Host: hst.Host} - if utilsnet.IsIPv6String(hst.Host) { - u.Host = net.JoinHostPort(hst.Host, "1234") - } - - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, []error{err} - } - - proxy, err := netutil.SetOldTransportDefaults(&http.Transport{}).Proxy(req) - if err != nil { - return nil, []error{err} - } - if proxy != nil { - return []error{errors.Errorf("Connection to %q uses proxy %q. If that is not intended, adjust your proxy settings", u, proxy)}, nil - } - return nil, nil -} - -// HTTPProxyCIDRCheck checks if https connection to specific subnet is going -// to be done directly or over proxy. If proxy detected, it will return warning. -// Similar to HTTPProxyCheck above, but operates with subnets and uses API -// machinery transport defaults to simulate kube-apiserver accessing cluster -// services and pods. -type HTTPProxyCIDRCheck struct { - Proto string - CIDR string -} - -// Name will return HTTPProxyCIDR as name for HTTPProxyCIDRCheck -func (HTTPProxyCIDRCheck) Name() string { - return "HTTPProxyCIDR" -} - -// Check validates http connectivity to first IP address in the CIDR. -// If it is not directly connected and goes via proxy it will produce warning. -func (subnet HTTPProxyCIDRCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating http connectivity to first IP address in the CIDR") - if len(subnet.CIDR) == 0 { - return nil, nil - } - - _, cidr, err := net.ParseCIDR(subnet.CIDR) - if err != nil { - return nil, []error{errors.Wrapf(err, "error parsing CIDR %q", subnet.CIDR)} - } - - testIP, err := utilsnet.GetIndexedIP(cidr, 1) - if err != nil { - return nil, []error{errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", cidr.String())} - } - - testIPstring := testIP.String() - if len(testIP) == net.IPv6len { - testIPstring = fmt.Sprintf("[%s]:1234", testIP) - } - url := fmt.Sprintf("%s://%s/", subnet.Proto, testIPstring) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, []error{err} - } - - // Utilize same transport defaults as it will be used by API server - proxy, err := netutil.SetOldTransportDefaults(&http.Transport{}).Proxy(req) - if err != nil { - return nil, []error{err} - } - if proxy != nil { - return []error{errors.Errorf("connection to %q uses proxy %q. This may lead to malfunctional cluster setup. Make sure that Pod and Services IP ranges specified correctly as exceptions in proxy configuration", subnet.CIDR, proxy)}, nil - } - return nil, nil -} - -// SystemVerificationCheck defines struct used for running the system verification node check in test/e2e_node/system -type SystemVerificationCheck struct { - IsDocker bool -} - -// Name will return SystemVerification as name for SystemVerificationCheck -func (SystemVerificationCheck) Name() string { - return "SystemVerification" -} - -// Check runs all individual checks -func (sysver SystemVerificationCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("running all checks") - // Create a buffered writer and choose a quite large value (1M) and suppose the output from the system verification test won't exceed the limit - // Run the system verification check, but write to out buffered writer instead of stdout - bufw := bufio.NewWriterSize(os.Stdout, 1*1024*1024) - reporter := &system.StreamReporter{WriteStream: bufw} - - var errs []error - var warns []error - // All the common validators we'd like to run: - var validators = []system.Validator{ - &system.KernelValidator{Reporter: reporter}} - - // run the docker validator only with docker runtime - if sysver.IsDocker { - validators = append(validators, &system.DockerValidator{Reporter: reporter}) - } - - if runtime.GOOS == "linux" { - //add linux validators - validators = append(validators, - &system.OSValidator{Reporter: reporter}, - &system.CgroupsValidator{Reporter: reporter}) - } - - // Run all validators - for _, v := range validators { - warn, err := v.Validate(system.DefaultSysSpec) - if err != nil { - errs = append(errs, err...) - } - if warn != nil { - warns = append(warns, warn...) - } - } - - if len(errs) != 0 { - // Only print the output from the system verification check if the check failed - fmt.Println("[preflight] The system verification failed. Printing the output from the verification:") - bufw.Flush() - return warns, errs - } - return warns, nil -} - -// KubernetesVersionCheck validates Kubernetes and kubeadm versions -type KubernetesVersionCheck struct { - KubeadmVersion string - KubernetesVersion string -} - -// Name will return KubernetesVersion as name for KubernetesVersionCheck -func (KubernetesVersionCheck) Name() string { - return "KubernetesVersion" -} - -// Check validates Kubernetes and kubeadm versions -func (kubever KubernetesVersionCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating Kubernetes and kubeadm version") - // Skip this check for "super-custom builds", where apimachinery/the overall codebase version is not set. - if strings.HasPrefix(kubever.KubeadmVersion, "v0.0.0") { - return nil, nil - } - - kadmVersion, err := versionutil.ParseSemantic(kubever.KubeadmVersion) - if err != nil { - return nil, []error{errors.Wrapf(err, "couldn't parse kubeadm version %q", kubever.KubeadmVersion)} - } - - k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) - if err != nil { - return nil, []error{errors.Wrapf(err, "couldn't parse Kubernetes version %q", kubever.KubernetesVersion)} - } - - // Checks if k8sVersion greater or equal than the first unsupported versions by current version of kubeadm, - // that is major.minor+1 (all patch and pre-releases versions included) - // NB. in semver patches number is a numeric, while prerelease is a string where numeric identifiers always have lower precedence than non-numeric identifiers. - // thus setting the value to x.y.0-0 we are defining the very first patch - prereleases within x.y minor release. - firstUnsupportedVersion := versionutil.MustParseSemantic(fmt.Sprintf("%d.%d.%s", kadmVersion.Major(), kadmVersion.Minor()+1, "0-0")) - if k8sVersion.AtLeast(firstUnsupportedVersion) { - return []error{errors.Errorf("Kubernetes version is greater than kubeadm version. Please consider to upgrade kubeadm. Kubernetes version: %s. Kubeadm version: %d.%d.x", k8sVersion, kadmVersion.Components()[0], kadmVersion.Components()[1])}, nil - } - - return nil, nil -} - -// KubeletVersionCheck validates installed kubelet version -type KubeletVersionCheck struct { - KubernetesVersion string - exec utilsexec.Interface -} - -// Name will return KubeletVersion as name for KubeletVersionCheck -func (KubeletVersionCheck) Name() string { - return "KubeletVersion" -} - -// Check validates kubelet version. It should be not less than minimal supported version -func (kubever KubeletVersionCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating kubelet version") - kubeletVersion, err := GetKubeletVersion(kubever.exec) - if err != nil { - return nil, []error{errors.Wrap(err, "couldn't get kubelet version")} - } - if kubeletVersion.LessThan(kubeadmconstants.MinimumKubeletVersion) { - return nil, []error{errors.Errorf("Kubelet version %q is lower than kubeadm can support. Please upgrade kubelet", kubeletVersion)} - } - - if kubever.KubernetesVersion != "" { - k8sVersion, err := versionutil.ParseSemantic(kubever.KubernetesVersion) - if err != nil { - return nil, []error{errors.Wrapf(err, "couldn't parse Kubernetes version %q", kubever.KubernetesVersion)} - } - if kubeletVersion.Major() > k8sVersion.Major() || kubeletVersion.Minor() > k8sVersion.Minor() { - return nil, []error{errors.Errorf("the kubelet version is higher than the control plane version. This is not a supported version skew and may lead to a malfunctional cluster. Kubelet version: %q Control plane version: %q", kubeletVersion, k8sVersion)} - } - } - return nil, nil -} - -// SwapCheck warns if swap is enabled -type SwapCheck struct{} - -// Name will return Swap as name for SwapCheck -func (SwapCheck) Name() string { - return "Swap" -} - -// Check validates whether swap is enabled or not -func (swc SwapCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating whether swap is enabled or not") - f, err := os.Open("/proc/swaps") - if err != nil { - // /proc/swaps not available, thus no reasons to warn - return nil, nil - } - defer f.Close() - var buf []string - scanner := bufio.NewScanner(f) - for scanner.Scan() { - buf = append(buf, scanner.Text()) - } - if err := scanner.Err(); err != nil { - return nil, []error{errors.Wrap(err, "error parsing /proc/swaps")} - } - - if len(buf) > 1 { - return nil, []error{errors.New("running with swap on is not supported. Please disable swap")} - } - - return nil, nil -} - -type etcdVersionResponse struct { - Etcdserver string `json:"etcdserver"` - Etcdcluster string `json:"etcdcluster"` -} - -// ExternalEtcdVersionCheck checks if version of external etcd meets the demand of kubeadm -type ExternalEtcdVersionCheck struct { - Etcd kubeadmapi.Etcd -} - -// Name will return ExternalEtcdVersion as name for ExternalEtcdVersionCheck -func (ExternalEtcdVersionCheck) Name() string { - return "ExternalEtcdVersion" -} - -// Check validates external etcd version -// TODO: Use the official etcd Golang client for this instead? -func (evc ExternalEtcdVersionCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating the external etcd version") - - // Return quickly if the user isn't using external etcd - if evc.Etcd.External.Endpoints == nil { - return nil, nil - } - - var config *tls.Config - var err error - if config, err = evc.configRootCAs(config); err != nil { - errorList = append(errorList, err) - return nil, errorList - } - if config, err = evc.configCertAndKey(config); err != nil { - errorList = append(errorList, err) - return nil, errorList - } - - client := evc.getHTTPClient(config) - for _, endpoint := range evc.Etcd.External.Endpoints { - if _, err := url.Parse(endpoint); err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to parse external etcd endpoint %s", endpoint)) - continue - } - resp := etcdVersionResponse{} - var err error - versionURL := fmt.Sprintf("%s/%s", endpoint, "version") - if tmpVersionURL, err := purell.NormalizeURLString(versionURL, purell.FlagRemoveDuplicateSlashes); err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to normalize external etcd version url %s", versionURL)) - continue - } else { - versionURL = tmpVersionURL - } - if err = getEtcdVersionResponse(client, versionURL, &resp); err != nil { - errorList = append(errorList, err) - continue - } - - etcdVersion, err := versionutil.ParseSemantic(resp.Etcdserver) - if err != nil { - errorList = append(errorList, errors.Wrapf(err, "couldn't parse external etcd version %q", resp.Etcdserver)) - continue - } - if etcdVersion.LessThan(minExternalEtcdVersion) { - errorList = append(errorList, errors.Errorf("this version of kubeadm only supports external etcd version >= %s. Current version: %s", kubeadmconstants.MinExternalEtcdVersion, resp.Etcdserver)) - continue - } - } - - return nil, errorList -} - -// configRootCAs configures and returns a reference to tls.Config instance if CAFile is provided -func (evc ExternalEtcdVersionCheck) configRootCAs(config *tls.Config) (*tls.Config, error) { - var CACertPool *x509.CertPool - if evc.Etcd.External.CAFile != "" { - CACert, err := ioutil.ReadFile(evc.Etcd.External.CAFile) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load external etcd's server certificate %s", evc.Etcd.External.CAFile) - } - CACertPool = x509.NewCertPool() - CACertPool.AppendCertsFromPEM(CACert) - } - if CACertPool != nil { - if config == nil { - config = &tls.Config{} - } - config.RootCAs = CACertPool - } - return config, nil -} - -// configCertAndKey configures and returns a reference to tls.Config instance if CertFile and KeyFile pair is provided -func (evc ExternalEtcdVersionCheck) configCertAndKey(config *tls.Config) (*tls.Config, error) { - var cert tls.Certificate - if evc.Etcd.External.CertFile != "" && evc.Etcd.External.KeyFile != "" { - var err error - cert, err = tls.LoadX509KeyPair(evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load external etcd's certificate and key pair %s, %s", evc.Etcd.External.CertFile, evc.Etcd.External.KeyFile) - } - if config == nil { - config = &tls.Config{} - } - config.Certificates = []tls.Certificate{cert} - } - return config, nil -} - -func (evc ExternalEtcdVersionCheck) getHTTPClient(config *tls.Config) *http.Client { - if config != nil { - transport := netutil.SetOldTransportDefaults(&http.Transport{ - TLSClientConfig: config, - }) - return &http.Client{ - Transport: transport, - Timeout: externalEtcdRequestTimeout, - } - } - return &http.Client{Timeout: externalEtcdRequestTimeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})} -} - -func getEtcdVersionResponse(client *http.Client, url string, target interface{}) error { - loopCount := externalEtcdRequestRetries + 1 - var err error - var stopRetry bool - for loopCount > 0 { - if loopCount <= externalEtcdRequestRetries { - time.Sleep(externalEtcdRequestInterval) - } - stopRetry, err = func() (stopRetry bool, err error) { - r, err := client.Get(url) - if err != nil { - loopCount-- - return false, err - } - //lint:ignore SA5011 If err != nil we are already returning. - defer r.Body.Close() - - if r != nil && r.StatusCode >= 500 && r.StatusCode <= 599 { - loopCount-- - return false, errors.Errorf("server responded with non-successful status: %s", r.Status) - } - return true, json.NewDecoder(r.Body).Decode(target) - - }() - if stopRetry { - break - } - } - return err -} - -// ImagePullCheck will pull container images used by kubeadm -type ImagePullCheck struct { - runtime utilruntime.ContainerRuntime - imageList []string -} - -// Name returns the label for ImagePullCheck -func (ImagePullCheck) Name() string { - return "ImagePull" -} - -// Check pulls images required by kubeadm. This is a mutating check -func (ipc ImagePullCheck) Check() (warnings, errorList []error) { - for _, image := range ipc.imageList { - ret, err := ipc.runtime.ImageExists(image) - if ret && err == nil { - klog.V(1).Infof("image exists: %s", image) - continue - } - if err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to check if image %s exists", image)) - } - klog.V(1).Infof("pulling %s", image) - if err := ipc.runtime.PullImage(image); err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to pull image %s", image)) - } - } - return warnings, errorList -} - -// NumCPUCheck checks if current number of CPUs is not less than required -type NumCPUCheck struct { - NumCPU int -} - -// Name returns the label for NumCPUCheck -func (NumCPUCheck) Name() string { - return "NumCPU" -} - -// Check number of CPUs required by kubeadm -func (ncc NumCPUCheck) Check() (warnings, errorList []error) { - numCPU := runtime.NumCPU() - if numCPU < ncc.NumCPU { - errorList = append(errorList, errors.Errorf("the number of available CPUs %d is less than the required %d", numCPU, ncc.NumCPU)) - } - return warnings, errorList -} - -// MemCheck checks if the number of megabytes of memory is not less than required -type MemCheck struct { - Mem uint64 -} - -// Name returns the label for memory -func (MemCheck) Name() string { - return "Mem" -} - -// RunInitNodeChecks executes all individual, applicable to control-plane node checks. -// The boolean flag 'isSecondaryControlPlane' controls whether we are running checks in a --join-control-plane scenario. -// The boolean flag 'downloadCerts' controls whether we should skip checks on certificates because we are downloading them. -// If the flag is set to true we should skip checks already executed by RunJoinNodeChecks. -func RunInitNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.InitConfiguration, ignorePreflightErrors sets.String, isSecondaryControlPlane bool, downloadCerts bool) error { - if !isSecondaryControlPlane { - // First, check if we're root separately from the other preflight checks and fail fast - if err := RunRootCheckOnly(ignorePreflightErrors); err != nil { - return err - } - } - - manifestsDir := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName) - checks := []Checker{ - NumCPUCheck{NumCPU: kubeadmconstants.ControlPlaneNumCPU}, - // Linux only - // TODO: support other OS, if control-plane is supported on it. - MemCheck{Mem: kubeadmconstants.ControlPlaneMem}, - KubernetesVersionCheck{KubernetesVersion: cfg.KubernetesVersion, KubeadmVersion: kubeadmversion.Get().GitVersion}, - FirewalldCheck{ports: []int{int(cfg.LocalAPIEndpoint.BindPort), kubeadmconstants.KubeletPort}}, - PortOpenCheck{port: int(cfg.LocalAPIEndpoint.BindPort)}, - PortOpenCheck{port: kubeadmconstants.KubeSchedulerPort}, - PortOpenCheck{port: kubeadmconstants.KubeControllerManagerPort}, - FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, manifestsDir)}, - FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeControllerManager, manifestsDir)}, - FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeScheduler, manifestsDir)}, - FileAvailableCheck{Path: kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestsDir)}, - HTTPProxyCheck{Proto: "https", Host: cfg.LocalAPIEndpoint.AdvertiseAddress}, - } - cidrs := strings.Split(cfg.Networking.ServiceSubnet, ",") - for _, cidr := range cidrs { - checks = append(checks, HTTPProxyCIDRCheck{Proto: "https", CIDR: cidr}) - } - cidrs = strings.Split(cfg.Networking.PodSubnet, ",") - for _, cidr := range cidrs { - checks = append(checks, HTTPProxyCIDRCheck{Proto: "https", CIDR: cidr}) - } - - if !isSecondaryControlPlane { - checks = addCommonChecks(execer, cfg.KubernetesVersion, &cfg.NodeRegistration, checks) - - // Check if Bridge-netfilter and IPv6 relevant flags are set - if ip := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress); ip != nil { - if utilsnet.IsIPv6(ip) { - checks = append(checks, - FileContentCheck{Path: bridgenf6, Content: []byte{'1'}}, - FileContentCheck{Path: ipv6DefaultForwarding, Content: []byte{'1'}}, - ) - } - } - - // if using an external etcd - if cfg.Etcd.External != nil { - // Check external etcd version before creating the cluster - checks = append(checks, ExternalEtcdVersionCheck{Etcd: cfg.Etcd}) - } - } - - if cfg.Etcd.Local != nil { - // Only do etcd related checks when required to install a local etcd - checks = append(checks, - PortOpenCheck{port: kubeadmconstants.EtcdListenClientPort}, - PortOpenCheck{port: kubeadmconstants.EtcdListenPeerPort}, - DirAvailableCheck{Path: cfg.Etcd.Local.DataDir}, - ) - } - - if cfg.Etcd.External != nil && !(isSecondaryControlPlane && downloadCerts) { - // Only check etcd certificates when using an external etcd and not joining with automatic download of certs - if cfg.Etcd.External.CAFile != "" { - checks = append(checks, FileExistingCheck{Path: cfg.Etcd.External.CAFile, Label: "ExternalEtcdClientCertificates"}) - } - if cfg.Etcd.External.CertFile != "" { - checks = append(checks, FileExistingCheck{Path: cfg.Etcd.External.CertFile, Label: "ExternalEtcdClientCertificates"}) - } - if cfg.Etcd.External.KeyFile != "" { - checks = append(checks, FileExistingCheck{Path: cfg.Etcd.External.KeyFile, Label: "ExternalEtcdClientCertificates"}) - } - } - - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// RunJoinNodeChecks executes all individual, applicable to node checks. -func RunJoinNodeChecks(execer utilsexec.Interface, cfg *kubeadmapi.JoinConfiguration, ignorePreflightErrors sets.String) error { - // First, check if we're root separately from the other preflight checks and fail fast - if err := RunRootCheckOnly(ignorePreflightErrors); err != nil { - return err - } - - checks := []Checker{ - DirAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}, - FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)}, - FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)}, - } - checks = addCommonChecks(execer, "", &cfg.NodeRegistration, checks) - if cfg.ControlPlane == nil { - checks = append(checks, FileAvailableCheck{Path: cfg.CACertPath}) - } - - addIPv6Checks := false - if cfg.Discovery.BootstrapToken != nil { - ipstr, _, err := net.SplitHostPort(cfg.Discovery.BootstrapToken.APIServerEndpoint) - if err == nil { - checks = append(checks, - HTTPProxyCheck{Proto: "https", Host: ipstr}, - ) - if ip := net.ParseIP(ipstr); ip != nil { - if utilsnet.IsIPv6(ip) { - addIPv6Checks = true - } - } - } - } - if addIPv6Checks { - checks = append(checks, - FileContentCheck{Path: bridgenf6, Content: []byte{'1'}}, - FileContentCheck{Path: ipv6DefaultForwarding, Content: []byte{'1'}}, - ) - } - - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// addCommonChecks is a helper function to duplicate checks that are common between both the -// kubeadm init and join commands -func addCommonChecks(execer utilsexec.Interface, k8sVersion string, nodeReg *kubeadmapi.NodeRegistrationOptions, checks []Checker) []Checker { - containerRuntime, err := utilruntime.NewContainerRuntime(execer, nodeReg.CRISocket) - isDocker := false - if err != nil { - fmt.Printf("[preflight] WARNING: Couldn't create the interface used for talking to the container runtime: %v\n", err) - } else { - checks = append(checks, ContainerRuntimeCheck{runtime: containerRuntime}) - if containerRuntime.IsDocker() { - isDocker = true - checks = append(checks, ServiceCheck{Service: "docker", CheckIfActive: true}) - // Linux only - // TODO: support other CRIs for this check eventually - // https://github.com/kubernetes/kubeadm/issues/874 - checks = append(checks, IsDockerSystemdCheck{}) - } - } - - // non-windows checks - if runtime.GOOS == "linux" { - if !isDocker { - checks = append(checks, InPathCheck{executable: "crictl", mandatory: true, exec: execer}) - } - checks = append(checks, - FileContentCheck{Path: bridgenf, Content: []byte{'1'}}, - FileContentCheck{Path: ipv4Forward, Content: []byte{'1'}}, - SwapCheck{}, - InPathCheck{executable: "conntrack", mandatory: true, exec: execer}, - InPathCheck{executable: "ip", mandatory: true, exec: execer}, - InPathCheck{executable: "iptables", mandatory: true, exec: execer}, - InPathCheck{executable: "mount", mandatory: true, exec: execer}, - InPathCheck{executable: "nsenter", mandatory: true, exec: execer}, - InPathCheck{executable: "ebtables", mandatory: false, exec: execer}, - InPathCheck{executable: "ethtool", mandatory: false, exec: execer}, - InPathCheck{executable: "socat", mandatory: false, exec: execer}, - InPathCheck{executable: "tc", mandatory: false, exec: execer}, - InPathCheck{executable: "touch", mandatory: false, exec: execer}) - } - checks = append(checks, - SystemVerificationCheck{IsDocker: isDocker}, - HostnameCheck{nodeName: nodeReg.Name}, - KubeletVersionCheck{KubernetesVersion: k8sVersion, exec: execer}, - ServiceCheck{Service: "kubelet", CheckIfActive: false}, - PortOpenCheck{port: kubeadmconstants.KubeletPort}) - return checks -} - -// RunRootCheckOnly initializes checks slice of structs and call RunChecks -func RunRootCheckOnly(ignorePreflightErrors sets.String) error { - checks := []Checker{ - IsPrivilegedUserCheck{}, - } - - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// RunPullImagesCheck will pull images kubeadm needs if they are not found on the system -func RunPullImagesCheck(execer utilsexec.Interface, cfg *kubeadmapi.InitConfiguration, ignorePreflightErrors sets.String) error { - containerRuntime, err := utilruntime.NewContainerRuntime(utilsexec.New(), cfg.NodeRegistration.CRISocket) - if err != nil { - return err - } - - checks := []Checker{ - ImagePullCheck{runtime: containerRuntime, imageList: images.GetControlPlaneImages(&cfg.ClusterConfiguration)}, - } - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// RunChecks runs each check, displays it's warnings/errors, and once all -// are processed will exit if any errors occurred. -func RunChecks(checks []Checker, ww io.Writer, ignorePreflightErrors sets.String) error { - var errsBuffer bytes.Buffer - - for _, c := range checks { - name := c.Name() - warnings, errs := c.Check() - - if setHasItemOrAll(ignorePreflightErrors, name) { - // Decrease severity of errors to warnings for this check - warnings = append(warnings, errs...) - errs = []error{} - } - - for _, w := range warnings { - io.WriteString(ww, fmt.Sprintf("\t[WARNING %s]: %v\n", name, w)) - } - for _, i := range errs { - errsBuffer.WriteString(fmt.Sprintf("\t[ERROR %s]: %v\n", name, i.Error())) - } - } - if errsBuffer.Len() > 0 { - return &Error{Msg: errsBuffer.String()} - } - return nil -} - -// setHasItemOrAll is helper function that return true if item is present in the set (case insensitive) or special key 'all' is present -func setHasItemOrAll(s sets.String, item string) bool { - if s.Has("all") || s.Has(strings.ToLower(item)) { - return true - } - return false -} diff --git a/cmd/kubeadm/app/preflight/checks_darwin.go b/cmd/kubeadm/app/preflight/checks_darwin.go deleted file mode 100644 index b6508b5ec446a..0000000000000 --- a/cmd/kubeadm/app/preflight/checks_darwin.go +++ /dev/null @@ -1,33 +0,0 @@ -// +build darwin - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -// This is a MacOS stub - -// Check validates if Docker is setup to use systemd as the cgroup driver. -// No-op for Darwin (MacOS). -func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) { - return nil, nil -} - -// Check number of memory required by kubeadm -// No-op for Darwin (MacOS). -func (mc MemCheck) Check() (warnings, errorList []error) { - return nil, nil -} diff --git a/cmd/kubeadm/app/preflight/checks_linux.go b/cmd/kubeadm/app/preflight/checks_linux.go deleted file mode 100644 index c89d689013631..0000000000000 --- a/cmd/kubeadm/app/preflight/checks_linux.go +++ /dev/null @@ -1,60 +0,0 @@ -// +build linux - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "syscall" - - "github.com/pkg/errors" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/utils/exec" -) - -// Check validates if Docker is setup to use systemd as the cgroup driver. -func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) { - driver, err := kubeadmutil.GetCgroupDriverDocker(exec.New()) - if err != nil { - return nil, []error{err} - } - if driver != kubeadmutil.CgroupDriverSystemd { - err = errors.Errorf("detected %q as the Docker cgroup driver. "+ - "The recommended driver is %q. "+ - "Please follow the guide at https://kubernetes.io/docs/setup/cri/", - driver, - kubeadmutil.CgroupDriverSystemd) - return []error{err}, nil - } - return nil, nil -} - -// Check number of memory required by kubeadm -func (mc MemCheck) Check() (warnings, errorList []error) { - info := syscall.Sysinfo_t{} - err := syscall.Sysinfo(&info) - if err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to get system info")) - } - - // Totalram returns bytes; convert to MB - actual := uint64(info.Totalram) / 1024 / 1024 - if actual < mc.Mem { - errorList = append(errorList, errors.Errorf("the system RAM (%d MB) is less than the minimum %d MB", actual, mc.Mem)) - } - return warnings, errorList -} diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go deleted file mode 100644 index 52a8bb127b471..0000000000000 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ /dev/null @@ -1,885 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "bytes" - "fmt" - "io/ioutil" - "runtime" - "strings" - "testing" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - - "net/http" - "os" - - "k8s.io/apimachinery/pkg/util/sets" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -var ( - externalEtcdRootCAFileContent = dedent.Dedent(` - -----BEGIN CERTIFICATE----- - MIIFrjCCA5agAwIBAgIUJAM5bQz/Ann8qye8T7Uyl+cAt3wwDQYJKoZIhvcNAQEN - BQAwbzEOMAwGA1UEBhMFQ2hpbmExDzANBgNVBAgTBkhhaW5hbjEOMAwGA1UEBxMF - U2FueWExDTALBgNVBAoTBGV0Y2QxFjAUBgNVBAsTDWV0Y2Qgc2VjdXJpdHkxFTAT - BgNVBAMTDGV0Y2Qtcm9vdC1jYTAeFw0xNzAyMjIwNzEyMDBaFw0yMjAyMjEwNzEy - MDBaMG8xDjAMBgNVBAYTBUNoaW5hMQ8wDQYDVQQIEwZIYWluYW4xDjAMBgNVBAcT - BVNhbnlhMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIHNlY3VyaXR5MRUw - EwYDVQQDEwxldGNkLXJvb3QtY2EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK - AoICAQDD16VNTwvEvy1yd/vt8Eq2NwTw51mKHGYlZwsDqdqMEnEiWoJ7Iv9HZ+cl - jX0FnahKnaV76j3xPO73L5WOvRYxnZ8MvU/aBdDO+Tct4ht3m7TJaav6s55otjDy - dQNmlpBt4fFEB/nDozQaocfu2mqr5nyKJOjJpe+57Uw4h0LshreDOlzHEs8CkP6W - /B9yGFARVyz84YgVtemUX8WTB3cVU49KEYMCuhqXY8s97xSTGT/4Tq/MruKb2V+w - uUPjvyO5eIUcWetjBhgEGsS37NrsSFhoUNMp/PtIkth0LQoWb9sjnG069KIQqm61 - 1PKxH7jgLYLf4q455iAuTFr0lF1OcmICTeJB+GiS+3ubOb1TH3AYICXvQUniNWJx - sDz3qUUu4GLHk9wHtdNmX2FXYB8kHMZAidDM4Zw3IhZZap6n6BlGVVBV5h8sNM3t - SB+pDLuAaZLx3/ah2ds6AwkfaMdYDsE/MWcWQqzBfhOp758Mx3dF16IY+6IQp0RS - 8qGKxgLDnTF9LgyHVOait2N/pT54faf8//ShSqTqzTK1wzHCkYwL6/B259zXWxeX - z4gOpQOk4rO4pgm/65QW9aKzHoQnpQ7lFQL2cdsKJv2tyC7pDfVrFy2uHWaUibbP - 7pDw3OD8MQwR1TuhflK1AIicpMQe/kTAuRwH4fneeaGVdddBQQIDAQABo0IwQDAO - BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUtoqcReNJ - p8z8Hz1/Q7XMK2fgi74wDQYJKoZIhvcNAQENBQADggIBADbh4HB//Gb0TUUEPoSw - VMJSUK1pb6KVTqAITSCKPwGT8KfCvVpUxEjh9J3dm1L8wbdr48yffdjhdl96cx2F - aGWdUIxRBIcpt5xvauBoj0OwfNcD5B9q1aKuh5XPNu4BndNeGw51vdJ8bJbtrZa8 - wKWF/PHciCo/wlzE/YgsemHeY5bYeXawXVP/+ocoLH82Fb8Aq0Af3ZABiA6fmawz - FiZlnIrZnHVJYSap4yDhC/AQECXKY5gj7kjSnDebsIYds5OrW0D3LeRzs+q5nQXE - xR35qg834kxUULS8AywqmR3+zjfeymm2FtsjT/PuzEImA80y29qpLZIpPg0meKHF - pCMJkEHaRh4/JAinLaKCGLpnchqBy7CR6yvVnGkx93J0louIbVyUfn63R6mxCvd7 - kL16a2xBMKgV4RDFcu+VYjbJTFdWOTGFrxPBmd/rLdwD3XNiwPtI0vXGM7I35DDP - SWwKVvR97F3uEnIQ1u8vHa1pNfQ1qSf/+hUJx2D9ypr7LTQ0LpLh1vUeTeUAVHmT - EEpcqzDg6lsqXw6KHJ55kd3QR/hRXd/Vr6EWUawDEnGjxyFVV2dTBbunsbSobNI4 - eKV+60oCk3NMwrZoLw4Fv5qs2saS62dgJNfxbKqBX9ljSQxGzHjRwh+hVByCnG8m - Z9JkQayesM6D7uwbQJXd5rgy - -----END CERTIFICATE----- - `) - - externalEtcdCertFileContent = dedent.Dedent(` - -----BEGIN CERTIFICATE----- - MIIGEjCCA/qgAwIBAgIURHJFslbPveA1WwQ4FaPJg1x6B8YwDQYJKoZIhvcNAQEN - BQAwbzEOMAwGA1UEBhMFQ2hpbmExDzANBgNVBAgTBkhhaW5hbjEOMAwGA1UEBxMF - U2FueWExDTALBgNVBAoTBGV0Y2QxFjAUBgNVBAsTDWV0Y2Qgc2VjdXJpdHkxFTAT - BgNVBAMTDGV0Y2Qtcm9vdC1jYTAeFw0xNzAyMjIwNzE0MDBaFw0yNzAyMjAwNzE0 - MDBaMGwxDjAMBgNVBAYTBUNoaW5hMQ8wDQYDVQQIEwZIYWluYW4xDjAMBgNVBAcT - BVNhbnlhMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIHNlY3VyaXR5MRIw - EAYDVQQDEwlteS1ldGNkLTEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC - AQCmCR4OSRrUCES90sUbj5tvjF24lPCMj7qP9MBUxcVvWfaJM12o4AxqBr8OThgd - lpNvlbKmRpfvbraXiDnuGty1vPa3z7RmKbwFgENfgKHz4fUw/MQ7CALOQ5PAvgf1 - rQ6Ii4cr49nWctpQmBXHtZRjvquBYnw70KrWfQ121DwPYy7cb/StuHLsTgqsgzhl - ECILWCj9GNqcGQr5+ZvwUxa2yam2CS1M+PLbB6HxX/4RBBTWKAt8+kjt6TxxMaSE - bNDHNDLWzQSpxg5qTLOQtrubFD4O3JT2E8DEj+LvXJKH7pJd1Z+r0m3ymQvBAIXr - 6OJs+sHbaaxKWS35k9m88NRojR+r5KPoEcBgxhtBtXUfMS5v5dTtcNsHl/mHmTC+ - gWiqpzA+tF55uUEWhRoA+pN7Ie2PviRhG43t99l7bsHVnrxZQqWsWlvCxMN1c2+7 - PRwhsYZFITyKcMSvd19Nb5HGc5hT7btZlWc2xKS2YNnDXbD8C5SdxZek5Cb/xRxL - T8taf2c1bHs8sZrzIK2DCGvaN3471WEnmaCuRWr2fqyJeCPwsvvWeNDVmgPP6v7g - ncyy+4QyyfNrdURTZFyw81ZbCiznPc070u7vtIYt3Sa0NXd0oEG1ybAZwBIYhMOY - 5ctepJLf7QxHXR70RdI0ksHEmZGZ1igk7gzhmHEgQM87pQIDAQABo4GoMIGlMA4G - A1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYD - VR0TAQH/BAIwADAdBgNVHQ4EFgQU0U/Zn4mc95UXm+LVO67wqJpL9gIwHwYDVR0j - BBgwFoAUtoqcReNJp8z8Hz1/Q7XMK2fgi74wJgYDVR0RBB8wHYIJbG9jYWxob3N0 - hwR/AAABhwQKcjPGhwQKcgwwMA0GCSqGSIb3DQEBDQUAA4ICAQCikW5SNpndBxEz - qblER72KkfSEXMFhQry3RZJeAw6rQiOl+PMJqMnylcepOAUrNi20emS270dQDh3z - Hw/JBgKftZ1JrjbF9NF4oFUZcFUKmTgyWYnhLH0BskgwJf2u+DpugFa4U8niQf15 - ciZGoUfWCGOJbgVP7esdnyhH/P/DpOEObWf8vOfvfQ49r7MzATyzMESyJjdtAH/F - c5JKACxpJhaYfTZ78F43jSw0vswBdLQ7fJWqg/sJBlTG0GBFJcEJzFVpwzYUxwZ4 - rUpAn4A02M2V9XDNlptrWvcQz/5Vs/aCmehz7GOiMJB6SLWcMSpJRLMqoJjaFVfO - OPm7bWMMaVOUPedzvcBKRXmEAg7HQnm3ibkVNjTW8Hr66n34Yk/dO9WXD+6IXnOQ - bMY+Mf9vpIsscSpGTO15sAKqiXCzHR9RWqNd4U3jvo3JtewkNMhIKzPThgYNfsO3 - 7HSrlfffeEQKc59rDUaC3Y9YSc5ERJRMC+mdOqXNMy2iedZnNEsmgYlaVDg6xfG8 - 65w9UkMOe+DTJtMHnMxP4rT6WE4cKysQeSYxkyo/jh+8rKEy9+AyuEntJAknABUc - N5mizdYu8nrtiSu9jdLKhwO41gC2IlXPUHizylo6g24RFVBjHLlzYAAsVMMMSQW1 - XRMVQjawUTknbAgHuE7/rEX8c27WUA== - -----END CERTIFICATE----- - `) - externalEtcdKeyFileContent = dedent.Dedent(` - -----BEGIN RSA PRIVATE KEY----- - MIIJKAIBAAKCAgEApgkeDkka1AhEvdLFG4+bb4xduJTwjI+6j/TAVMXFb1n2iTNd - qOAMaga/Dk4YHZaTb5WypkaX7262l4g57hrctbz2t8+0Zim8BYBDX4Ch8+H1MPzE - OwgCzkOTwL4H9a0OiIuHK+PZ1nLaUJgVx7WUY76rgWJ8O9Cq1n0NdtQ8D2Mu3G/0 - rbhy7E4KrIM4ZRAiC1go/RjanBkK+fmb8FMWtsmptgktTPjy2weh8V/+EQQU1igL - fPpI7ek8cTGkhGzQxzQy1s0EqcYOakyzkLa7mxQ+DtyU9hPAxI/i71ySh+6SXdWf - q9Jt8pkLwQCF6+jibPrB22msSlkt+ZPZvPDUaI0fq+Sj6BHAYMYbQbV1HzEub+XU - 7XDbB5f5h5kwvoFoqqcwPrReeblBFoUaAPqTeyHtj74kYRuN7ffZe27B1Z68WUKl - rFpbwsTDdXNvuz0cIbGGRSE8inDEr3dfTW+RxnOYU+27WZVnNsSktmDZw12w/AuU - ncWXpOQm/8UcS0/LWn9nNWx7PLGa8yCtgwhr2jd+O9VhJ5mgrkVq9n6siXgj8LL7 - 1njQ1ZoDz+r+4J3MsvuEMsnza3VEU2RcsPNWWwos5z3NO9Lu77SGLd0mtDV3dKBB - tcmwGcASGITDmOXLXqSS3+0MR10e9EXSNJLBxJmRmdYoJO4M4ZhxIEDPO6UCAwEA - AQKCAgEAmr3OlDPP3CLkpiFEcJ5TmA+y3S96TRY7IqVRhvBXRKMMoOwNczF0gHBP - Ka7gzNqkCA/1UwBh49VEOU/N5bqFTp+RNNhQYhKtWFck82H4Dkrd8EzzOa0KqF/U - 2YKB+pbR/7JCRUZypGmgTBKh4eG6LYfrYYd/D2Q3g/VCUigU3aZrayiwWiOYf+Fw - Ez2slowFnpsIgHHkdCzmzPi0O7PEbJDgKXa+EInIFRg09renGwa5wKnLoyvEQm7o - VPqWQJEFt1JPu1+R5ARhNPLNO6cCi9K+z60G65yXQNp0/u5A5o0TPn609DcHH11B - 1ht9tNL0C+tcNvhyiUw6C+uet3egDVu1TqptzAfb2Y3MQK6UV/by7KJxcFxBAzWl - UQ4zDaQzCcU81T92zI+XeRSJuCPuOL61mH7zEiPZZPOLV8MbxBX/7lj+IJTBL+vJ - Idq7Nn/+LRtuSy5PH2MzZ5DzIMmjkjQ/ScpzAr9Zpkm3dpTcGTpFV0uqHseE77Re - 55tz9uB7pxV1n6Gz4uMNnsioEYsFIRfzst4QWDdaQqcYJQuKvW9pXNmgRgSCIlft - 54DxQ98a1PVFmS40TT9mjUg0P66m+8bk5vEb58iAjoYJRcoriZhlT6cOcuPW6hos - 3PfA2gMXuWu61mAjzdP0zbzNBXCn5nRppqLNmWMVZCI0nLjmyZUCggEBAMEpCQu9 - cRWc/GjvmnfXHewvqQHu3A3J1HCLR0VqJo8rcIIvhSe7dPRAMtUFxV1R2eOfMvSZ - Y4y69tMHZPVTgnp2t5TSavjpMqSQLvXyBkgL8FnGEl5l6HEQTm8y0C13Cm+CUB5a - uxQnQflkX539SjWX0XdOmYuLORmrKGxgcDOd9652fDJcFSXYa0mx6KN2JZHh9psA - 9ldHhUIq1ngoVnrctlK53MptckPrFwMFdXRCKiMfkvpUkXTeXu4D7Z1VNh2V/3gF - lmRNioXaxp7W8omBSQlwaHY5btPj5jktiC9/so4ORqJjHvbCURrIvdkPPaXi/YJy - HdoOgHYFnn3p6M8CggEBANwNDtdbHWwwVC7Op6TNc8qK+SWAId5RqPOmM70XBVvg - u9nxT7a5vmRTs81fcVoxtE0t+KWIfOXquxqTbk0ONqIsl2CLTiTFaNHoHlvwgFBT - aYukORiGILIzOJr82RPugAw1+j8jmw3OsCOXnf2odGs+oC/V9vEd9NyZpDHPohtK - a8Bk8p326mQam23ArUesIqnw31fG22KRpoLXuk/9nNcAAAZd1Qd9hGWf0HHxunXB - wj6e3VTm0G4NPTli5vmVavYRPMFUUJpU5lwTHhlrHTSmANHTjZGnn0mEOfIrfodF - ODwJjwoyq4rPls0fqOvyAyBCnhop4fC8yOd4cQcLSUsCggEAbv9Br3lhLmZTtYla - XltDWqHYoL+9vD6q0TF39y+UkNkJggYEolxaTLFHhJoYXBPY/bBR+7TZO9mEVKf/ - H+qpI+5seByiU/7NlzszgSle6q/RogTsMUqmU7JnIAc3EalCWemsWIUS0/XrN4Cy - YXtX1Yw0VjbYjROn8FQmmoCgeUjhN2Pm4pl/nYvLu0F8ydHurPIIX/IhnO4AaZFs - RQgJCfki3E7pzXkvHFBPnPDaGcCbritKrodCPsI6EtQ3Cx4YRtAXScUMMv9MBrc9 - Q7GJFfMxITdzD9zZDvH7Lgg4JfNfi7owZMhI1su7B4UrczwK1PSncPpapR+IOkno - VbrAiQKCAQB2xGV6PqdGuV72VHuPK4SPkSqf3uRoxdJWjyHlsQMnb8hz/RZ1HRNx - uuuUsSrQ73rNHT7SuTQQM/0AfwpNdJpwNXkOlqF6n0HP6WRZYxkeQab5w409e0cy - ZwrqPAY+B7/81zVV1rXdYe0XiMGxIraTG54Bs44w3WZHmnVQnSx1Zll54gJA1//y - P5ocRp4/zNx4tJUXHzFRpiMlA6J/gfag5FMfHI3aGRjYcMVken+VBxr8CWqUZG+i - tmqRCpx3oPm2Dd+oyQUoByK+F2NrfLCqtd5DYddLAhmq6D8OQgNspyOO4+ncKzUD - Gr/dvnTBxEGDq/EBVhGoiXw10n/OuXy5AoIBAAUAoTyt4gQjjC0ddtMLN7+R1Ymp - eNULpq2XTvidj7jaysIW9Q52ncvN6h2Vds/Z3Ujvdne2jMq7Q/C96fKcrhgMH9ca - ADGLWtD+VkP4NgFjj7R2jabF8d9IQdJDXAgvR/kokojF0RsJuvD2hawN6lQkkj6S - fNNGMBk4sGyt7gzAn3iO4Zoy+QjtALNnZcaH6s7oIg3UKf6OwskiBB60Q5P1U3/E - RPtTxhex3jFuySNJ413JgyGkvcP+qjuzi6eyVDxkfiyNohQYGuZ8rieFX7QfQFAY - TIXptchVUTxmGKWzcpLC3AfkwFvV2IPoMk8YnDSp270D30cqWiI9puSEcxQ= - -----END RSA PRIVATE KEY----- - `) -) - -type preflightCheckTest struct { - msg string -} - -func (pfct preflightCheckTest) Name() string { - return "preflightCheckTest" -} - -func (pfct preflightCheckTest) Check() (warning, errorList []error) { - if pfct.msg == "warning" { - return []error{errors.New("warning")}, nil - } - if pfct.msg != "" { - return nil, []error{errors.New("fake error")} - } - return -} - -func TestRunInitNodeChecks(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.InitConfiguration - expected bool - isSecondaryControlPlane bool - downloadCerts bool - }{ - {name: "Test valid advertised address", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "foo"}, - }, - expected: false, - }, - { - name: "Test CA file exists if specified", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CAFile: "/foo"}}, - }, - }, - expected: false, - }, - { - name: "Skip test CA file exists if specified/download certs", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CAFile: "/foo"}}, - }, - }, - expected: true, - isSecondaryControlPlane: true, - downloadCerts: true, - }, - { - name: "Test Cert file exists if specified", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CertFile: "/foo"}}, - }, - }, - expected: false, - }, - { - name: "Test Key file exists if specified", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CertFile: "/foo"}}, - }, - }, - expected: false, - }, - { - name: "Test APIEndpoint exists if exists", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "2001:1234::1:15"}, - }, - expected: false, - }, - } - for _, rt := range tests { - // TODO: Make RunInitNodeChecks accept a ClusterConfiguration object instead of InitConfiguration - actual := RunInitNodeChecks(exec.New(), rt.cfg, sets.NewString(), rt.isSecondaryControlPlane, rt.downloadCerts) - if (actual == nil) != rt.expected { - t.Errorf( - "failed RunInitNodeChecks:%v\n\texpected: %t\n\t actual: %t\n\t error: %v", - rt.name, - rt.expected, - (actual == nil), - actual, - ) - } - } -} - -func TestRunJoinNodeChecks(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.JoinConfiguration - expected bool - }{ - { - name: "Check empty JoinConfiguration", - cfg: &kubeadmapi.JoinConfiguration{}, - expected: false, - }, - { - name: "Check TLS Bootstrap APIServerEndpoint IPv4 addr", - cfg: &kubeadmapi.JoinConfiguration{ - Discovery: kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - APIServerEndpoint: "192.168.1.15", - }, - }, - }, - expected: false, - }, - { - name: "Check TLS Bootstrap APIServerEndpoint IPv6 addr", - cfg: &kubeadmapi.JoinConfiguration{ - Discovery: kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - APIServerEndpoint: "2001:1234::1:15", - }, - }, - }, - expected: false, - }, - } - - for _, rt := range tests { - actual := RunJoinNodeChecks(exec.New(), rt.cfg, sets.NewString()) - if (actual == nil) != rt.expected { - t.Errorf( - "failed RunJoinNodeChecks:%v\n\texpected: %t\n\t actual: %t", - rt.name, - rt.expected, - (actual != nil), - ) - } - } -} - -func TestRunChecks(t *testing.T) { - var tokenTest = []struct { - p []Checker - expected bool - output string - }{ - {[]Checker{}, true, ""}, - {[]Checker{preflightCheckTest{"warning"}}, true, "\t[WARNING preflightCheckTest]: warning\n"}, // should just print warning - {[]Checker{preflightCheckTest{"error"}}, false, ""}, - {[]Checker{preflightCheckTest{"test"}}, false, ""}, - {[]Checker{DirAvailableCheck{Path: "/does/not/exist"}}, true, ""}, - {[]Checker{DirAvailableCheck{Path: "/"}}, false, ""}, - {[]Checker{FileAvailableCheck{Path: "/does/not/exist"}}, true, ""}, - {[]Checker{FileContentCheck{Path: "/does/not/exist"}}, false, ""}, - {[]Checker{FileContentCheck{Path: "/"}}, true, ""}, - {[]Checker{FileContentCheck{Path: "/", Content: []byte("does not exist")}}, false, ""}, - {[]Checker{InPathCheck{executable: "foobarbaz", exec: exec.New()}}, true, "\t[WARNING FileExisting-foobarbaz]: foobarbaz not found in system path\n"}, - {[]Checker{InPathCheck{executable: "foobarbaz", mandatory: true, exec: exec.New()}}, false, ""}, - {[]Checker{InPathCheck{executable: "foobar", mandatory: false, exec: exec.New(), suggestion: "install foobar"}}, true, "\t[WARNING FileExisting-foobar]: foobar not found in system path\nSuggestion: install foobar\n"}, - } - for _, rt := range tokenTest { - buf := new(bytes.Buffer) - actual := RunChecks(rt.p, buf, sets.NewString()) - if (actual == nil) != rt.expected { - t.Errorf( - "failed RunChecks:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) - } - if buf.String() != rt.output { - t.Errorf( - "failed RunChecks:\n\texpected: %s\n\t actual: %s", - rt.output, - buf.String(), - ) - } - } -} -func TestConfigRootCAs(t *testing.T) { - f, err := ioutil.TempFile(os.TempDir(), "kubeadm-external-etcd-test-cafile") - if err != nil { - t.Errorf("failed configRootCAs:\n\texpected: succeed creating temp CA file\n\tactual:%v", err) - } - defer os.Remove(f.Name()) - if err := ioutil.WriteFile(f.Name(), []byte(externalEtcdRootCAFileContent), 0644); err != nil { - t.Errorf("failed configRootCAs:\n\texpected: succeed writing contents to temp CA file %s\n\tactual:%v", f.Name(), err) - } - - c := ExternalEtcdVersionCheck{Etcd: kubeadmapi.Etcd{External: &kubeadmapi.ExternalEtcd{CAFile: f.Name()}}} - - config, err := c.configRootCAs(nil) - if err != nil { - t.Errorf( - "failed configRootCAs:\n\texpected: has no error\n\tactual:%v", - err, - ) - } - if config.RootCAs == nil { - t.Errorf( - "failed configRootCAs:\n\texpected: RootCAs not equal to nil\n\tactual:%v", - config.RootCAs, - ) - } -} -func TestConfigCertAndKey(t *testing.T) { - certFile, err := ioutil.TempFile(os.TempDir(), "kubeadm-external-etcd-test-certfile") - if err != nil { - t.Errorf( - "failed configCertAndKey:\n\texpected: succeed creating temp CertFile file\n\tactual:%v", - err, - ) - } - defer os.Remove(certFile.Name()) - if err := ioutil.WriteFile(certFile.Name(), []byte(externalEtcdCertFileContent), 0644); err != nil { - t.Errorf( - "failed configCertAndKey:\n\texpected: succeed writing contents to temp CertFile file %s\n\tactual:%v", - certFile.Name(), - err, - ) - } - - keyFile, err := ioutil.TempFile(os.TempDir(), "kubeadm-external-etcd-test-keyfile") - if err != nil { - t.Errorf( - "failed configCertAndKey:\n\texpected: succeed creating temp KeyFile file\n\tactual:%v", - err, - ) - } - defer os.Remove(keyFile.Name()) - if err := ioutil.WriteFile(keyFile.Name(), []byte(externalEtcdKeyFileContent), 0644); err != nil { - t.Errorf( - "failed configCertAndKey:\n\texpected: succeed writing contents to temp KeyFile file %s\n\tactual:%v", - keyFile.Name(), - err, - ) - } - c := ExternalEtcdVersionCheck{ - Etcd: kubeadmapi.Etcd{ - External: &kubeadmapi.ExternalEtcd{ - CertFile: certFile.Name(), - KeyFile: keyFile.Name(), - }, - }, - } - - config, err := c.configCertAndKey(nil) - if err != nil { - t.Errorf( - "failed configCertAndKey:\n\texpected: has no error\n\tactual:%v", - err, - ) - } - if config.Certificates == nil { - t.Errorf( - "failed configCertAndKey:\n\texpected: Certificates not equal to nil\n\tactual:%v", - config.Certificates, - ) - } -} - -func TestKubernetesVersionCheck(t *testing.T) { - var tests = []struct { - check KubernetesVersionCheck - expectWarnings bool - }{ - { - check: KubernetesVersionCheck{ - KubeadmVersion: "v1.6.6", //Same version - KubernetesVersion: "v1.6.6", - }, - expectWarnings: false, - }, - { - check: KubernetesVersionCheck{ - KubeadmVersion: "v1.6.6", //KubernetesVersion version older than KubeadmVersion - KubernetesVersion: "v1.5.5", - }, - expectWarnings: false, - }, - { - check: KubernetesVersionCheck{ - KubeadmVersion: "v1.6.6", //KubernetesVersion newer than KubeadmVersion, within the same minor release (new patch) - KubernetesVersion: "v1.6.7", - }, - expectWarnings: false, - }, - { - check: KubernetesVersionCheck{ - KubeadmVersion: "v1.6.6", //KubernetesVersion newer than KubeadmVersion, in a different minor/in pre-release - KubernetesVersion: "v1.7.0-alpha.0", - }, - expectWarnings: true, - }, - { - check: KubernetesVersionCheck{ - KubeadmVersion: "v1.6.6", //KubernetesVersion newer than KubeadmVersion, in a different minor/stable - KubernetesVersion: "v1.7.0", - }, - expectWarnings: true, - }, - { - check: KubernetesVersionCheck{ - KubeadmVersion: "v0.0.0", //"super-custom" builds - Skip this check - KubernetesVersion: "v1.7.0", - }, - expectWarnings: false, - }, - } - - for _, rt := range tests { - warning, _ := rt.check.Check() - if (warning != nil) != rt.expectWarnings { - t.Errorf( - "failed KubernetesVersionCheck:\n\texpected: %t\n\t actual: %t (KubeadmVersion:%s, KubernetesVersion: %s)", - rt.expectWarnings, - (warning != nil), - rt.check.KubeadmVersion, - rt.check.KubernetesVersion, - ) - } - } -} - -func TestHTTPProxyCIDRCheck(t *testing.T) { - var tests = []struct { - check HTTPProxyCIDRCheck - expectWarnings bool - }{ - { - check: HTTPProxyCIDRCheck{ - Proto: "https", - CIDR: "127.0.0.0/8", - }, // Loopback addresses never should produce proxy warnings - expectWarnings: false, - }, - { - check: HTTPProxyCIDRCheck{ - Proto: "https", - CIDR: "10.96.0.0/12", - }, // Expected to be accessed directly, we set NO_PROXY to 10.0.0.0/8 - expectWarnings: false, - }, - { - check: HTTPProxyCIDRCheck{ - Proto: "https", - CIDR: "192.168.0.0/16", - }, // Expected to go via proxy as this range is not listed in NO_PROXY - expectWarnings: true, - }, - { - check: HTTPProxyCIDRCheck{ - Proto: "https", - CIDR: "2001:db8::/56", - }, // Expected to be accessed directly, part of 2001:db8::/48 in NO_PROXY - expectWarnings: false, - }, - { - check: HTTPProxyCIDRCheck{ - Proto: "https", - CIDR: "2001:db8:1::/56", - }, // Expected to go via proxy, range is not in 2001:db8::/48 - expectWarnings: true, - }, - } - - // Save current content of *_proxy and *_PROXY variables. - savedEnv := resetProxyEnv(t) - defer restoreEnv(savedEnv) - - for _, rt := range tests { - warning, _ := rt.check.Check() - if (warning != nil) != rt.expectWarnings { - t.Errorf( - "failed HTTPProxyCIDRCheck:\n\texpected: %t\n\t actual: %t (CIDR:%s). Warnings: %v", - rt.expectWarnings, - (warning != nil), - rt.check.CIDR, - warning, - ) - } - } -} - -func TestHTTPProxyCheck(t *testing.T) { - var tests = []struct { - name string - check HTTPProxyCheck - expectWarnings bool - }{ - { - name: "Loopback address", - check: HTTPProxyCheck{ - Proto: "https", - Host: "127.0.0.1", - }, // Loopback addresses never should produce proxy warnings - expectWarnings: false, - }, - { - name: "IPv4 direct access", - check: HTTPProxyCheck{ - Proto: "https", - Host: "10.96.0.1", - }, // Expected to be accessed directly, we set NO_PROXY to 10.0.0.0/8 - expectWarnings: false, - }, - { - name: "IPv4 via proxy", - check: HTTPProxyCheck{ - Proto: "https", - Host: "192.168.0.1", - }, // Expected to go via proxy as this range is not listed in NO_PROXY - expectWarnings: true, - }, - { - name: "IPv6 direct access", - check: HTTPProxyCheck{ - Proto: "https", - Host: "[2001:db8::1:15]", - }, // Expected to be accessed directly, part of 2001:db8::/48 in NO_PROXY - expectWarnings: false, - }, - { - name: "IPv6 via proxy", - check: HTTPProxyCheck{ - Proto: "https", - Host: "[2001:db8:1::1:15]", - }, // Expected to go via proxy, range is not in 2001:db8::/48 - expectWarnings: true, - }, - { - name: "IPv6 direct access, no brackets", - check: HTTPProxyCheck{ - Proto: "https", - Host: "2001:db8::1:15", - }, // Expected to be accessed directly, part of 2001:db8::/48 in NO_PROXY - expectWarnings: false, - }, - { - name: "IPv6 via proxy, no brackets", - check: HTTPProxyCheck{ - Proto: "https", - Host: "2001:db8:1::1:15", - }, // Expected to go via proxy, range is not in 2001:db8::/48 - expectWarnings: true, - }, - } - - // Save current content of *_proxy and *_PROXY variables. - savedEnv := resetProxyEnv(t) - defer restoreEnv(savedEnv) - - for _, rt := range tests { - warning, _ := rt.check.Check() - if (warning != nil) != rt.expectWarnings { - t.Errorf( - "%s failed HTTPProxyCheck:\n\texpected: %t\n\t actual: %t (Host:%s). Warnings: %v", - rt.name, - rt.expectWarnings, - (warning != nil), - rt.check.Host, - warning, - ) - } - } -} - -// resetProxyEnv is helper function that unsets all *_proxy variables -// and return previously set values as map. This can be used to restore -// original state of the environment. -func resetProxyEnv(t *testing.T) map[string]string { - savedEnv := make(map[string]string) - for _, e := range os.Environ() { - pair := strings.Split(e, "=") - if strings.HasSuffix(strings.ToLower(pair[0]), "_proxy") { - savedEnv[pair[0]] = pair[1] - os.Unsetenv(pair[0]) - } - } - t.Log("Saved environment: ", savedEnv) - - os.Setenv("HTTP_PROXY", "http://proxy.example.com:3128") - os.Setenv("NO_PROXY", "example.com,10.0.0.0/8,2001:db8::/48") - // Check if we can reliably execute tests: - // ProxyFromEnvironment caches the *_proxy environment variables and - // if ProxyFromEnvironment already executed before our test with empty - // HTTP_PROXY it will make these tests return false positive failures - req, err := http.NewRequest("GET", "http://host.fake.tld/", nil) - if err != nil { - t.Fatalf("unexpected err: %v", err) - } - proxy, err := http.ProxyFromEnvironment(req) - if err != nil { - t.Fatalf("unexpected err: %v", err) - } - if proxy == nil { - t.Skip("test skipped as ProxyFromEnvironment already initialized in environment without defined HTTP proxy") - } - t.Log("http.ProxyFromEnvironment is usable, continue executing test") - return savedEnv -} - -// restoreEnv is helper function to restores values -// of environment variables from saved state in the map -func restoreEnv(e map[string]string) { - for k, v := range e { - os.Setenv(k, v) - } -} - -func TestKubeletVersionCheck(t *testing.T) { - cases := []struct { - kubeletVersion string - k8sVersion string - expectErrors bool - expectWarnings bool - }{ - {"v" + constants.CurrentKubernetesVersion.WithPatch(2).String(), "", false, false}, // check minimally supported version when there is no information about control plane - {"v1.11.3", "v1.11.8", true, false}, // too old kubelet (older than kubeadmconstants.MinimumKubeletVersion), should fail. - {"v" + constants.MinimumKubeletVersion.String(), constants.MinimumControlPlaneVersion.WithPatch(5).String(), false, false}, // kubelet within same major.minor as control plane - {"v" + constants.MinimumKubeletVersion.WithPatch(5).String(), constants.MinimumControlPlaneVersion.WithPatch(1).String(), false, false}, // kubelet is newer, but still within same major.minor as control plane - {"v" + constants.MinimumKubeletVersion.String(), constants.CurrentKubernetesVersion.WithPatch(1).String(), false, false}, // kubelet is lower than control plane, but newer than minimally supported - {"v" + constants.CurrentKubernetesVersion.WithPreRelease("alpha.1").String(), constants.MinimumControlPlaneVersion.WithPatch(1).String(), true, false}, // kubelet is newer (development build) than control plane, should fail. - {"v" + constants.CurrentKubernetesVersion.String(), constants.MinimumControlPlaneVersion.WithPatch(5).String(), true, false}, // kubelet is newer (release) than control plane, should fail. - } - - for _, tc := range cases { - t.Run(tc.kubeletVersion, func(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - OutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("Kubernetes " + tc.kubeletVersion), nil, nil }, - }, - } - fexec := &fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } - - check := KubeletVersionCheck{KubernetesVersion: tc.k8sVersion, exec: fexec} - warnings, errors := check.Check() - - switch { - case warnings != nil && !tc.expectWarnings: - t.Errorf("KubeletVersionCheck: unexpected warnings for kubelet version %q and Kubernetes version %q. Warnings: %v", tc.kubeletVersion, tc.k8sVersion, warnings) - case warnings == nil && tc.expectWarnings: - t.Errorf("KubeletVersionCheck: expected warnings for kubelet version %q and Kubernetes version %q but got nothing", tc.kubeletVersion, tc.k8sVersion) - case errors != nil && !tc.expectErrors: - t.Errorf("KubeletVersionCheck: unexpected errors for kubelet version %q and Kubernetes version %q. errors: %v", tc.kubeletVersion, tc.k8sVersion, errors) - case errors == nil && tc.expectErrors: - t.Errorf("KubeletVersionCheck: expected errors for kubelet version %q and Kubernetes version %q but got nothing", tc.kubeletVersion, tc.k8sVersion) - } - }) - } -} - -func TestSetHasItemOrAll(t *testing.T) { - var tests = []struct { - ignoreSet sets.String - testString string - expectedResult bool - }{ - {sets.NewString(), "foo", false}, - {sets.NewString("all"), "foo", true}, - {sets.NewString("all", "bar"), "foo", true}, - {sets.NewString("bar"), "foo", false}, - {sets.NewString("baz", "foo", "bar"), "foo", true}, - {sets.NewString("baz", "bar", "foo"), "Foo", true}, - } - - for _, rt := range tests { - t.Run(rt.testString, func(t *testing.T) { - result := setHasItemOrAll(rt.ignoreSet, rt.testString) - if result != rt.expectedResult { - t.Errorf( - "setHasItemOrAll: expected: %v actual: %v (arguments: %q, %q)", - rt.expectedResult, result, - rt.ignoreSet, - rt.testString, - ) - } - }) - } -} - -func TestImagePullCheck(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - RunScript: []fakeexec.FakeAction{ - // Test case 1: img1 and img2 exist, img3 doesn't exist - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - - // Test case 2: images don't exist - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - }, - CombinedOutputScript: []fakeexec.FakeAction{ - // Test case1: pull only img3 - func() ([]byte, []byte, error) { return nil, nil, nil }, - // Test case 2: fail to pull image2 and image3 - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - - fexec := fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, - } - - containerRuntime, err := utilruntime.NewContainerRuntime(&fexec, constants.DefaultDockerCRISocket) - if err != nil { - t.Errorf("unexpected NewContainerRuntime error: %v", err) - } - - check := ImagePullCheck{ - runtime: containerRuntime, - imageList: []string{"img1", "img2", "img3"}, - } - warnings, errors := check.Check() - if len(warnings) != 0 { - t.Fatalf("did not expect any warnings but got %q", warnings) - } - if len(errors) != 0 { - t.Fatalf("expected 1 errors but got %d: %q", len(errors), errors) - } - - warnings, errors = check.Check() - if len(warnings) != 0 { - t.Fatalf("did not expect any warnings but got %q", warnings) - } - if len(errors) != 2 { - t.Fatalf("expected 2 errors but got %d: %q", len(errors), errors) - } -} - -func TestNumCPUCheck(t *testing.T) { - var tests = []struct { - numCPU int - numErrors int - numWarnings int - }{ - {0, 0, 0}, - {999999999, 1, 0}, - } - - for _, rt := range tests { - t.Run(fmt.Sprintf("number of CPUs: %d", rt.numCPU), func(t *testing.T) { - warnings, errors := NumCPUCheck{NumCPU: rt.numCPU}.Check() - if len(warnings) != rt.numWarnings { - t.Errorf("expected %d warning(s) but got %d: %q", rt.numWarnings, len(warnings), warnings) - } - if len(errors) != rt.numErrors { - t.Errorf("expected %d warning(s) but got %d: %q", rt.numErrors, len(errors), errors) - } - }) - } -} - -func TestMemCheck(t *testing.T) { - // skip this test, if OS in not Linux, since it will ONLY pass on Linux. - if runtime.GOOS != "linux" { - t.Skip("unsupported OS for memory check test ") - } - - var tests = []struct { - minimum uint64 - expectedErrors int - }{ - {0, 0}, - {9999999999999999, 1}, - } - - for _, rt := range tests { - t.Run(fmt.Sprintf("MemoryCheck{%d}", rt.minimum), func(t *testing.T) { - warnings, errors := MemCheck{Mem: rt.minimum}.Check() - if len(warnings) > 0 { - t.Errorf("expected 0 warnings but got %d: %q", len(warnings), warnings) - } else if len(errors) != rt.expectedErrors { - t.Errorf("expected %d error(s) but got %d: %q", rt.expectedErrors, len(errors), errors) - } - }) - } -} diff --git a/cmd/kubeadm/app/preflight/checks_unix.go b/cmd/kubeadm/app/preflight/checks_unix.go deleted file mode 100644 index 2bbbd9ca2d377..0000000000000 --- a/cmd/kubeadm/app/preflight/checks_unix.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build !windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "os" - - "github.com/pkg/errors" -) - -// Check validates if an user has elevated (root) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { - if os.Getuid() != 0 { - return nil, []error{errors.New("user is not running as root")} - } - - return nil, nil -} diff --git a/cmd/kubeadm/app/preflight/checks_windows.go b/cmd/kubeadm/app/preflight/checks_windows.go deleted file mode 100644 index 57e5e3b8e05cb..0000000000000 --- a/cmd/kubeadm/app/preflight/checks_windows.go +++ /dev/null @@ -1,62 +0,0 @@ -// +build windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "os/user" - - "github.com/pkg/errors" -) - -// The "Well-known SID" of Administrator group -// https://support.microsoft.com/en-us/help/243330/well-known-security-identifiers-in-windows-operating-systems -const administratorSID = "S-1-5-32-544" - -// Check validates if a user has elevated (administrator) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { - currUser, err := user.Current() - if err != nil { - return nil, []error{errors.Wrap(err, "cannot get current user")} - } - - groupIds, err := currUser.GroupIds() - if err != nil { - return nil, []error{errors.Wrap(err, "cannot get group IDs for current user")} - } - - for _, sid := range groupIds { - if sid == administratorSID { - return nil, nil - } - } - - return nil, []error{errors.New("user is not running as administrator")} -} - -// Check validates if Docker is setup to use systemd as the cgroup driver. -// No-op for Windows. -func (idsc IsDockerSystemdCheck) Check() (warnings, errorList []error) { - return nil, nil -} - -// Check number of memory required by kubeadm -// No-op for Windows. -func (mc MemCheck) Check() (warnings, errorList []error) { - return nil, nil -} diff --git a/cmd/kubeadm/app/preflight/utils.go b/cmd/kubeadm/app/preflight/utils.go deleted file mode 100644 index 1587f64e08d53..0000000000000 --- a/cmd/kubeadm/app/preflight/utils.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "regexp" - "strings" - - "github.com/pkg/errors" - - "k8s.io/apimachinery/pkg/util/version" - utilsexec "k8s.io/utils/exec" -) - -// GetKubeletVersion is helper function that returns version of kubelet available in $PATH -func GetKubeletVersion(execer utilsexec.Interface) (*version.Version, error) { - kubeletVersionRegex := regexp.MustCompile(`^\s*Kubernetes v((0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)([-0-9a-zA-Z_\.+]*)?)\s*$`) - - command := execer.Command("kubelet", "--version") - out, err := command.Output() - if err != nil { - return nil, errors.Wrap(err, "cannot execute 'kubelet --version'") - } - - cleanOutput := strings.TrimSpace(string(out)) - subs := kubeletVersionRegex.FindAllStringSubmatch(cleanOutput, -1) - if len(subs) != 1 || len(subs[0]) < 2 { - return nil, errors.Errorf("Unable to parse output from Kubelet: %q", cleanOutput) - } - return version.ParseSemantic(subs[0][1]) -} diff --git a/cmd/kubeadm/app/preflight/utils_test.go b/cmd/kubeadm/app/preflight/utils_test.go deleted file mode 100644 index a39eab77dbf92..0000000000000 --- a/cmd/kubeadm/app/preflight/utils_test.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "testing" - - "github.com/pkg/errors" - - utilsexec "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -func TestGetKubeletVersion(t *testing.T) { - cases := []struct { - output string - expected string - err error - valid bool - }{ - {"Kubernetes v1.7.0", "1.7.0", nil, true}, - {"Kubernetes v1.8.0-alpha.2.1231+afabd012389d53a", "1.8.0-alpha.2.1231+afabd012389d53a", nil, true}, - {"something-invalid", "", nil, false}, - {"command not found", "", errors.New("kubelet not found"), false}, - {"", "", nil, false}, - } - - for _, tc := range cases { - t.Run(tc.output, func(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - OutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte(tc.output), nil, tc.err }, - }, - } - fexec := &fakeexec.FakeExec{ - CommandScript: []fakeexec.FakeCommandAction{ - func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, - }, - } - ver, err := GetKubeletVersion(fexec) - switch { - case err != nil && tc.valid: - t.Errorf("GetKubeletVersion: unexpected error for %q. Error: %v", tc.output, err) - case err == nil && !tc.valid: - t.Errorf("GetKubeletVersion: error expected for key %q, but result is %q", tc.output, ver) - case ver != nil && ver.String() != tc.expected: - t.Errorf("GetKubeletVersion: unexpected version result for key %q. Expected: %q Actual: %q", tc.output, tc.expected, ver) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/BUILD b/cmd/kubeadm/app/util/BUILD deleted file mode 100644 index eb3e8ff5cb2ca..0000000000000 --- a/cmd/kubeadm/app/util/BUILD +++ /dev/null @@ -1,98 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "arguments.go", - "cgroupdriver.go", - "chroot_unix.go", - "chroot_windows.go", - "copy.go", - "endpoint.go", - "env.go", - "error.go", - "marshal.go", - "net.go", - "template.go", - "version.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/yaml:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "arguments_test.go", - "cgroupdriver_test.go", - "endpoint_test.go", - "error_test.go", - "marshal_test.go", - "net_test.go", - "template_test.go", - "version_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/util/apiclient:all-srcs", - "//cmd/kubeadm/app/util/certs:all-srcs", - "//cmd/kubeadm/app/util/config:all-srcs", - "//cmd/kubeadm/app/util/crypto:all-srcs", - "//cmd/kubeadm/app/util/dryrun:all-srcs", - "//cmd/kubeadm/app/util/etcd:all-srcs", - "//cmd/kubeadm/app/util/image:all-srcs", - "//cmd/kubeadm/app/util/initsystem:all-srcs", - "//cmd/kubeadm/app/util/kubeconfig:all-srcs", - "//cmd/kubeadm/app/util/output:all-srcs", - "//cmd/kubeadm/app/util/patches:all-srcs", - "//cmd/kubeadm/app/util/pkiutil:all-srcs", - "//cmd/kubeadm/app/util/pubkeypin:all-srcs", - "//cmd/kubeadm/app/util/runtime:all-srcs", - "//cmd/kubeadm/app/util/staticpod:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/apiclient/BUILD b/cmd/kubeadm/app/util/apiclient/BUILD deleted file mode 100644 index dce3f09193007..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/BUILD +++ /dev/null @@ -1,79 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "clientbacked_dryrun.go", - "dryrunclient.go", - "idempotency.go", - "init_dryrun.go", - "wait.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient", - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/api/apps/v1:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/dynamic:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/util/retry:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = [ - "dryrunclient_test.go", - "idempotency_test.go", - "init_dryrun_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/api/rbac/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go b/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go deleted file mode 100644 index e8a82f1657660..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/clientbacked_dryrun.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" - clientset "k8s.io/client-go/kubernetes" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - core "k8s.io/client-go/testing" - "k8s.io/client-go/tools/clientcmd" -) - -// ClientBackedDryRunGetter implements the DryRunGetter interface for use in NewDryRunClient() and proxies all GET and LIST requests to the backing API server reachable via rest.Config -type ClientBackedDryRunGetter struct { - client clientset.Interface - dynamicClient dynamic.Interface -} - -// InitDryRunGetter should implement the DryRunGetter interface -var _ DryRunGetter = &ClientBackedDryRunGetter{} - -// NewClientBackedDryRunGetter creates a new ClientBackedDryRunGetter instance based on the rest.Config object -func NewClientBackedDryRunGetter(config *rest.Config) (*ClientBackedDryRunGetter, error) { - client, err := clientset.NewForConfig(config) - if err != nil { - return nil, err - } - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return nil, err - } - - return &ClientBackedDryRunGetter{ - client: client, - dynamicClient: dynamicClient, - }, nil -} - -// NewClientBackedDryRunGetterFromKubeconfig creates a new ClientBackedDryRunGetter instance from the given KubeConfig file -func NewClientBackedDryRunGetterFromKubeconfig(file string) (*ClientBackedDryRunGetter, error) { - config, err := clientcmd.LoadFromFile(file) - if err != nil { - return nil, errors.Wrap(err, "failed to load kubeconfig") - } - clientConfig, err := clientcmd.NewDefaultClientConfig(*config, &clientcmd.ConfigOverrides{}).ClientConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to create API client configuration from kubeconfig") - } - return NewClientBackedDryRunGetter(clientConfig) -} - -// HandleGetAction handles GET actions to the dryrun clientset this interface supports -func (clg *ClientBackedDryRunGetter) HandleGetAction(action core.GetAction) (bool, runtime.Object, error) { - unstructuredObj, err := clg.dynamicClient.Resource(action.GetResource()).Namespace(action.GetNamespace()).Get(context.TODO(), action.GetName(), metav1.GetOptions{}) - // Inform the user that the requested object wasn't found. - printIfNotExists(err) - if err != nil { - return true, nil, err - } - newObj, err := decodeUnstructuredIntoAPIObject(action, unstructuredObj) - if err != nil { - fmt.Printf("error after decode: %v %v\n", unstructuredObj, err) - return true, nil, err - } - return true, newObj, err -} - -// HandleListAction handles LIST actions to the dryrun clientset this interface supports -func (clg *ClientBackedDryRunGetter) HandleListAction(action core.ListAction) (bool, runtime.Object, error) { - listOpts := metav1.ListOptions{ - LabelSelector: action.GetListRestrictions().Labels.String(), - FieldSelector: action.GetListRestrictions().Fields.String(), - } - - unstructuredList, err := clg.dynamicClient.Resource(action.GetResource()).Namespace(action.GetNamespace()).List(context.TODO(), listOpts) - if err != nil { - return true, nil, err - } - newObj, err := decodeUnstructuredIntoAPIObject(action, unstructuredList) - if err != nil { - fmt.Printf("error after decode: %v %v\n", unstructuredList, err) - return true, nil, err - } - return true, newObj, err -} - -// Client gets the backing clientset.Interface -func (clg *ClientBackedDryRunGetter) Client() clientset.Interface { - return clg.client -} - -// decodeUnversionedIntoAPIObject converts the *unversioned.Unversioned object returned from the dynamic client -// to bytes; and then decodes it back _to an external api version (k8s.io/api)_ using the normal API machinery -func decodeUnstructuredIntoAPIObject(action core.Action, unstructuredObj runtime.Unstructured) (runtime.Object, error) { - objBytes, err := json.Marshal(unstructuredObj) - if err != nil { - return nil, err - } - newObj, err := runtime.Decode(clientsetscheme.Codecs.UniversalDecoder(action.GetResource().GroupVersion()), objBytes) - if err != nil { - return nil, err - } - return newObj, nil -} - -func printIfNotExists(err error) { - if apierrors.IsNotFound(err) { - fmt.Println("[dryrun] The GET request didn't yield any result, the API Server returned a NotFound error.") - } -} diff --git a/cmd/kubeadm/app/util/apiclient/dryrunclient.go b/cmd/kubeadm/app/util/apiclient/dryrunclient.go deleted file mode 100644 index 69ec71d5e714b..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/dryrunclient.go +++ /dev/null @@ -1,243 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "bufio" - "bytes" - "fmt" - "io" - "strings" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - clientset "k8s.io/client-go/kubernetes" - fakeclientset "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -// DryRunGetter is an interface that must be supplied to the NewDryRunClient function in order to construct a fully functional fake dryrun clientset -type DryRunGetter interface { - HandleGetAction(core.GetAction) (bool, runtime.Object, error) - HandleListAction(core.ListAction) (bool, runtime.Object, error) -} - -// MarshalFunc takes care of converting any object to a byte array for displaying the object to the user -type MarshalFunc func(runtime.Object, schema.GroupVersion) ([]byte, error) - -// DefaultMarshalFunc is the default MarshalFunc used; uses YAML to print objects to the user -func DefaultMarshalFunc(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) { - return kubeadmutil.MarshalToYaml(obj, gv) -} - -// DryRunClientOptions specifies options to pass to NewDryRunClientWithOpts in order to get a dryrun clientset -type DryRunClientOptions struct { - Writer io.Writer - Getter DryRunGetter - PrependReactors []core.Reactor - AppendReactors []core.Reactor - MarshalFunc MarshalFunc - PrintGETAndLIST bool -} - -// GetDefaultDryRunClientOptions returns the default DryRunClientOptions values -func GetDefaultDryRunClientOptions(drg DryRunGetter, w io.Writer) DryRunClientOptions { - return DryRunClientOptions{ - Writer: w, - Getter: drg, - PrependReactors: []core.Reactor{}, - AppendReactors: []core.Reactor{}, - MarshalFunc: DefaultMarshalFunc, - PrintGETAndLIST: false, - } -} - -// actionWithName is the generic interface for an action that has a name associated with it -// This just makes it easier to catch all actions that has a name; instead of hard-coding all request that has it associated -type actionWithName interface { - core.Action - GetName() string -} - -// actionWithObject is the generic interface for an action that has an object associated with it -// This just makes it easier to catch all actions that has an object; instead of hard-coding all request that has it associated -type actionWithObject interface { - core.Action - GetObject() runtime.Object -} - -// NewDryRunClient is a wrapper for NewDryRunClientWithOpts using some default values -func NewDryRunClient(drg DryRunGetter, w io.Writer) clientset.Interface { - return NewDryRunClientWithOpts(GetDefaultDryRunClientOptions(drg, w)) -} - -// NewDryRunClientWithOpts returns a clientset.Interface that can be used normally for talking to the Kubernetes API. -// This client doesn't apply changes to the backend. The client gets GET/LIST values from the DryRunGetter implementation. -// This client logs all I/O to the writer w in YAML format -func NewDryRunClientWithOpts(opts DryRunClientOptions) clientset.Interface { - // Build a chain of reactors to act like a normal clientset; but log everything that is happening and don't change any state - client := fakeclientset.NewSimpleClientset() - - // Build the chain of reactors. Order matters; first item here will be invoked first on match, then the second one will be evaluated, etc. - defaultReactorChain := []core.Reactor{ - // Log everything that happens. Default the object if it's about to be created/updated so that the logged object is representative. - &core.SimpleReactor{ - Verb: "*", - Resource: "*", - Reaction: func(action core.Action) (bool, runtime.Object, error) { - logDryRunAction(action, opts.Writer, opts.MarshalFunc) - - return false, nil, nil - }, - }, - // Let the DryRunGetter implementation take care of all GET requests. - // The DryRunGetter implementation may call a real API Server behind the scenes or just fake everything - &core.SimpleReactor{ - Verb: "get", - Resource: "*", - Reaction: func(action core.Action) (bool, runtime.Object, error) { - getAction, ok := action.(core.GetAction) - if !ok { - // something's wrong, we can't handle this event - return true, nil, errors.New("can't cast get reactor event action object to GetAction interface") - } - handled, obj, err := opts.Getter.HandleGetAction(getAction) - - if opts.PrintGETAndLIST { - // Print the marshalled object format with one tab indentation - objBytes, err := opts.MarshalFunc(obj, action.GetResource().GroupVersion()) - if err == nil { - fmt.Println("[dryrun] Returning faked GET response:") - PrintBytesWithLinePrefix(opts.Writer, objBytes, "\t") - } - } - - return handled, obj, err - }, - }, - // Let the DryRunGetter implementation take care of all GET requests. - // The DryRunGetter implementation may call a real API Server behind the scenes or just fake everything - &core.SimpleReactor{ - Verb: "list", - Resource: "*", - Reaction: func(action core.Action) (bool, runtime.Object, error) { - listAction, ok := action.(core.ListAction) - if !ok { - // something's wrong, we can't handle this event - return true, nil, errors.New("can't cast list reactor event action object to ListAction interface") - } - handled, objs, err := opts.Getter.HandleListAction(listAction) - - if opts.PrintGETAndLIST { - // Print the marshalled object format with one tab indentation - objBytes, err := opts.MarshalFunc(objs, action.GetResource().GroupVersion()) - if err == nil { - fmt.Println("[dryrun] Returning faked LIST response:") - PrintBytesWithLinePrefix(opts.Writer, objBytes, "\t") - } - } - - return handled, objs, err - }, - }, - // For the verbs that modify anything on the server; just return the object if present and exit successfully - &core.SimpleReactor{ - Verb: "create", - Resource: "*", - Reaction: successfulModificationReactorFunc, - }, - &core.SimpleReactor{ - Verb: "update", - Resource: "*", - Reaction: successfulModificationReactorFunc, - }, - &core.SimpleReactor{ - Verb: "delete", - Resource: "*", - Reaction: successfulModificationReactorFunc, - }, - &core.SimpleReactor{ - Verb: "delete-collection", - Resource: "*", - Reaction: successfulModificationReactorFunc, - }, - &core.SimpleReactor{ - Verb: "patch", - Resource: "*", - Reaction: successfulModificationReactorFunc, - }, - } - - // The chain of reactors will look like this: - // opts.PrependReactors | defaultReactorChain | opts.AppendReactors | client.Fake.ReactionChain (default reactors for the fake clientset) - fullReactorChain := append(opts.PrependReactors, defaultReactorChain...) - fullReactorChain = append(fullReactorChain, opts.AppendReactors...) - - // Prepend the reaction chain with our reactors. Important, these MUST be prepended; not appended due to how the fake clientset works by default - client.Fake.ReactionChain = append(fullReactorChain, client.Fake.ReactionChain...) - return client -} - -// successfulModificationReactorFunc is a no-op that just returns the POSTed/PUTed value if present; but does nothing to edit any backing data store. -func successfulModificationReactorFunc(action core.Action) (bool, runtime.Object, error) { - objAction, ok := action.(actionWithObject) - if ok { - return true, objAction.GetObject(), nil - } - return true, nil, nil -} - -// logDryRunAction logs the action that was recorded by the "catch-all" (*,*) reactor and tells the user what would have happened in an user-friendly way -func logDryRunAction(action core.Action, w io.Writer, marshalFunc MarshalFunc) { - - group := action.GetResource().Group - if len(group) == 0 { - group = "core" - } - fmt.Fprintf(w, "[dryrun] Would perform action %s on resource %q in API group \"%s/%s\"\n", strings.ToUpper(action.GetVerb()), action.GetResource().Resource, group, action.GetResource().Version) - - namedAction, ok := action.(actionWithName) - if ok { - fmt.Fprintf(w, "[dryrun] Resource name: %q\n", namedAction.GetName()) - } - - objAction, ok := action.(actionWithObject) - if ok && objAction.GetObject() != nil { - // Print the marshalled object with a tab indentation - objBytes, err := marshalFunc(objAction.GetObject(), action.GetResource().GroupVersion()) - if err == nil { - fmt.Println("[dryrun] Attached object:") - PrintBytesWithLinePrefix(w, objBytes, "\t") - } - } - - patchAction, ok := action.(core.PatchAction) - if ok { - // Replace all occurrences of \" with a simple " when printing - fmt.Fprintf(w, "[dryrun] Attached patch:\n\t%s\n", strings.Replace(string(patchAction.GetPatch()), `\"`, `"`, -1)) - } -} - -// PrintBytesWithLinePrefix prints objBytes to writer w with linePrefix in the beginning of every line -func PrintBytesWithLinePrefix(w io.Writer, objBytes []byte, linePrefix string) { - scanner := bufio.NewScanner(bytes.NewReader(objBytes)) - for scanner.Scan() { - fmt.Fprintf(w, "%s%s\n", linePrefix, scanner.Text()) - } -} diff --git a/cmd/kubeadm/app/util/apiclient/dryrunclient_test.go b/cmd/kubeadm/app/util/apiclient/dryrunclient_test.go deleted file mode 100644 index 7ef41c175850e..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/dryrunclient_test.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "bytes" - "testing" - - "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - core "k8s.io/client-go/testing" -) - -func TestLogDryRunAction(t *testing.T) { - var tests = []struct { - name string - action core.Action - expectedBytes []byte - buf *bytes.Buffer - }{ - { - name: "action GET on services", - action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", "kubernetes"), - expectedBytes: []byte(`[dryrun] Would perform action GET on resource "services" in API group "core/v1" -[dryrun] Resource name: "kubernetes" -`), - }, - { - name: "action GET on clusterrolebindings", - action: core.NewRootGetAction(schema.GroupVersionResource{Group: rbac.GroupName, Version: rbac.SchemeGroupVersion.Version, Resource: "clusterrolebindings"}, "system:node"), - expectedBytes: []byte(`[dryrun] Would perform action GET on resource "clusterrolebindings" in API group "rbac.authorization.k8s.io/v1" -[dryrun] Resource name: "system:node" -`), - }, - { - name: "action LIST on services", - action: core.NewListAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, schema.GroupVersionKind{Version: "v1", Kind: "Service"}, "default", metav1.ListOptions{}), - expectedBytes: []byte(`[dryrun] Would perform action LIST on resource "services" in API group "core/v1" -`), - }, - { - name: "action CREATE on services", - action: core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }, - Spec: v1.ServiceSpec{ - ClusterIP: "1.1.1.1", - }, - }), - expectedBytes: []byte(`[dryrun] Would perform action CREATE on resource "services" in API group "core/v1" - apiVersion: v1 - kind: Service - metadata: - creationTimestamp: null - name: foo - spec: - clusterIP: 1.1.1.1 - status: - loadBalancer: {} -`), - }, - { - name: "action PATCH on nodes", - action: core.NewPatchAction(schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, "default", "my-node", "application/strategic-merge-patch+json", []byte(`{"spec":{"taints":[{"key": "foo", "value": "bar"}]}}`)), - expectedBytes: []byte(`[dryrun] Would perform action PATCH on resource "nodes" in API group "core/v1" -[dryrun] Resource name: "my-node" -[dryrun] Attached patch: - {"spec":{"taints":[{"key": "foo", "value": "bar"}]}} -`), - }, - { - name: "action DELETE on pods", - action: core.NewDeleteAction(schema.GroupVersionResource{Version: "v1", Resource: "pods"}, "default", "my-pod"), - expectedBytes: []byte(`[dryrun] Would perform action DELETE on resource "pods" in API group "core/v1" -[dryrun] Resource name: "my-pod" -`), - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - rt.buf = bytes.NewBufferString("") - logDryRunAction(rt.action, rt.buf, DefaultMarshalFunc) - actualBytes := rt.buf.Bytes() - - if !bytes.Equal(actualBytes, rt.expectedBytes) { - t.Errorf( - "failed LogDryRunAction:\n\texpected bytes: %q\n\t actual: %q", - rt.expectedBytes, - actualBytes, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/apiclient/idempotency.go b/cmd/kubeadm/app/util/apiclient/idempotency.go deleted file mode 100644 index 208028ac7d134..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/idempotency.go +++ /dev/null @@ -1,355 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - - apps "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - rbac "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - clientsetretry "k8s.io/client-go/util/retry" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// ConfigMapMutator is a function that mutates the given ConfigMap and optionally returns an error -type ConfigMapMutator func(*v1.ConfigMap) error - -// TODO: We should invent a dynamic mechanism for this using the dynamic client instead of hard-coding these functions per-type - -// CreateOrUpdateConfigMap creates a ConfigMap if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateConfigMap(client clientset.Interface, cm *v1.ConfigMap) error { - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create ConfigMap") - } - - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update ConfigMap") - } - } - return nil -} - -// CreateOrMutateConfigMap tries to create the ConfigMap provided as cm. If the resource exists already, the latest version will be fetched from -// the cluster and mutator callback will be called on it, then an Update of the mutated ConfigMap will be performed. This function is resilient -// to conflicts, and a retry will be issued if the ConfigMap was modified on the server between the refresh and the update (while the mutation was -// taking place) -func CreateOrMutateConfigMap(client clientset.Interface, cm *v1.ConfigMap, mutator ConfigMapMutator) error { - var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - lastError = err - if apierrors.IsAlreadyExists(err) { - lastError = MutateConfigMap(client, metav1.ObjectMeta{Namespace: cm.ObjectMeta.Namespace, Name: cm.ObjectMeta.Name}, mutator) - return lastError == nil, nil - } - return false, nil - } - return true, nil - }) - if err == nil { - return nil - } - return lastError -} - -// MutateConfigMap takes a ConfigMap Object Meta (namespace and name), retrieves the resource from the server and tries to mutate it -// by calling to the mutator callback, then an Update of the mutated ConfigMap will be performed. This function is resilient -// to conflicts, and a retry will be issued if the ConfigMap was modified on the server between the refresh and the update (while the mutation was -// taking place). -func MutateConfigMap(client clientset.Interface, meta metav1.ObjectMeta, mutator ConfigMapMutator) error { - var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { - configMap, err := client.CoreV1().ConfigMaps(meta.Namespace).Get(context.TODO(), meta.Name, metav1.GetOptions{}) - if err != nil { - lastError = err - return false, nil - } - if err = mutator(configMap); err != nil { - lastError = errors.Wrap(err, "unable to mutate ConfigMap") - return false, nil - } - _, lastError = client.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) - return lastError == nil, nil - }) - if err == nil { - return nil - } - return lastError -} - -// CreateOrRetainConfigMap creates a ConfigMap if the target resource doesn't exist. If the resource exists already, this function will retain the resource instead. -func CreateOrRetainConfigMap(client clientset.Interface, cm *v1.ConfigMap, configMapName string) error { - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}); err != nil { - if !apierrors.IsNotFound(err) { - return nil - } - if _, err := client.CoreV1().ConfigMaps(cm.ObjectMeta.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create ConfigMap") - } - } - } - return nil -} - -// CreateOrUpdateSecret creates a Secret if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateSecret(client clientset.Interface, secret *v1.Secret) error { - if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create secret") - } - - if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update secret") - } - } - return nil -} - -// CreateOrUpdateServiceAccount creates a ServiceAccount if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateServiceAccount(client clientset.Interface, sa *v1.ServiceAccount) error { - if _, err := client.CoreV1().ServiceAccounts(sa.ObjectMeta.Namespace).Create(context.TODO(), sa, metav1.CreateOptions{}); err != nil { - // Note: We don't run .Update here afterwards as that's probably not required - // Only thing that could be updated is annotations/labels in .metadata, but we don't use that currently - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create serviceaccount") - } - } - return nil -} - -// CreateOrUpdateDeployment creates a Deployment if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateDeployment(client clientset.Interface, deploy *apps.Deployment) error { - if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Create(context.TODO(), deploy, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create deployment") - } - - if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update deployment") - } - } - return nil -} - -// CreateOrRetainDeployment creates a Deployment if the target resource doesn't exist. If the resource exists already, this function will retain the resource instead. -func CreateOrRetainDeployment(client clientset.Interface, deploy *apps.Deployment, deployName string) error { - if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Get(context.TODO(), deployName, metav1.GetOptions{}); err != nil { - if !apierrors.IsNotFound(err) { - return nil - } - if _, err := client.AppsV1().Deployments(deploy.ObjectMeta.Namespace).Create(context.TODO(), deploy, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create deployment") - } - } - } - return nil -} - -// CreateOrUpdateDaemonSet creates a DaemonSet if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateDaemonSet(client clientset.Interface, ds *apps.DaemonSet) error { - if _, err := client.AppsV1().DaemonSets(ds.ObjectMeta.Namespace).Create(context.TODO(), ds, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create daemonset") - } - - if _, err := client.AppsV1().DaemonSets(ds.ObjectMeta.Namespace).Update(context.TODO(), ds, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update daemonset") - } - } - return nil -} - -// DeleteDaemonSetForeground deletes the specified DaemonSet in foreground mode; i.e. it blocks until/makes sure all the managed Pods are deleted -func DeleteDaemonSetForeground(client clientset.Interface, namespace, name string) error { - foregroundDelete := metav1.DeletePropagationForeground - return client.AppsV1().DaemonSets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{PropagationPolicy: &foregroundDelete}) -} - -// DeleteDeploymentForeground deletes the specified Deployment in foreground mode; i.e. it blocks until/makes sure all the managed Pods are deleted -func DeleteDeploymentForeground(client clientset.Interface, namespace, name string) error { - foregroundDelete := metav1.DeletePropagationForeground - return client.AppsV1().Deployments(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{PropagationPolicy: &foregroundDelete}) -} - -// CreateOrUpdateRole creates a Role if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateRole(client clientset.Interface, role *rbac.Role) error { - var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { - if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Create(context.TODO(), role, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - lastError = errors.Wrap(err, "unable to create RBAC role") - return false, nil - } - - if _, err := client.RbacV1().Roles(role.ObjectMeta.Namespace).Update(context.TODO(), role, metav1.UpdateOptions{}); err != nil { - lastError = errors.Wrap(err, "unable to update RBAC role") - return false, nil - } - } - return true, nil - }) - if err == nil { - return nil - } - return lastError -} - -// CreateOrUpdateRoleBinding creates a RoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateRoleBinding(client clientset.Interface, roleBinding *rbac.RoleBinding) error { - var lastError error - err := wait.PollImmediate(constants.APICallRetryInterval, constants.APICallWithWriteTimeout, func() (bool, error) { - if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - lastError = errors.Wrap(err, "unable to create RBAC rolebinding") - return false, nil - } - - if _, err := client.RbacV1().RoleBindings(roleBinding.ObjectMeta.Namespace).Update(context.TODO(), roleBinding, metav1.UpdateOptions{}); err != nil { - lastError = errors.Wrap(err, "unable to update RBAC rolebinding") - return false, nil - } - } - return true, nil - }) - if err == nil { - return nil - } - return lastError -} - -// CreateOrUpdateClusterRole creates a ClusterRole if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateClusterRole(client clientset.Interface, clusterRole *rbac.ClusterRole) error { - if _, err := client.RbacV1().ClusterRoles().Create(context.TODO(), clusterRole, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create RBAC clusterrole") - } - - if _, err := client.RbacV1().ClusterRoles().Update(context.TODO(), clusterRole, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update RBAC clusterrole") - } - } - return nil -} - -// CreateOrUpdateClusterRoleBinding creates a ClusterRoleBinding if the target resource doesn't exist. If the resource exists already, this function will update the resource instead. -func CreateOrUpdateClusterRoleBinding(client clientset.Interface, clusterRoleBinding *rbac.ClusterRoleBinding) error { - if _, err := client.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterRoleBinding, metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return errors.Wrap(err, "unable to create RBAC clusterrolebinding") - } - - if _, err := client.RbacV1().ClusterRoleBindings().Update(context.TODO(), clusterRoleBinding, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "unable to update RBAC clusterrolebinding") - } - } - return nil -} - -// PatchNodeOnce executes patchFn on the node object found by the node name. -// This is a condition function meant to be used with wait.Poll. false, nil -// implies it is safe to try again, an error indicates no more tries should be -// made and true indicates success. -func PatchNodeOnce(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) func() (bool, error) { - return func() (bool, error) { - // First get the node object - n, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil { - // TODO this should only be for timeouts - return false, nil - } - - // The node may appear to have no labels at first, - // so we wait for it to get hostname label. - if _, found := n.ObjectMeta.Labels[v1.LabelHostname]; !found { - return false, nil - } - - oldData, err := json.Marshal(n) - if err != nil { - return false, errors.Wrapf(err, "failed to marshal unmodified node %q into JSON", n.Name) - } - - // Execute the mutating function - patchFn(n) - - newData, err := json.Marshal(n) - if err != nil { - return false, errors.Wrapf(err, "failed to marshal modified node %q into JSON", n.Name) - } - - patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{}) - if err != nil { - return false, errors.Wrap(err, "failed to create two way merge patch") - } - - if _, err := client.CoreV1().Nodes().Patch(context.TODO(), n.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil { - // TODO also check for timeouts - if apierrors.IsConflict(err) { - fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)") - return false, nil - } - return false, errors.Wrapf(err, "error patching node %q through apiserver", n.Name) - } - - return true, nil - } -} - -// PatchNode tries to patch a node using patchFn for the actual mutating logic. -// Retries are provided by the wait package. -func PatchNode(client clientset.Interface, nodeName string, patchFn func(*v1.Node)) error { - // wait.Poll will rerun the condition function every interval function if - // the function returns false. If the condition function returns an error - // then the retries end and the error is returned. - return wait.Poll(constants.APICallRetryInterval, constants.PatchNodeTimeout, PatchNodeOnce(client, nodeName, patchFn)) -} - -// GetConfigMapWithRetry tries to retrieve a ConfigMap using the given client, -// retrying if we get an unexpected error. -// -// TODO: evaluate if this can be done better. Potentially remove the retry if feasible. -func GetConfigMapWithRetry(client clientset.Interface, namespace, name string) (*v1.ConfigMap, error) { - var cm *v1.ConfigMap - var lastError error - err := wait.ExponentialBackoff(clientsetretry.DefaultBackoff, func() (bool, error) { - var err error - cm, err = client.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err == nil { - return true, nil - } - lastError = err - return false, nil - }) - if err == nil { - return cm, nil - } - return nil, lastError -} diff --git a/cmd/kubeadm/app/util/apiclient/idempotency_test.go b/cmd/kubeadm/app/util/apiclient/idempotency_test.go deleted file mode 100644 index 13879cf0edd7f..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/idempotency_test.go +++ /dev/null @@ -1,182 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "context" - "testing" - - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" - core "k8s.io/client-go/testing" -) - -const configMapName = "configmap" - -func TestPatchNodeNonErrorCases(t *testing.T) { - testcases := []struct { - name string - lookupName string - node v1.Node - success bool - }{ - { - name: "simple update", - lookupName: "testnode", - node: v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testnode", - Labels: map[string]string{v1.LabelHostname: ""}, - }, - }, - success: true, - }, - { - name: "node does not exist", - lookupName: "whale", - success: false, - }, - { - name: "node not labelled yet", - lookupName: "robin", - node: v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "robin", - }, - }, - success: false, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset() - _, err := client.CoreV1().Nodes().Create(context.TODO(), &tc.node, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create node to fake client: %v", err) - } - conditionFunction := PatchNodeOnce(client, tc.lookupName, func(node *v1.Node) { - node.Annotations = map[string]string{ - "updatedBy": "test", - } - }) - success, err := conditionFunction() - if err != nil { - t.Fatalf("did not expect error: %v", err) - } - if success != tc.success { - t.Fatalf("expected %v got %v", tc.success, success) - } - }) - } -} - -func TestCreateOrMutateConfigMap(t *testing.T) { - client := fake.NewSimpleClientset() - err := CreateOrMutateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - "key": "some-value", - }, - }, func(cm *v1.ConfigMap) error { - t.Fatal("mutate should not have been called, since the ConfigMap should have been created instead of mutated") - return nil - }) - if err != nil { - t.Fatalf("error creating ConfigMap: %v", err) - } - _, err = client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if err != nil { - t.Fatalf("error retrieving ConfigMap: %v", err) - } -} - -func createClientAndConfigMap(t *testing.T) *fake.Clientset { - client := fake.NewSimpleClientset() - _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(context.TODO(), &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, - Data: map[string]string{ - "key": "some-value", - }, - }, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("error creating ConfigMap: %v", err) - } - return client -} - -func TestMutateConfigMap(t *testing.T) { - client := createClientAndConfigMap(t) - - err := MutateConfigMap(client, metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, func(cm *v1.ConfigMap) error { - cm.Data["key"] = "some-other-value" - return nil - }) - if err != nil { - t.Fatalf("error mutating regular ConfigMap: %v", err) - } - - cm, _ := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if cm.Data["key"] != "some-other-value" { - t.Fatalf("ConfigMap mutation was invalid, has: %q", cm.Data["key"]) - } -} - -func TestMutateConfigMapWithConflict(t *testing.T) { - client := createClientAndConfigMap(t) - - // Mimic that the first 5 updates of the ConfigMap returns a conflict, whereas the sixth update - // succeeds - conflict := 5 - client.PrependReactor("update", "configmaps", func(action core.Action) (bool, runtime.Object, error) { - update := action.(core.UpdateAction) - if conflict > 0 { - conflict-- - return true, update.GetObject(), apierrors.NewConflict(action.GetResource().GroupResource(), configMapName, errors.New("conflict")) - } - return false, update.GetObject(), nil - }) - - err := MutateConfigMap(client, metav1.ObjectMeta{ - Name: configMapName, - Namespace: metav1.NamespaceSystem, - }, func(cm *v1.ConfigMap) error { - cm.Data["key"] = "some-other-value" - return nil - }) - if err != nil { - t.Fatalf("error mutating conflicting ConfigMap: %v", err) - } - - cm, _ := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(context.TODO(), configMapName, metav1.GetOptions{}) - if cm.Data["key"] != "some-other-value" { - t.Fatalf("ConfigMap mutation with conflict was invalid, has: %q", cm.Data["key"]) - } -} diff --git a/cmd/kubeadm/app/util/apiclient/init_dryrun.go b/cmd/kubeadm/app/util/apiclient/init_dryrun.go deleted file mode 100644 index c8ce32eb5e0fe..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/init_dryrun.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "net" - "strings" - - "github.com/pkg/errors" - - "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" - core "k8s.io/client-go/testing" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - utilnet "k8s.io/utils/net" -) - -// InitDryRunGetter implements the DryRunGetter interface and can be used to GET/LIST values in the dryrun fake clientset -// Need to handle these routes in a special manner: -// - GET /default/services/kubernetes -- must return a valid Service -// - GET /clusterrolebindings/system:nodes -- can safely return a NotFound error -// - GET /kube-system/secrets/bootstrap-token-* -- can safely return a NotFound error -// - GET /nodes/ -- must return a valid Node -// - ...all other, unknown GETs/LISTs will be logged -type InitDryRunGetter struct { - controlPlaneName string - serviceSubnet string -} - -// InitDryRunGetter should implement the DryRunGetter interface -var _ DryRunGetter = &InitDryRunGetter{} - -// NewInitDryRunGetter creates a new instance of the InitDryRunGetter struct -func NewInitDryRunGetter(controlPlaneName string, serviceSubnet string) *InitDryRunGetter { - return &InitDryRunGetter{ - controlPlaneName: controlPlaneName, - serviceSubnet: serviceSubnet, - } -} - -// HandleGetAction handles GET actions to the dryrun clientset this interface supports -func (idr *InitDryRunGetter) HandleGetAction(action core.GetAction) (bool, runtime.Object, error) { - funcs := []func(core.GetAction) (bool, runtime.Object, error){ - idr.handleKubernetesService, - idr.handleGetNode, - idr.handleSystemNodesClusterRoleBinding, - idr.handleGetBootstrapToken, - } - for _, f := range funcs { - handled, obj, err := f(action) - if handled { - return handled, obj, err - } - } - - return false, nil, nil -} - -// HandleListAction handles GET actions to the dryrun clientset this interface supports. -// Currently there are no known LIST calls during kubeadm init this code has to take care of. -func (idr *InitDryRunGetter) HandleListAction(action core.ListAction) (bool, runtime.Object, error) { - return false, nil, nil -} - -// handleKubernetesService returns a faked Kubernetes service in order to be able to continue running kubeadm init. -// The kube-dns addon code GETs the Kubernetes service in order to extract the service subnet -func (idr *InitDryRunGetter) handleKubernetesService(action core.GetAction) (bool, runtime.Object, error) { - if action.GetName() != "kubernetes" || action.GetNamespace() != metav1.NamespaceDefault || action.GetResource().Resource != "services" { - // We can't handle this event - return false, nil, nil - } - - _, svcSubnet, err := net.ParseCIDR(idr.serviceSubnet) - if err != nil { - return true, nil, errors.Wrapf(err, "error parsing CIDR %q", idr.serviceSubnet) - } - - internalAPIServerVirtualIP, err := utilnet.GetIndexedIP(svcSubnet, 1) - if err != nil { - return true, nil, errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", svcSubnet.String()) - } - - // The only used field of this Service object is the ClusterIP, which kube-dns uses to calculate its own IP - return true, &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubernetes", - Namespace: metav1.NamespaceDefault, - Labels: map[string]string{ - "component": "apiserver", - "provider": "kubernetes", - }, - }, - Spec: v1.ServiceSpec{ - ClusterIP: internalAPIServerVirtualIP.String(), - Ports: []v1.ServicePort{ - { - Name: "https", - Port: 443, - TargetPort: intstr.FromInt(6443), - }, - }, - }, - }, nil -} - -// handleGetNode returns a fake node object for the purpose of moving kubeadm init forwards. -func (idr *InitDryRunGetter) handleGetNode(action core.GetAction) (bool, runtime.Object, error) { - if action.GetName() != idr.controlPlaneName || action.GetResource().Resource != "nodes" { - // We can't handle this event - return false, nil, nil - } - - return true, &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: idr.controlPlaneName, - Labels: map[string]string{ - "kubernetes.io/hostname": idr.controlPlaneName, - }, - Annotations: map[string]string{}, - }, - }, nil -} - -// handleSystemNodesClusterRoleBinding handles the GET call to the system:nodes clusterrolebinding -func (idr *InitDryRunGetter) handleSystemNodesClusterRoleBinding(action core.GetAction) (bool, runtime.Object, error) { - if action.GetName() != constants.NodesClusterRoleBinding || action.GetResource().Resource != "clusterrolebindings" { - // We can't handle this event - return false, nil, nil - } - // We can safely return a NotFound error here as the code will just proceed normally and don't care about modifying this clusterrolebinding - // This can only happen on an upgrade; and in that case the ClientBackedDryRunGetter impl will be used - return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "clusterrolebinding not found") -} - -// handleGetBootstrapToken handles the case where kubeadm init creates the default token; and the token code GETs the -// bootstrap token secret first in order to check if it already exists -func (idr *InitDryRunGetter) handleGetBootstrapToken(action core.GetAction) (bool, runtime.Object, error) { - if !strings.HasPrefix(action.GetName(), "bootstrap-token-") || action.GetNamespace() != metav1.NamespaceSystem || action.GetResource().Resource != "secrets" { - // We can't handle this event - return false, nil, nil - } - // We can safely return a NotFound error here as the code will just proceed normally and create the Bootstrap Token - return true, nil, apierrors.NewNotFound(action.GetResource().GroupResource(), "secret not found") -} diff --git a/cmd/kubeadm/app/util/apiclient/init_dryrun_test.go b/cmd/kubeadm/app/util/apiclient/init_dryrun_test.go deleted file mode 100644 index e3393b417cb96..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/init_dryrun_test.go +++ /dev/null @@ -1,137 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "bytes" - "encoding/json" - "testing" - - rbac "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - core "k8s.io/client-go/testing" -) - -func TestHandleGetAction(t *testing.T) { - controlPlaneName := "control-plane-foo" - serviceSubnet := "10.96.0.1/12" - idr := NewInitDryRunGetter(controlPlaneName, serviceSubnet) - - var tests = []struct { - name string - action core.GetActionImpl - expectedHandled bool - expectedObjectJSON []byte - expectedErr bool - }{ - { - name: "get default services", - action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", "kubernetes"), - expectedHandled: true, - expectedObjectJSON: []byte(`{"metadata":{"name":"kubernetes","namespace":"default","creationTimestamp":null,"labels":{"component":"apiserver","provider":"kubernetes"}},"spec":{"ports":[{"name":"https","port":443,"targetPort":6443}],"clusterIP":"10.96.0.1"},"status":{"loadBalancer":{}}}`), - expectedErr: false, - }, - { - name: "get nodes", - action: core.NewRootGetAction(schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, controlPlaneName), - expectedHandled: true, - expectedObjectJSON: []byte(`{"metadata":{"name":"control-plane-foo","creationTimestamp":null,"labels":{"kubernetes.io/hostname":"control-plane-foo"}},"spec":{},"status":{"daemonEndpoints":{"kubeletEndpoint":{"Port":0}},"nodeInfo":{"machineID":"","systemUUID":"","bootID":"","kernelVersion":"","osImage":"","containerRuntimeVersion":"","kubeletVersion":"","kubeProxyVersion":"","operatingSystem":"","architecture":""}}}`), - expectedErr: false, - }, - { - name: "get clusterrolebinings", - action: core.NewRootGetAction(schema.GroupVersionResource{Group: rbac.GroupName, Version: rbac.SchemeGroupVersion.Version, Resource: "clusterrolebindings"}, "system:node"), - expectedHandled: true, - expectedObjectJSON: []byte(``), - expectedErr: true, // we expect a NotFound error here - }, - { - name: "get kube-system secret bootstrap-token-abcdef", - action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, "kube-system", "bootstrap-token-abcdef"), - expectedHandled: true, - expectedObjectJSON: []byte(``), - expectedErr: true, // we expect a NotFound error here - }, - { // an ask for a kubernetes service in the _kube-system_ ns should not be answered - name: "get kube-system services", - action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "kube-system", "kubernetes"), - expectedHandled: false, - expectedObjectJSON: []byte(``), - expectedErr: false, - }, - { // an ask for an other service than kubernetes should not be answered - name: "get default my-other-service", - action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "services"}, "default", "my-other-service"), - expectedHandled: false, - expectedObjectJSON: []byte(``), - expectedErr: false, - }, - { // an ask for an other node than the control-plane should not be answered - name: "get other-node", - action: core.NewRootGetAction(schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, "other-node"), - expectedHandled: false, - expectedObjectJSON: []byte(``), - expectedErr: false, - }, - { // an ask for a secret in any other ns than kube-system should not be answered - name: "get default secret bootstrap-token-abcdef", - action: core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, "default", "bootstrap-token-abcdef"), - expectedHandled: false, - expectedObjectJSON: []byte(``), - expectedErr: false, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - handled, obj, actualErr := idr.HandleGetAction(rt.action) - objBytes := []byte(``) - if obj != nil { - var err error - objBytes, err = json.Marshal(obj) - if err != nil { - t.Fatalf("couldn't marshal returned object") - } - } - - if handled != rt.expectedHandled { - t.Errorf( - "failed HandleGetAction:\n\texpected handled: %t\n\t actual: %t %v", - rt.expectedHandled, - handled, - rt.action, - ) - } - - if !bytes.Equal(objBytes, rt.expectedObjectJSON) { - t.Errorf( - "failed HandleGetAction:\n\texpected object: %q\n\t actual: %q", - rt.expectedObjectJSON, - objBytes, - ) - } - - if (actualErr != nil) != rt.expectedErr { - t.Errorf( - "failed HandleGetAction:\n\texpected error: %t\n\t actual: %t %v", - rt.expectedErr, - (actualErr != nil), - rt.action, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go deleted file mode 100644 index afeaf74fa81d8..0000000000000 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ /dev/null @@ -1,270 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiclient - -import ( - "context" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - netutil "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// Waiter is an interface for waiting for criteria in Kubernetes to happen -type Waiter interface { - // WaitForAPI waits for the API Server's /healthz endpoint to become "ok" - WaitForAPI() error - // WaitForPodsWithLabel waits for Pods in the kube-system namespace to become Ready - WaitForPodsWithLabel(kvLabel string) error - // WaitForPodToDisappear waits for the given Pod in the kube-system namespace to be deleted - WaitForPodToDisappear(staticPodName string) error - // WaitForStaticPodSingleHash fetches sha256 hash for the control plane static pod - WaitForStaticPodSingleHash(nodeName string, component string) (string, error) - // WaitForStaticPodHashChange waits for the given static pod component's static pod hash to get updated. - // By doing that we can be sure that the kubelet has restarted the given Static Pod - WaitForStaticPodHashChange(nodeName, component, previousHash string) error - // WaitForStaticPodControlPlaneHashes fetches sha256 hashes for the control plane static pods - WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error) - // WaitForHealthyKubelet blocks until the kubelet /healthz endpoint returns 'ok' - WaitForHealthyKubelet(initialTimeout time.Duration, healthzEndpoint string) error - // WaitForKubeletAndFunc is a wrapper for WaitForHealthyKubelet that also blocks for a function - WaitForKubeletAndFunc(f func() error) error - // SetTimeout adjusts the timeout to the specified duration - SetTimeout(timeout time.Duration) -} - -// KubeWaiter is an implementation of Waiter that is backed by a Kubernetes client -type KubeWaiter struct { - client clientset.Interface - timeout time.Duration - writer io.Writer -} - -// NewKubeWaiter returns a new Waiter object that talks to the given Kubernetes cluster -func NewKubeWaiter(client clientset.Interface, timeout time.Duration, writer io.Writer) Waiter { - return &KubeWaiter{ - client: client, - timeout: timeout, - writer: writer, - } -} - -// WaitForAPI waits for the API Server's /healthz endpoint to report "ok" -func (w *KubeWaiter) WaitForAPI() error { - start := time.Now() - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - healthStatus := 0 - w.client.Discovery().RESTClient().Get().AbsPath("/healthz").Do(context.TODO()).StatusCode(&healthStatus) - if healthStatus != http.StatusOK { - return false, nil - } - - fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds()) - return true, nil - }) -} - -// WaitForPodsWithLabel will lookup pods with the given label and wait until they are all -// reporting status as running. -func (w *KubeWaiter) WaitForPodsWithLabel(kvLabel string) error { - - lastKnownPodNumber := -1 - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - listOpts := metav1.ListOptions{LabelSelector: kvLabel} - pods, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), listOpts) - if err != nil { - fmt.Fprintf(w.writer, "[apiclient] Error getting Pods with label selector %q [%v]\n", kvLabel, err) - return false, nil - } - - if lastKnownPodNumber != len(pods.Items) { - fmt.Fprintf(w.writer, "[apiclient] Found %d Pods for label selector %s\n", len(pods.Items), kvLabel) - lastKnownPodNumber = len(pods.Items) - } - - if len(pods.Items) == 0 { - return false, nil - } - - for _, pod := range pods.Items { - if pod.Status.Phase != v1.PodRunning { - return false, nil - } - } - - return true, nil - }) -} - -// WaitForPodToDisappear blocks until it timeouts or gets a "NotFound" response from the API Server when getting the Static Pod in question -func (w *KubeWaiter) WaitForPodToDisappear(podName string) error { - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - _, err := w.client.CoreV1().Pods(metav1.NamespaceSystem).Get(context.TODO(), podName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - fmt.Printf("[apiclient] The old Pod %q is now removed (which is desired)\n", podName) - return true, nil - } - return false, nil - }) -} - -// WaitForHealthyKubelet blocks until the kubelet /healthz endpoint returns 'ok' -func (w *KubeWaiter) WaitForHealthyKubelet(initialTimeout time.Duration, healthzEndpoint string) error { - time.Sleep(initialTimeout) - fmt.Printf("[kubelet-check] Initial timeout of %v passed.\n", initialTimeout) - return TryRunCommand(func() error { - client := &http.Client{Transport: netutil.SetOldTransportDefaults(&http.Transport{})} - resp, err := client.Get(healthzEndpoint) - if err != nil { - fmt.Println("[kubelet-check] It seems like the kubelet isn't running or healthy.") - fmt.Printf("[kubelet-check] The HTTP call equal to 'curl -sSL %s' failed with error: %v.\n", healthzEndpoint, err) - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - fmt.Println("[kubelet-check] It seems like the kubelet isn't running or healthy.") - fmt.Printf("[kubelet-check] The HTTP call equal to 'curl -sSL %s' returned HTTP code %d\n", healthzEndpoint, resp.StatusCode) - return errors.New("the kubelet healthz endpoint is unhealthy") - } - return nil - }, 5) // a failureThreshold of five means waiting for a total of 155 seconds -} - -// WaitForKubeletAndFunc waits primarily for the function f to execute, even though it might take some time. If that takes a long time, and the kubelet -// /healthz continuously are unhealthy, kubeadm will error out after a period of exponential backoff -func (w *KubeWaiter) WaitForKubeletAndFunc(f func() error) error { - errorChan := make(chan error, 1) - - go func(errC chan error, waiter Waiter) { - if err := waiter.WaitForHealthyKubelet(40*time.Second, fmt.Sprintf("http://localhost:%d/healthz", kubeadmconstants.KubeletHealthzPort)); err != nil { - errC <- err - } - }(errorChan, w) - - go func(errC chan error, waiter Waiter) { - // This main goroutine sends whatever the f function returns (error or not) to the channel - // This in order to continue on success (nil error), or just fail if the function returns an error - errC <- f() - }(errorChan, w) - - // This call is blocking until one of the goroutines sends to errorChan - return <-errorChan -} - -// SetTimeout adjusts the timeout to the specified duration -func (w *KubeWaiter) SetTimeout(timeout time.Duration) { - w.timeout = timeout -} - -// WaitForStaticPodControlPlaneHashes blocks until it timeouts or gets a hash map for all components and their Static Pods -func (w *KubeWaiter) WaitForStaticPodControlPlaneHashes(nodeName string) (map[string]string, error) { - - componentHash := "" - var err error - mirrorPodHashes := map[string]string{} - for _, component := range kubeadmconstants.ControlPlaneComponents { - err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - componentHash, err = getStaticPodSingleHash(w.client, nodeName, component) - if err != nil { - return false, nil - } - return true, nil - }) - if err != nil { - return nil, err - } - mirrorPodHashes[component] = componentHash - } - - return mirrorPodHashes, nil -} - -// WaitForStaticPodSingleHash blocks until it timeouts or gets a hash for a single component and its Static Pod -func (w *KubeWaiter) WaitForStaticPodSingleHash(nodeName string, component string) (string, error) { - - componentPodHash := "" - var err error - err = wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - componentPodHash, err = getStaticPodSingleHash(w.client, nodeName, component) - if err != nil { - return false, nil - } - return true, nil - }) - - return componentPodHash, err -} - -// WaitForStaticPodHashChange blocks until it timeouts or notices that the Mirror Pod (for the Static Pod, respectively) has changed -// This implicitly means this function blocks until the kubelet has restarted the Static Pod in question -func (w *KubeWaiter) WaitForStaticPodHashChange(nodeName, component, previousHash string) error { - return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, w.timeout, func() (bool, error) { - - hash, err := getStaticPodSingleHash(w.client, nodeName, component) - if err != nil { - return false, nil - } - // We should continue polling until the UID changes - if hash == previousHash { - return false, nil - } - - return true, nil - }) -} - -// getStaticPodSingleHash computes hashes for a single Static Pod resource -func getStaticPodSingleHash(client clientset.Interface, nodeName string, component string) (string, error) { - - staticPodName := fmt.Sprintf("%s-%s", component, nodeName) - staticPod, err := client.CoreV1().Pods(metav1.NamespaceSystem).Get(context.TODO(), staticPodName, metav1.GetOptions{}) - if err != nil { - return "", err - } - - staticPodHash := staticPod.Annotations["kubernetes.io/config.hash"] - fmt.Printf("Static pod: %s hash: %s\n", staticPodName, staticPodHash) - return staticPodHash, nil -} - -// TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned -func TryRunCommand(f func() error, failureThreshold int) error { - backoff := wait.Backoff{ - Duration: 5 * time.Second, - Factor: 2, // double the timeout for every failure - Steps: failureThreshold, - } - return wait.ExponentialBackoff(backoff, func() (bool, error) { - err := f() - if err != nil { - // Retry until the timeout - return false, nil - } - // The last f() call was a success, return cleanly - return true, nil - }) -} diff --git a/cmd/kubeadm/app/util/arguments.go b/cmd/kubeadm/app/util/arguments.go deleted file mode 100644 index 7f88b471a2227..0000000000000 --- a/cmd/kubeadm/app/util/arguments.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "sort" - "strings" - - "github.com/pkg/errors" -) - -// BuildArgumentListFromMap takes two string-string maps, one with the base arguments and one -// with optional override arguments. In the return list override arguments will precede base -// arguments -func BuildArgumentListFromMap(baseArguments map[string]string, overrideArguments map[string]string) []string { - var command []string - var keys []string - - argsMap := make(map[string]string) - - for k, v := range baseArguments { - argsMap[k] = v - } - - for k, v := range overrideArguments { - argsMap[k] = v - } - - for k := range argsMap { - keys = append(keys, k) - } - - sort.Strings(keys) - for _, k := range keys { - command = append(command, fmt.Sprintf("--%s=%s", k, argsMap[k])) - } - - return command -} - -// ParseArgumentListToMap parses a CLI argument list in the form "--foo=bar" to a string-string map -func ParseArgumentListToMap(arguments []string) map[string]string { - resultingMap := map[string]string{} - for i, arg := range arguments { - key, val, err := parseArgument(arg) - - // Ignore if the first argument doesn't satisfy the criteria, it's most often the binary name - // Warn in all other cases, but don't error out. This can happen only if the user has edited the argument list by hand, so they might know what they are doing - if err != nil { - if i != 0 { - fmt.Printf("[kubeadm] WARNING: The component argument %q could not be parsed correctly. The argument must be of the form %q. Skipping...\n", arg, "--") - } - continue - } - - resultingMap[key] = val - } - return resultingMap -} - -// ReplaceArgument gets a command list; converts it to a map for easier modification, runs the provided function that -// returns a new modified map, and then converts the map back to a command string slice -func ReplaceArgument(command []string, argMutateFunc func(map[string]string) map[string]string) []string { - argMap := ParseArgumentListToMap(command) - - // Save the first command (the executable) if we're sure it's not an argument (i.e. no --) - var newCommand []string - if len(command) > 0 && !strings.HasPrefix(command[0], "--") { - newCommand = append(newCommand, command[0]) - } - newArgMap := argMutateFunc(argMap) - newCommand = append(newCommand, BuildArgumentListFromMap(newArgMap, map[string]string{})...) - return newCommand -} - -// parseArgument parses the argument "--foo=bar" to "foo" and "bar" -func parseArgument(arg string) (string, string, error) { - if !strings.HasPrefix(arg, "--") { - return "", "", errors.New("the argument should start with '--'") - } - if !strings.Contains(arg, "=") { - return "", "", errors.New("the argument should have a '=' between the flag and the value") - } - // Remove the starting -- - arg = strings.TrimPrefix(arg, "--") - // Split the string on =. Return only two substrings, since we want only key/value, but the value can include '=' as well - keyvalSlice := strings.SplitN(arg, "=", 2) - - // Make sure both a key and value is present - if len(keyvalSlice) != 2 { - return "", "", errors.New("the argument must have both a key and a value") - } - if len(keyvalSlice[0]) == 0 { - return "", "", errors.New("the argument must have a key") - } - - return keyvalSlice[0], keyvalSlice[1], nil -} diff --git a/cmd/kubeadm/app/util/arguments_test.go b/cmd/kubeadm/app/util/arguments_test.go deleted file mode 100644 index 48c93318e9e44..0000000000000 --- a/cmd/kubeadm/app/util/arguments_test.go +++ /dev/null @@ -1,358 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "reflect" - "sort" - "testing" -) - -func TestBuildArgumentListFromMap(t *testing.T) { - var tests = []struct { - name string - base map[string]string - overrides map[string]string - expected []string - }{ - { - name: "override an argument from the base", - base: map[string]string{ - "admission-control": "NamespaceLifecycle", - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - "--insecure-bind-address=127.0.0.1", - }, - }, - { - name: "add an argument that is not in base", - base: map[string]string{ - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - "--insecure-bind-address=127.0.0.1", - }, - }, - { - name: "allow empty strings in base", - base: map[string]string{ - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - "something-that-allows-empty-string": "", - }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - "--insecure-bind-address=127.0.0.1", - "--something-that-allows-empty-string=", - }, - }, - { - name: "allow empty strings in overrides", - base: map[string]string{ - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - "something-that-allows-empty-string": "foo", - }, - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "something-that-allows-empty-string": "", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--allow-privileged=true", - "--insecure-bind-address=127.0.0.1", - "--something-that-allows-empty-string=", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := BuildArgumentListFromMap(rt.base, rt.overrides) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf("failed BuildArgumentListFromMap:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) - } - }) - } -} - -func TestParseArgumentListToMap(t *testing.T) { - var tests = []struct { - name string - args []string - expectedMap map[string]string - }{ - { - name: "normal case", - args: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - expectedMap: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - }, - { - name: "test that feature-gates is working", - args: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - "--feature-gates=EnableFoo=true,EnableBar=false", - }, - expectedMap: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - "feature-gates": "EnableFoo=true,EnableBar=false", - }, - }, - { - name: "test that a binary can be the first arg", - args: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - "--feature-gates=EnableFoo=true,EnableBar=false", - }, - expectedMap: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - "feature-gates": "EnableFoo=true,EnableBar=false", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualMap := ParseArgumentListToMap(rt.args) - if !reflect.DeepEqual(actualMap, rt.expectedMap) { - t.Errorf("failed ParseArgumentListToMap:\nexpected:\n%v\nsaw:\n%v", rt.expectedMap, actualMap) - } - }) - } -} - -func TestReplaceArgument(t *testing.T) { - var tests = []struct { - name string - args []string - mutateFunc func(map[string]string) map[string]string - expectedArgs []string - }{ - { - name: "normal case", - args: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - mutateFunc: func(argMap map[string]string) map[string]string { - argMap["admission-control"] = "NamespaceLifecycle,LimitRanger,ResourceQuota" - return argMap - }, - expectedArgs: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger,ResourceQuota", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - }, - { - name: "another normal case", - args: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - mutateFunc: func(argMap map[string]string) map[string]string { - argMap["new-arg-here"] = "foo" - return argMap - }, - expectedArgs: []string{ - "kube-apiserver", - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - "--new-arg-here=foo", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualArgs := ReplaceArgument(rt.args, rt.mutateFunc) - sort.Strings(actualArgs) - sort.Strings(rt.expectedArgs) - if !reflect.DeepEqual(actualArgs, rt.expectedArgs) { - t.Errorf("failed ReplaceArgument:\nexpected:\n%v\nsaw:\n%v", rt.expectedArgs, actualArgs) - } - }) - } -} - -func TestRoundtrip(t *testing.T) { - var tests = []struct { - name string - args []string - }{ - { - name: "normal case", - args: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - }, - { - name: "test that feature-gates is working", - args: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - "--feature-gates=EnableFoo=true,EnableBar=false", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - // These two methods should be each other's opposite functions, test that by chaining the methods and see if you get the same result back - actual := BuildArgumentListFromMap(ParseArgumentListToMap(rt.args), map[string]string{}) - sort.Strings(actual) - sort.Strings(rt.args) - - if !reflect.DeepEqual(actual, rt.args) { - t.Errorf("failed TestRoundtrip:\nexpected:\n%v\nsaw:\n%v", rt.args, actual) - } - }) - } -} - -func TestParseArgument(t *testing.T) { - var tests = []struct { - name string - arg string - expectedKey string - expectedVal string - expectedErr bool - }{ - { - name: "arg cannot be empty", - arg: "", - expectedErr: true, - }, - { - name: "arg must contain -- and =", - arg: "a", - expectedErr: true, - }, - { - name: "arg must contain -- and =", - arg: "a-z", - expectedErr: true, - }, - { - name: "arg must contain --", - arg: "a=b", - expectedErr: true, - }, - { - name: "arg must contain a key", - arg: "--=b", - expectedErr: true, - }, - { - name: "arg can contain key but no value", - arg: "--a=", - expectedKey: "a", - expectedVal: "", - expectedErr: false, - }, - { - name: "simple case", - arg: "--a=b", - expectedKey: "a", - expectedVal: "b", - expectedErr: false, - }, - { - name: "keys/values with '-' should be supported", - arg: "--very-long-flag-name=some-value", - expectedKey: "very-long-flag-name", - expectedVal: "some-value", - expectedErr: false, - }, - { - name: "numbers should be handled correctly", - arg: "--some-number=0.2", - expectedKey: "some-number", - expectedVal: "0.2", - expectedErr: false, - }, - { - name: "lists should be handled correctly", - arg: "--admission-control=foo,bar,baz", - expectedKey: "admission-control", - expectedVal: "foo,bar,baz", - expectedErr: false, - }, - { - name: "more than one '=' should be allowed", - arg: "--feature-gates=EnableFoo=true,EnableBar=false", - expectedKey: "feature-gates", - expectedVal: "EnableFoo=true,EnableBar=false", - expectedErr: false, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - key, val, actual := parseArgument(rt.arg) - if (actual != nil) != rt.expectedErr { - t.Errorf("failed parseArgument:\nexpected error:\n%t\nsaw error:\n%v", rt.expectedErr, actual) - } - if (key != rt.expectedKey) || (val != rt.expectedVal) { - t.Errorf("failed parseArgument:\nexpected key: %s\nsaw key: %s\nexpected value: %s\nsaw value: %s", rt.expectedKey, key, rt.expectedVal, val) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/certs/BUILD b/cmd/kubeadm/app/util/certs/BUILD deleted file mode 100644 index d3a08ea29aa56..0000000000000 --- a/cmd/kubeadm/app/util/certs/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["util.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/certs", - deps = [ - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/certs/util.go b/cmd/kubeadm/app/util/certs/util.go deleted file mode 100644 index 51af8543dff7e..0000000000000 --- a/cmd/kubeadm/app/util/certs/util.go +++ /dev/null @@ -1,253 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certs - -import ( - "crypto" - "crypto/rsa" - "crypto/x509" - "net" - "path" - "testing" - - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -// SetupCertificateAuthority is a utility function for kubeadm testing that creates a -// CertificateAuthority cert/key pair -func SetupCertificateAuthority(t *testing.T) (*x509.Certificate, crypto.Signer) { - caCert, caKey, err := pkiutil.NewCertificateAuthority(&pkiutil.CertConfig{ - Config: certutil.Config{CommonName: "kubernetes"}, - }) - if err != nil { - t.Fatalf("failure while generating CA certificate and key: %v", err) - } - - return caCert, caKey -} - -// AssertCertificateIsSignedByCa is a utility function for kubeadm testing that asserts if a given certificate is signed -// by the expected CA -func AssertCertificateIsSignedByCa(t *testing.T, cert *x509.Certificate, signingCa *x509.Certificate) { - if err := cert.CheckSignatureFrom(signingCa); err != nil { - t.Error("cert is not signed by signing CA as expected") - } -} - -// AssertCertificateHasCommonName is a utility function for kubeadm testing that asserts if a given certificate has -// the expected SubjectCommonName -func AssertCertificateHasCommonName(t *testing.T, cert *x509.Certificate, commonName string) { - if cert.Subject.CommonName != commonName { - t.Errorf("cert has Subject.CommonName %s, expected %s", cert.Subject.CommonName, commonName) - } -} - -// AssertCertificateHasOrganizations is a utility function for kubeadm testing that asserts if a given certificate has -// the expected Subject.Organization -func AssertCertificateHasOrganizations(t *testing.T, cert *x509.Certificate, organizations ...string) { - for _, organization := range organizations { - found := false - for i := range cert.Subject.Organization { - if cert.Subject.Organization[i] == organization { - found = true - } - } - if !found { - t.Errorf("cert does not contain Subject.Organization %s as expected", organization) - } - } -} - -// AssertCertificateHasClientAuthUsage is a utility function for kubeadm testing that asserts if a given certificate has -// the expected ExtKeyUsageClientAuth -func AssertCertificateHasClientAuthUsage(t *testing.T, cert *x509.Certificate) { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageClientAuth { - return - } - } - t.Error("cert has not ClientAuth usage as expected") -} - -// AssertCertificateHasServerAuthUsage is a utility function for kubeadm testing that asserts if a given certificate has -// the expected ExtKeyUsageServerAuth -func AssertCertificateHasServerAuthUsage(t *testing.T, cert *x509.Certificate) { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth { - return - } - } - t.Error("cert is not a ServerAuth") -} - -// AssertCertificateHasDNSNames is a utility function for kubeadm testing that asserts if a given certificate has -// the expected DNSNames -func AssertCertificateHasDNSNames(t *testing.T, cert *x509.Certificate, DNSNames ...string) { - for _, DNSName := range DNSNames { - found := false - for _, val := range cert.DNSNames { - if val == DNSName { - found = true - break - } - } - - if !found { - t.Errorf("cert does not contain DNSName %s", DNSName) - } - } -} - -// AssertCertificateHasIPAddresses is a utility function for kubeadm testing that asserts if a given certificate has -// the expected IPAddresses -func AssertCertificateHasIPAddresses(t *testing.T, cert *x509.Certificate, IPAddresses ...net.IP) { - for _, IPAddress := range IPAddresses { - found := false - for _, val := range cert.IPAddresses { - if val.Equal(IPAddress) { - found = true - break - } - } - - if !found { - t.Errorf("cert does not contain IPAddress %s", IPAddress) - } - } -} - -// CreateCACert creates a generic CA cert. -func CreateCACert(t *testing.T) (*x509.Certificate, crypto.Signer) { - certCfg := &pkiutil.CertConfig{Config: certutil.Config{CommonName: "kubernetes"}} - cert, key, err := pkiutil.NewCertificateAuthority(certCfg) - if err != nil { - t.Fatalf("couldn't create CA: %v", err) - } - return cert, key -} - -// CreateTestCert makes a generic certificate with the given CA and alternative names. -func CreateTestCert(t *testing.T, caCert *x509.Certificate, caKey crypto.Signer, altNames certutil.AltNames) (*x509.Certificate, crypto.Signer, *pkiutil.CertConfig) { - config := &pkiutil.CertConfig{ - Config: certutil.Config{ - CommonName: "testCert", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, - AltNames: altNames, - }, - } - cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, config) - if err != nil { - t.Fatalf("couldn't create test cert: %v", err) - } - return cert, key, config -} - -// CertTestCase is a configuration of certificates and whether it's expected to work. -type CertTestCase struct { - Name string - Files PKIFiles - ExpectError bool -} - -// GetSparseCertTestCases produces a series of cert configurations and their intended outcomes. -func GetSparseCertTestCases(t *testing.T) []CertTestCase { - - caCert, caKey := CreateCACert(t) - fpCACert, fpCAKey := CreateCACert(t) - etcdCACert, etcdCAKey := CreateCACert(t) - - fpCert, fpKey, _ := CreateTestCert(t, fpCACert, fpCAKey, certutil.AltNames{}) - - return []CertTestCase{ - { - Name: "nothing present", - }, - { - Name: "CAs already exist", - Files: PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "front-proxy-ca.crt": fpCACert, - "front-proxy-ca.key": fpCAKey, - "etcd/ca.crt": etcdCACert, - "etcd/ca.key": etcdCAKey, - }, - }, - { - Name: "CA certs only", - Files: PKIFiles{ - "ca.crt": caCert, - "front-proxy-ca.crt": fpCACert, - "etcd/ca.crt": etcdCACert, - }, - ExpectError: true, - }, - { - Name: "FrontProxyCA with certs", - Files: PKIFiles{ - "ca.crt": caCert, - "ca.key": caKey, - "front-proxy-ca.crt": fpCACert, - "front-proxy-client.crt": fpCert, - "front-proxy-client.key": fpKey, - "etcd/ca.crt": etcdCACert, - "etcd/ca.key": etcdCAKey, - }, - }, - { - Name: "FrontProxy certs missing CA", - Files: PKIFiles{ - "front-proxy-client.crt": fpCert, - "front-proxy-client.key": fpKey, - }, - ExpectError: true, - }, - } -} - -// PKIFiles are a list of files that should be created for a test case -type PKIFiles map[string]interface{} - -// WritePKIFiles writes the given files out to the given directory -func WritePKIFiles(t *testing.T, dir string, files PKIFiles) { - for filename, body := range files { - switch body := body.(type) { - case *x509.Certificate: - if err := certutil.WriteCert(path.Join(dir, filename), pkiutil.EncodeCertPEM(body)); err != nil { - t.Errorf("unable to write certificate to file %q: [%v]", dir, err) - } - case *rsa.PublicKey: - publicKeyBytes, err := pkiutil.EncodePublicKeyPEM(body) - if err != nil { - t.Errorf("unable to write public key to file %q: [%v]", filename, err) - } - if err := keyutil.WriteKey(path.Join(dir, filename), publicKeyBytes); err != nil { - t.Errorf("unable to write public key to file %q: [%v]", filename, err) - } - case *rsa.PrivateKey: - privateKey, err := keyutil.MarshalPrivateKeyToPEM(body) - if err != nil { - t.Errorf("unable to write private key to file %q: [%v]", filename, err) - } - if err := keyutil.WriteKey(path.Join(dir, filename), privateKey); err != nil { - t.Errorf("unable to write private key to file %q: [%v]", filename, err) - } - } - } -} diff --git a/cmd/kubeadm/app/util/cgroupdriver.go b/cmd/kubeadm/app/util/cgroupdriver.go deleted file mode 100644 index ca552b1d7f039..0000000000000 --- a/cmd/kubeadm/app/util/cgroupdriver.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "strings" - - "github.com/pkg/errors" - - utilsexec "k8s.io/utils/exec" -) - -const ( - // CgroupDriverSystemd holds the systemd driver type - CgroupDriverSystemd = "systemd" - // CgroupDriverCgroupfs holds the cgroupfs driver type - CgroupDriverCgroupfs = "cgroupfs" -) - -// TODO: add support for detecting the cgroup driver for CRI other than -// Docker. Currently only Docker driver detection is supported: -// Discussion: -// https://github.com/kubernetes/kubeadm/issues/844 - -// GetCgroupDriverDocker runs 'docker info -f "{{.CgroupDriver}}"' to obtain the docker cgroup driver -func GetCgroupDriverDocker(execer utilsexec.Interface) (string, error) { - driver, err := callDockerInfo(execer) - if err != nil { - return "", err - } - return strings.TrimSuffix(driver, "\n"), nil -} - -func callDockerInfo(execer utilsexec.Interface) (string, error) { - out, err := execer.Command("docker", "info", "-f", "{{.CgroupDriver}}").Output() - if err != nil { - return "", errors.Wrap(err, "cannot execute 'docker info -f {{.CgroupDriver}}'") - } - return string(out), nil -} diff --git a/cmd/kubeadm/app/util/cgroupdriver_test.go b/cmd/kubeadm/app/util/cgroupdriver_test.go deleted file mode 100644 index 3d6b66373d3e8..0000000000000 --- a/cmd/kubeadm/app/util/cgroupdriver_test.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "testing" -) - -func TestGetCgroupDriverDocker(t *testing.T) { - testCases := []struct { - name string - driver string - expectedError bool - }{ - { - name: "valid: value is 'cgroupfs'", - driver: `cgroupfs`, - expectedError: false, - }, - { - name: "valid: value is 'systemd'", - driver: `systemd`, - expectedError: false, - }, - { - name: "invalid: empty 'Cgroup Driver' value", - driver: ``, - expectedError: true, - }, - { - name: "invalid: unknown 'Cgroup Driver' value", - driver: `invalid-value`, - expectedError: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := tc.driver != CgroupDriverCgroupfs && tc.driver != CgroupDriverSystemd - if result != tc.expectedError { - t.Fatalf("expected error: %v, saw: %v", tc.expectedError, result) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/chroot_unix.go b/cmd/kubeadm/app/util/chroot_unix.go deleted file mode 100644 index b8b83c373f344..0000000000000 --- a/cmd/kubeadm/app/util/chroot_unix.go +++ /dev/null @@ -1,40 +0,0 @@ -// +build !windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "github.com/pkg/errors" - "os" - "path/filepath" - "syscall" -) - -// Chroot chroot()s to the new path. -// NB: All file paths after this call are effectively relative to -// `rootfs` -func Chroot(rootfs string) error { - if err := syscall.Chroot(rootfs); err != nil { - return errors.Wrapf(err, "unable to chroot to %s", rootfs) - } - root := filepath.FromSlash("/") - if err := os.Chdir(root); err != nil { - return errors.Wrapf(err, "unable to chdir to %s", root) - } - return nil -} diff --git a/cmd/kubeadm/app/util/chroot_windows.go b/cmd/kubeadm/app/util/chroot_windows.go deleted file mode 100644 index b7e434e08289e..0000000000000 --- a/cmd/kubeadm/app/util/chroot_windows.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "github.com/pkg/errors" -) - -// Chroot chroot()s to the new path. -// NB: All file paths after this call are effectively relative to -// `rootfs` -func Chroot(rootfs string) error { - return errors.New("chroot is not implemented on Windows") -} diff --git a/cmd/kubeadm/app/util/config/BUILD b/cmd/kubeadm/app/util/config/BUILD deleted file mode 100644 index 05d06bcc9e187..0000000000000 --- a/cmd/kubeadm/app/util/config/BUILD +++ /dev/null @@ -1,93 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "cluster.go", - "common.go", - "initconfiguration.go", - "joinconfiguration.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/config", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/validation:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/config/strict:go_default_library", - "//cmd/kubeadm/app/util/runtime:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "cluster_test.go", - "common_test.go", - "initconfiguration_test.go", - "joinconfiguration_test.go", - ], - data = glob(["testdata/**"]), - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/test/resources:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/app/util/config/strict:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go deleted file mode 100644 index 83188f6a15179..0000000000000 --- a/cmd/kubeadm/app/util/config/cluster.go +++ /dev/null @@ -1,323 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "context" - "crypto/x509" - "fmt" - "io" - "path/filepath" - "strings" - - "github.com/pkg/errors" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - errorsutil "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// unretriableError is an error used temporarily while we are migrating from the -// ClusterStatus struct to an annotation Pod based information. When performing -// the upgrade of all control plane nodes with `kubeadm upgrade apply` and -// `kubeadm upgrade node` we don't want to retry as if we were hitting connectivity -// issues when the pod annotation is missing on the API server pods. This error will -// be used in such scenario, for failing fast, and falling back to the ClusterStatus -// retrieval in those cases. -type unretriableError struct { - err error -} - -func newUnretriableError(err error) *unretriableError { - return &unretriableError{err: err} -} - -func (ue *unretriableError) Error() string { - return fmt.Sprintf("unretriable error: %s", ue.err.Error()) -} - -// FetchInitConfigurationFromCluster fetches configuration from a ConfigMap in the cluster -func FetchInitConfigurationFromCluster(client clientset.Interface, w io.Writer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { - fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix) - fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -o yaml'\n", logPrefix, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) - - // Fetch the actual config from cluster - cfg, err := getInitConfigurationFromCluster(constants.KubernetesDir, client, newControlPlane, skipComponentConfigs) - if err != nil { - return nil, err - } - - // Apply dynamic defaults - if err := SetInitDynamicDefaults(cfg); err != nil { - return nil, err - } - - return cfg, nil -} - -// getInitConfigurationFromCluster is separate only for testing purposes, don't call it directly, use FetchInitConfigurationFromCluster instead -func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Interface, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { - // Also, the config map really should be KubeadmConfigConfigMap... - configMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) - if err != nil { - return nil, errors.Wrap(err, "failed to get config map") - } - - // InitConfiguration is composed with data from different places - initcfg := &kubeadmapi.InitConfiguration{} - - // gets ClusterConfiguration from kubeadm-config - clusterConfigurationData, ok := configMap.Data[constants.ClusterConfigurationConfigMapKey] - if !ok { - return nil, errors.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterConfigurationConfigMapKey) - } - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigurationData), &initcfg.ClusterConfiguration); err != nil { - return nil, errors.Wrap(err, "failed to decode cluster configuration data") - } - - if !skipComponentConfigs { - // get the component configs from the corresponding config maps - if err := componentconfigs.FetchFromCluster(&initcfg.ClusterConfiguration, client); err != nil { - return nil, errors.Wrap(err, "failed to get component configs") - } - } - - // if this isn't a new controlplane instance (e.g. in case of kubeadm upgrades) - // get nodes specific information as well - if !newControlPlane { - // gets the nodeRegistration for the current from the node object - if err := getNodeRegistration(kubeconfigDir, client, &initcfg.NodeRegistration); err != nil { - return nil, errors.Wrap(err, "failed to get node registration") - } - // gets the APIEndpoint for the current node - if err := getAPIEndpoint(client, initcfg.NodeRegistration.Name, &initcfg.LocalAPIEndpoint); err != nil { - return nil, errors.Wrap(err, "failed to getAPIEndpoint") - } - } else { - // In the case where newControlPlane is true we don't go through getNodeRegistration() and initcfg.NodeRegistration.CRISocket is empty. - // This forces DetectCRISocket() to be called later on, and if there is more than one CRI installed on the system, it will error out, - // while asking for the user to provide an override for the CRI socket. Even if the user provides an override, the call to - // DetectCRISocket() can happen too early and thus ignore it (while still erroring out). - // However, if newControlPlane == true, initcfg.NodeRegistration is not used at all and it's overwritten later on. - // Thus it's necessary to supply some default value, that will avoid the call to DetectCRISocket() and as - // initcfg.NodeRegistration is discarded, setting whatever value here is harmless. - initcfg.NodeRegistration.CRISocket = "/var/run/unknown.sock" - } - return initcfg, nil -} - -// getNodeRegistration returns the nodeRegistration for the current node -func getNodeRegistration(kubeconfigDir string, client clientset.Interface, nodeRegistration *kubeadmapi.NodeRegistrationOptions) error { - // gets the name of the current node - nodeName, err := getNodeNameFromKubeletConfig(kubeconfigDir) - if err != nil { - return errors.Wrap(err, "failed to get node name from kubelet config") - } - - // gets the corresponding node and retrieves attributes stored there. - node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) - if err != nil { - return errors.Wrap(err, "failed to get corresponding node") - } - - criSocket, ok := node.ObjectMeta.Annotations[constants.AnnotationKubeadmCRISocket] - if !ok { - return errors.Errorf("node %s doesn't have %s annotation", nodeName, constants.AnnotationKubeadmCRISocket) - } - - // returns the nodeRegistration attributes - nodeRegistration.Name = nodeName - nodeRegistration.CRISocket = criSocket - nodeRegistration.Taints = node.Spec.Taints - // NB. currently nodeRegistration.KubeletExtraArgs isn't stored at node level but only in the kubeadm-flags.env - // that isn't modified during upgrades - // in future we might reconsider this thus enabling changes to the kubeadm-flags.env during upgrades as well - return nil -} - -// getNodeNameFromKubeletConfig gets the node name from a kubelet config file -// TODO: in future we want to switch to a more canonical way for doing this e.g. by having this -// information in the local kubelet config.yaml -func getNodeNameFromKubeletConfig(kubeconfigDir string) (string, error) { - // loads the kubelet.conf file - fileName := filepath.Join(kubeconfigDir, constants.KubeletKubeConfigFileName) - config, err := clientcmd.LoadFromFile(fileName) - if err != nil { - return "", err - } - - // gets the info about the current user - currentContext, exists := config.Contexts[config.CurrentContext] - if !exists { - return "", errors.Errorf("invalid kubeconfig file %s: missing context %s", fileName, config.CurrentContext) - } - authInfo, exists := config.AuthInfos[currentContext.AuthInfo] - if !exists { - return "", errors.Errorf("invalid kubeconfig file %s: missing AuthInfo %s", fileName, currentContext.AuthInfo) - } - - // gets the X509 certificate with current user credentials - var certs []*x509.Certificate - if len(authInfo.ClientCertificateData) > 0 { - // if the config file uses an embedded x509 certificate (e.g. kubelet.conf created by kubeadm), parse it - if certs, err = certutil.ParseCertsPEM(authInfo.ClientCertificateData); err != nil { - return "", err - } - } else if len(authInfo.ClientCertificate) > 0 { - // if the config file links an external x509 certificate (e.g. kubelet.conf created by TLS bootstrap), load it - if certs, err = certutil.CertsFromFile(authInfo.ClientCertificate); err != nil { - return "", err - } - } else { - return "", errors.Errorf("invalid kubeconfig file %s. x509 certificate expected", fileName) - } - - // We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one - // TODO: Support multiple certs here in order to be able to rotate certs - cert := certs[0] - - // gets the node name from the certificate common name - return strings.TrimPrefix(cert.Subject.CommonName, constants.NodesUserPrefix), nil -} - -func getAPIEndpoint(client clientset.Interface, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error { - return getAPIEndpointWithBackoff(client, nodeName, apiEndpoint, constants.StaticPodMirroringDefaultRetry) -} - -func getAPIEndpointWithBackoff(client clientset.Interface, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint, backoff wait.Backoff) error { - var err error - var errs []error - - if err = getAPIEndpointFromPodAnnotation(client, nodeName, apiEndpoint, backoff); err == nil { - return nil - } - errs = append(errs, errors.WithMessagef(err, "could not retrieve API endpoints for node %q using pod annotations", nodeName)) - - // NB: this is a fallback when there is no annotation found in the API server pod that contains - // the API endpoint, and so we fallback to reading the ClusterStatus struct present in the - // kubeadm-config ConfigMap. This can happen for example, when performing the first - // `kubeadm upgrade apply` and `kubeadm upgrade node` cycle on the whole cluster. This logic - // will be removed when the cluster status struct is removed from the kubeadm-config ConfigMap. - if err = getAPIEndpointFromClusterStatus(client, nodeName, apiEndpoint); err == nil { - return nil - } - errs = append(errs, errors.WithMessagef(err, "could not retrieve API endpoints for node %q using cluster status", nodeName)) - - return errorsutil.NewAggregate(errs) -} - -func getAPIEndpointFromPodAnnotation(client clientset.Interface, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint, backoff wait.Backoff) error { - var rawAPIEndpoint string - var lastErr error - // Let's tolerate some unexpected transient failures from the API server or load balancers. Also, if - // static pods were not yet mirrored into the API server we want to wait for this propagation. - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - rawAPIEndpoint, lastErr = getRawAPIEndpointFromPodAnnotationWithoutRetry(client, nodeName) - // TODO (ereslibre): this logic will need tweaking once that we get rid of the ClusterStatus, since we won't have - // the ClusterStatus safety net, we will want to remove the UnretriableError and not make the distinction here - // anymore. - if _, ok := lastErr.(*unretriableError); ok { - // Fail fast scenario, to be removed once we get rid of the ClusterStatus - return true, errors.Wrapf(lastErr, "API server Pods exist, but no API endpoint annotations were found") - } - return lastErr == nil, nil - }) - if err != nil { - return err - } - parsedAPIEndpoint, err := kubeadmapi.APIEndpointFromString(rawAPIEndpoint) - if err != nil { - return errors.Wrapf(err, "could not parse API endpoint for node %q", nodeName) - } - *apiEndpoint = parsedAPIEndpoint - return nil -} - -func getRawAPIEndpointFromPodAnnotationWithoutRetry(client clientset.Interface, nodeName string) (string, error) { - podList, err := client.CoreV1().Pods(metav1.NamespaceSystem).List( - context.TODO(), - metav1.ListOptions{ - FieldSelector: fmt.Sprintf("spec.nodeName=%s", nodeName), - LabelSelector: fmt.Sprintf("component=%s,tier=%s", constants.KubeAPIServer, constants.ControlPlaneTier), - }, - ) - if err != nil { - return "", errors.Wrap(err, "could not retrieve list of pods to determine api server endpoints") - } - if len(podList.Items) != 1 { - return "", errors.Errorf("API server pod for node name %q has %d entries, only one was expected", nodeName, len(podList.Items)) - } - if apiServerEndpoint, ok := podList.Items[0].Annotations[constants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey]; ok { - return apiServerEndpoint, nil - } - return "", newUnretriableError(errors.Errorf("API server pod for node name %q hasn't got a %q annotation, cannot retrieve API endpoint", nodeName, constants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey)) -} - -// TODO: remove after 1.20, when the ClusterStatus struct is removed from the kubeadm-config ConfigMap. -func getAPIEndpointFromClusterStatus(client clientset.Interface, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error { - clusterStatus, err := GetClusterStatus(client) - if err != nil { - return errors.Wrap(err, "could not retrieve cluster status") - } - if statusAPIEndpoint, ok := clusterStatus.APIEndpoints[nodeName]; ok { - *apiEndpoint = statusAPIEndpoint - return nil - } - return errors.Errorf("could not find node %s in the cluster status", nodeName) -} - -// GetClusterStatus returns the kubeadm cluster status read from the kubeadm-config ConfigMap -func GetClusterStatus(client clientset.Interface) (*kubeadmapi.ClusterStatus, error) { - configMap, err := apiclient.GetConfigMapWithRetry(client, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) - if apierrors.IsNotFound(err) { - return &kubeadmapi.ClusterStatus{}, nil - } - if err != nil { - return nil, err - } - - clusterStatus, err := UnmarshalClusterStatus(configMap.Data) - if err != nil { - return nil, err - } - - return clusterStatus, nil -} - -// UnmarshalClusterStatus takes raw ConfigMap.Data and converts it to a ClusterStatus object -func UnmarshalClusterStatus(data map[string]string) (*kubeadmapi.ClusterStatus, error) { - clusterStatusData, ok := data[constants.ClusterStatusConfigMapKey] - if !ok { - return nil, errors.Errorf("unexpected error when reading kubeadm-config ConfigMap: %s key value pair missing", constants.ClusterStatusConfigMapKey) - } - clusterStatus := &kubeadmapi.ClusterStatus{} - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterStatusData), clusterStatus); err != nil { - return nil, err - } - - return clusterStatus, nil -} diff --git a/cmd/kubeadm/app/util/config/cluster_test.go b/cmd/kubeadm/app/util/config/cluster_test.go deleted file mode 100644 index 13e221ff827ca..0000000000000 --- a/cmd/kubeadm/app/util/config/cluster_test.go +++ /dev/null @@ -1,1033 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "context" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "strconv" - "strings" - "testing" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/apimachinery/pkg/util/wait" - clientsetfake "k8s.io/client-go/kubernetes/fake" - clienttesting "k8s.io/client-go/testing" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - testresources "k8s.io/kubernetes/cmd/kubeadm/test/resources" -) - -var k8sVersionString = kubeadmconstants.MinimumControlPlaneVersion.String() -var k8sVersion = version.MustParseGeneric(k8sVersionString) -var nodeName = "mynode" -var cfgFiles = map[string][]byte{ - "InitConfiguration_v1beta1": []byte(` -apiVersion: kubeadm.k8s.io/v1beta1 -kind: InitConfiguration -`), - "ClusterConfiguration_v1beta1": []byte(` -apiVersion: kubeadm.k8s.io/v1beta1 -kind: ClusterConfiguration -kubernetesVersion: ` + k8sVersionString + ` -`), - "InitConfiguration_v1beta2": []byte(` -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -`), - "ClusterConfiguration_v1beta2": []byte(` -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -kubernetesVersion: ` + k8sVersionString + ` -`), - "Kube-proxy_componentconfig": []byte(` -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -`), - "Kubelet_componentconfig": []byte(` -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -`), -} - -var kubeletConfFiles = map[string][]byte{ - "withoutX509Cert": []byte(` -apiVersion: v1 -clusters: -- cluster: - server: https://10.0.2.15:6443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: system:node:mynode - name: system:node:mynode@kubernetes -current-context: system:node:mynode@kubernetes -kind: Config -preferences: {} -users: -- name: system:node:mynode - user: -`), - "configWithEmbeddedCert": []byte(` -apiVersion: v1 -clusters: -- cluster: - server: https://10.0.2.15:6443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: system:node:mynode - name: system:node:mynode@kubernetes -current-context: system:node:mynode@kubernetes -kind: Config -preferences: {} -users: -- name: system:node:mynode - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJQWl3VURhYk5vZ1F3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB4T0RBNU1ERXhOVE14TWpaYUZ3MHhPVEE1TURFeE5qQXhOVGxhTURReApGVEFUQmdOVkJBb1RESE41YzNSbGJUcHViMlJsY3pFYk1Ca0dBMVVFQXhNU2MzbHpkR1Z0T201dlpHVTZiWGx1CmIyUmxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQWs2UXUzeStyNEZYUzZ4VkoKWU1vNE9kSkt3R1d1MHY4TEJIUnhhOUhvVHo1RXZLQnB1OVJoemt5dStUaFczb0xta2ZTRmNJcitHa0M5MW0zOApFelRmVE5JY2dsL0V5YkpmR1QvdGdUazZYd1kxY1UrUUdmSEFNNTBCVzFXTFVHc25CSllJZjA5eENnZTVoTkxLCnREeUJOWWNQZzg1bUJpOU9CNFJ2UlgyQVFRMjJwZ0xrQUpJWklOU0FEdUFrODN2b051SXM2YVY2bHBkR2Vva3YKdDlpTFdNR3p3a3ZTZUZQTlNGeWZ3Q055eENjb1FDQUNtSnJRS3NoeUE2bWNzdVhORWVXdlRQdVVFSWZPVFl4dwpxdkszRVBOK0xUYlA2anhUMWtTcFJUOSt4Z29uSlFhU0RsbUNBd20zRGJkSVppWUt3R2ppMkxKL0kvYWc0cTlzCjNLb0J2UUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLcVVrU21jdW85OG5EK015b005VFdEV0pyTndySXpQTUNqRQpCSkdyREhVaHIwcEZlRjc0RHViODNzRXlaNjFxNUVQd2Y0enNLSzdzdDRUTzZhcE9pZWJYVmN3dnZoa09HQ2dFCmFVdGNOMjFhUGxtU0tOd0c4ai8yK3ZhbU80bGplK1NnZzRUUVB0eDZWejh5VXN2RFhxSUZycjNNd1gzSDA1RW4KWXAzN05JYkhKbGxHUW5LVHA5aTg5aXF4WXVhSERqZldiVHlEY3B5NldNVjdVaFYvY1plc3lGL0NBamNHd1V6YgowRlo5bW5tMnFONlBGWHZ4RmdMSGFWZzN2SVVCbkNmVVVyY1BDNE94VFNPK21aUmUxazh3eUFpVWovSk0rZllvCkcrMi9sbThUYVZqb1U3Rmk1S2E1RzVIWTJHTGFSN1ArSXhZY3JNSENsNjJZN1JxY3JuYz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= -`), - "configWithLinkedCert": []byte(` -apiVersion: v1 -clusters: -- cluster: - server: https://10.0.2.15:6443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: system:node:mynode - name: system:node:mynode@kubernetes -current-context: system:node:mynode@kubernetes -kind: Config -preferences: {} -users: -- name: system:node:mynode - user: - client-certificate: kubelet.pem -`), - "configWithInvalidContext": []byte(` -apiVersion: v1 -clusters: -- cluster: - server: https://10.0.2.15:6443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: system:node:mynode - name: system:node:mynode@kubernetes -current-context: invalidContext -kind: Config -preferences: {} -users: -- name: system:node:mynode - user: - client-certificate: kubelet.pem -`), - "configWithInvalidUser": []byte(` -apiVersion: v1 -clusters: -- cluster: - server: https://10.0.2.15:6443 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: invalidUser - name: system:node:mynode@kubernetes -current-context: system:node:mynode@kubernetes -kind: Config -preferences: {} -users: -- name: system:node:mynode - user: - client-certificate: kubelet.pem -`), -} - -var pemFiles = map[string][]byte{ - "mynode.pem": []byte(` ------BEGIN CERTIFICATE----- -MIIC8jCCAdqgAwIBAgIIAiwUDabNogQwDQYJKoZIhvcNAQELBQAwFTETMBEGA1UE -AxMKa3ViZXJuZXRlczAeFw0xODA5MDExNTMxMjZaFw0xOTA5MDExNjAxNTlaMDQx -FTATBgNVBAoTDHN5c3RlbTpub2RlczEbMBkGA1UEAxMSc3lzdGVtOm5vZGU6bXlu -b2RlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk6Qu3y+r4FXS6xVJ -YMo4OdJKwGWu0v8LBHRxa9HoTz5EvKBpu9Rhzkyu+ThW3oLmkfSFcIr+GkC91m38 -EzTfTNIcgl/EybJfGT/tgTk6XwY1cU+QGfHAM50BW1WLUGsnBJYIf09xCge5hNLK -tDyBNYcPg85mBi9OB4RvRX2AQQ22pgLkAJIZINSADuAk83voNuIs6aV6lpdGeokv -t9iLWMGzwkvSeFPNSFyfwCNyxCcoQCACmJrQKshyA6mcsuXNEeWvTPuUEIfOTYxw -qvK3EPN+LTbP6jxT1kSpRT9+xgonJQaSDlmCAwm3DbdIZiYKwGji2LJ/I/ag4q9s -3KoBvQIDAQABoycwJTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUH -AwIwDQYJKoZIhvcNAQELBQADggEBAKqUkSmcuo98nD+MyoM9TWDWJrNwrIzPMCjE -BJGrDHUhr0pFeF74Dub83sEyZ61q5EPwf4zsKK7st4TO6apOiebXVcwvvhkOGCgE -aUtcN21aPlmSKNwG8j/2+vamO4lje+Sgg4TQPtx6Vz8yUsvDXqIFrr3MwX3H05En -Yp37NIbHJllGQnKTp9i89iqxYuaHDjfWbTyDcpy6WMV7UhV/cZesyF/CAjcGwUzb -0FZ9mnm2qN6PFXvxFgLHaVg3vIUBnCfUUrcPC4OxTSO+mZRe1k8wyAiUj/JM+fYo -G+2/lm8TaVjoU7Fi5Ka5G5HY2GLaR7P+IxYcrMHCl62Y7Rqcrnc= ------END CERTIFICATE----- -`), -} - -func TestGetNodeNameFromKubeletConfig(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - var tests = []struct { - name string - kubeconfigContent []byte - pemContent []byte - expectedError bool - }{ - { - name: "valid - with embedded cert", - kubeconfigContent: kubeletConfFiles["configWithEmbeddedCert"], - }, - { - name: "invalid - linked cert missing", - kubeconfigContent: kubeletConfFiles["configWithLinkedCert"], - expectedError: true, - }, - { - name: "valid - with linked cert", - kubeconfigContent: kubeletConfFiles["configWithLinkedCert"], - pemContent: pemFiles["mynode.pem"], - }, - { - name: "invalid - without embedded or linked X509Cert", - kubeconfigContent: kubeletConfFiles["withoutX509Cert"], - expectedError: true, - }, - { - name: "invalid - the current context is invalid", - kubeconfigContent: kubeletConfFiles["configWithInvalidContext"], - expectedError: true, - }, - { - name: "invalid - the user of the current context is invalid", - kubeconfigContent: kubeletConfFiles["configWithInvalidUser"], - expectedError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - if len(rt.pemContent) > 0 { - pemPath := filepath.Join(tmpdir, "kubelet.pem") - err := ioutil.WriteFile(pemPath, rt.pemContent, 0644) - if err != nil { - t.Errorf("Couldn't create pem file: %v", err) - return - } - rt.kubeconfigContent = []byte(strings.Replace(string(rt.kubeconfigContent), "kubelet.pem", pemPath, -1)) - } - - kubeconfigPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName) - err := ioutil.WriteFile(kubeconfigPath, rt.kubeconfigContent, 0644) - if err != nil { - t.Errorf("Couldn't create kubeconfig: %v", err) - return - } - - name, err := getNodeNameFromKubeletConfig(tmpdir) - if rt.expectedError != (err != nil) { - t.Errorf("unexpected return err from getNodeRegistration: %v", err) - return - } - if rt.expectedError { - return - } - - if name != nodeName { - t.Errorf("invalid name") - } - }) - } -} - -func TestGetNodeRegistration(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - var tests = []struct { - name string - fileContents []byte - node *v1.Node - expectedError bool - }{ - { - name: "invalid - no kubelet.conf", - expectedError: true, - }, - { - name: "valid", - fileContents: kubeletConfFiles["configWithEmbeddedCert"], - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Annotations: map[string]string{ - kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket", - }, - }, - Spec: v1.NodeSpec{ - Taints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - }, - }, - }, - { - name: "invalid - no node", - fileContents: kubeletConfFiles["configWithEmbeddedCert"], - expectedError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - cfgPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName) - if len(rt.fileContents) > 0 { - err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644) - if err != nil { - t.Errorf("Couldn't create file") - return - } - } - - client := clientsetfake.NewSimpleClientset() - - if rt.node != nil { - _, err := client.CoreV1().Nodes().Create(context.TODO(), rt.node, metav1.CreateOptions{}) - if err != nil { - t.Errorf("couldn't create Node") - return - } - } - - cfg := &kubeadmapi.InitConfiguration{} - err = getNodeRegistration(tmpdir, client, &cfg.NodeRegistration) - if rt.expectedError != (err != nil) { - t.Errorf("unexpected return err from getNodeRegistration: %v", err) - return - } - if rt.expectedError { - return - } - - if cfg.NodeRegistration.Name != nodeName { - t.Errorf("invalid cfg.NodeRegistration.Name") - } - if cfg.NodeRegistration.CRISocket != "myCRIsocket" { - t.Errorf("invalid cfg.NodeRegistration.CRISocket") - } - if len(cfg.NodeRegistration.Taints) != 1 { - t.Errorf("invalid cfg.NodeRegistration.Taints") - } - }) - } -} - -func TestGetAPIEndpointWithBackoff(t *testing.T) { - var tests = []struct { - name string - nodeName string - staticPod *testresources.FakeStaticPod - configMap *testresources.FakeConfigMap - expectedEndpoint *kubeadmapi.APIEndpoint - expectedErr bool - }{ - { - name: "no pod annotations; no ClusterStatus", - nodeName: nodeName, - expectedErr: true, - }, - { - name: "valid ipv4 endpoint in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234", - }, - }, - expectedEndpoint: &kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - }, - { - name: "invalid ipv4 endpoint in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3::1234", - }, - }, - expectedErr: true, - }, - { - name: "invalid negative port with ipv4 address in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:-1234", - }, - }, - expectedErr: true, - }, - { - name: "invalid high port with ipv4 address in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:65536", - }, - }, - expectedErr: true, - }, - { - name: "valid ipv6 endpoint in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "[::1]:1234", - }, - }, - expectedEndpoint: &kubeadmapi.APIEndpoint{AdvertiseAddress: "::1", BindPort: 1234}, - }, - { - name: "invalid ipv6 endpoint in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "[::1:1234", - }, - }, - expectedErr: true, - }, - { - name: "invalid negative port with ipv6 address in pod annotation; no ClusterStatus", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "[::1]:-1234", - }, - }, - expectedErr: true, - }, - { - name: "invalid high port with ipv6 address in pod annotation", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "[::1]:65536", - }, - }, - expectedErr: true, - }, - { - name: "no pod annotations; ClusterStatus with valid ipv4 endpoint", - nodeName: nodeName, - configMap: testresources.ClusterStatusWithAPIEndpoint(nodeName, kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}), - expectedEndpoint: &kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - }, - { - name: "invalid ipv4 endpoint in pod annotation; ClusterStatus with valid ipv4 endpoint", - nodeName: nodeName, - staticPod: &testresources.FakeStaticPod{ - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3::1234", - }, - }, - configMap: testresources.ClusterStatusWithAPIEndpoint(nodeName, kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}), - expectedEndpoint: &kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - if rt.staticPod != nil { - rt.staticPod.NodeName = rt.nodeName - if err := rt.staticPod.Create(client); err != nil { - t.Error("could not create static pod") - return - } - } - if rt.configMap != nil { - if err := rt.configMap.Create(client); err != nil { - t.Error("could not create ConfigMap") - return - } - } - apiEndpoint := kubeadm.APIEndpoint{} - err := getAPIEndpointWithBackoff(client, rt.nodeName, &apiEndpoint, wait.Backoff{Duration: 0, Jitter: 0, Steps: 1}) - if err != nil && !rt.expectedErr { - t.Errorf("got error %q; was expecting no errors", err) - return - } else if err == nil && rt.expectedErr { - t.Error("got no error; was expecting an error") - return - } - - if rt.expectedEndpoint != nil && !reflect.DeepEqual(apiEndpoint, *rt.expectedEndpoint) { - t.Errorf("expected API endpoint: %v; got %v", rt.expectedEndpoint, apiEndpoint) - } - }) - } -} - -func TestGetInitConfigurationFromCluster(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - var tests = []struct { - name string - fileContents []byte - node *v1.Node - staticPods []testresources.FakeStaticPod - configMaps []testresources.FakeConfigMap - newControlPlane bool - expectedError bool - }{ - { - name: "invalid - No kubeadm-config ConfigMap", - expectedError: true, - }, - { - name: "invalid - No ClusterConfiguration in kubeadm-config ConfigMap", - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, // ClusterConfiguration from kubeadm-config. - Data: map[string]string{}, - }, - }, - expectedError: true, - }, - { - name: "valid v1beta1 - new control plane == false", // InitConfiguration composed with data from different places, with also node specific information from ClusterStatus and node - staticPods: []testresources.FakeStaticPod{ - { - NodeName: nodeName, - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234", - }, - }, - }, - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, // ClusterConfiguration from kubeadm-config. - Data: map[string]string{ - kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1beta1"]), - }, - }, - { - Name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), - }, - }, - { - Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), - }, - }, - }, - fileContents: kubeletConfFiles["configWithEmbeddedCert"], - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Annotations: map[string]string{ - kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket", - }, - }, - Spec: v1.NodeSpec{ - Taints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - }, - }, - }, - { - name: "valid v1beta1 - new control plane == true", // InitConfiguration composed with data from different places, without node specific information - staticPods: []testresources.FakeStaticPod{ - { - NodeName: nodeName, - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234", - }, - }, - }, - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, // ClusterConfiguration from kubeadm-config. - Data: map[string]string{ - kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1beta1"]), - }, - }, - { - Name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), - }, - }, - { - Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), - }, - }, - }, - newControlPlane: true, - }, - { - name: "valid v1beta2 - new control plane == false", // InitConfiguration composed with data from different places, with also node specific information from ClusterStatus and node - staticPods: []testresources.FakeStaticPod{ - { - NodeName: nodeName, - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234", - }, - }, - }, - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, // ClusterConfiguration from kubeadm-config. - Data: map[string]string{ - kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1beta2"]), - }, - }, - { - Name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), - }, - }, - { - Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), - }, - }, - }, - fileContents: kubeletConfFiles["configWithEmbeddedCert"], - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Annotations: map[string]string{ - kubeadmconstants.AnnotationKubeadmCRISocket: "myCRIsocket", - }, - }, - Spec: v1.NodeSpec{ - Taints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - }, - }, - }, - { - name: "valid v1beta2 - new control plane == true", // InitConfiguration composed with data from different places, without node specific information - staticPods: []testresources.FakeStaticPod{ - { - NodeName: nodeName, - Component: kubeadmconstants.KubeAPIServer, - Annotations: map[string]string{ - kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234", - }, - }, - }, - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, // ClusterConfiguration from kubeadm-config. - Data: map[string]string{ - kubeadmconstants.ClusterConfigurationConfigMapKey: string(cfgFiles["ClusterConfiguration_v1beta2"]), - }, - }, - { - Name: kubeadmconstants.KubeProxyConfigMap, // Kube-proxy component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeProxyConfigMapKey: string(cfgFiles["Kube-proxy_componentconfig"]), - }, - }, - { - Name: kubeadmconstants.GetKubeletConfigMapName(k8sVersion), // Kubelet component config from corresponding ConfigMap. - Data: map[string]string{ - kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(cfgFiles["Kubelet_componentconfig"]), - }, - }, - }, - newControlPlane: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - cfgPath := filepath.Join(tmpdir, kubeadmconstants.KubeletKubeConfigFileName) - if len(rt.fileContents) > 0 { - err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644) - if err != nil { - t.Errorf("Couldn't create file") - return - } - } - - client := clientsetfake.NewSimpleClientset() - - if rt.node != nil { - _, err := client.CoreV1().Nodes().Create(context.TODO(), rt.node, metav1.CreateOptions{}) - if err != nil { - t.Errorf("couldn't create Node") - return - } - } - - for _, p := range rt.staticPods { - err := p.Create(client) - if err != nil { - t.Errorf("couldn't create pod for nodename %s", p.NodeName) - return - } - } - - for _, c := range rt.configMaps { - err := c.Create(client) - if err != nil { - t.Errorf("couldn't create ConfigMap %s", c.Name) - return - } - } - - cfg, err := getInitConfigurationFromCluster(tmpdir, client, rt.newControlPlane, false) - if rt.expectedError != (err != nil) { - t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err) - return - } - if rt.expectedError { - return - } - - // Test expected values in InitConfiguration - if cfg == nil { - t.Errorf("unexpected nil return value") - return - } - if cfg.ClusterConfiguration.KubernetesVersion != k8sVersionString { - t.Errorf("invalid ClusterConfiguration.KubernetesVersion") - } - if !rt.newControlPlane && (cfg.LocalAPIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.LocalAPIEndpoint.BindPort != 1234) { - t.Errorf("invalid cfg.LocalAPIEndpoint: %v", cfg.LocalAPIEndpoint) - } - if _, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup]; !ok { - t.Errorf("no cfg.ComponentConfigs[%q]", componentconfigs.KubeletGroup) - } - if _, ok := cfg.ComponentConfigs[componentconfigs.KubeProxyGroup]; !ok { - t.Errorf("no cfg.ComponentConfigs[%q]", componentconfigs.KubeProxyGroup) - } - }) - } -} - -func TestGetGetClusterStatus(t *testing.T) { - var tests = []struct { - name string - configMaps []testresources.FakeConfigMap - expectedError bool - }{ - { - name: "invalid missing config map", - }, - { - name: "valid v1beta1", - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, - Data: map[string]string{ - kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1beta1"]), - }, - }, - }, - }, - { - name: "valid v1beta2", - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, - Data: map[string]string{ - kubeadmconstants.ClusterStatusConfigMapKey: string(cfgFiles["ClusterStatus_v1beta2"]), - }, - }, - }, - }, - { - name: "invalid missing ClusterStatusConfigMapKey in the config map", - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, - Data: map[string]string{}, - }, - }, - expectedError: true, - }, - { - name: "invalid wrong value in the config map", - configMaps: []testresources.FakeConfigMap{ - { - Name: kubeadmconstants.KubeadmConfigConfigMap, - Data: map[string]string{ - kubeadmconstants.ClusterStatusConfigMapKey: "not a kubeadm type", - }, - }, - }, - expectedError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - - for _, c := range rt.configMaps { - err := c.Create(client) - if err != nil { - t.Errorf("couldn't create ConfigMap %s", c.Name) - return - } - } - - _, err := GetClusterStatus(client) - if rt.expectedError != (err != nil) { - t.Errorf("unexpected return err from GetClusterStatus: %v", err) - return - } - if rt.expectedError { - return - } - }) - } -} - -func TestGetAPIEndpointFromPodAnnotation(t *testing.T) { - var tests = []struct { - name string - nodeName string - pods []testresources.FakeStaticPod - clientSetup func(*clientsetfake.Clientset) - expectedEndpoint kubeadmapi.APIEndpoint - expectedErr bool - }{ - { - name: "exactly one pod with annotation", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234"}, - }, - }, - expectedEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}, - }, - { - name: "no pods with annotation", - nodeName: nodeName, - expectedErr: true, - }, - { - name: "exactly one pod with annotation; all requests fail", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234"}, - }, - }, - clientSetup: func(clientset *clientsetfake.Clientset) { - clientset.PrependReactor("list", "pods", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apierrors.NewInternalError(errors.New("API server down")) - }) - }, - expectedErr: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - for i, pod := range rt.pods { - pod.NodeName = rt.nodeName - if err := pod.CreateWithPodSuffix(client, strconv.Itoa(i)); err != nil { - t.Errorf("error setting up test creating pod for node %q", pod.NodeName) - return - } - } - if rt.clientSetup != nil { - rt.clientSetup(client) - } - apiEndpoint := kubeadmapi.APIEndpoint{} - err := getAPIEndpointFromPodAnnotation(client, rt.nodeName, &apiEndpoint, wait.Backoff{Duration: 0, Jitter: 0, Steps: 1}) - if err != nil && !rt.expectedErr { - t.Errorf("got error %v, but wasn't expecting any error", err) - return - } else if err == nil && rt.expectedErr { - t.Error("didn't get any error; but was expecting an error") - return - } else if err != nil && rt.expectedErr { - return - } - if !reflect.DeepEqual(apiEndpoint, rt.expectedEndpoint) { - t.Errorf("expected API endpoint: %v; got %v", rt.expectedEndpoint, apiEndpoint) - } - }) - } -} - -func TestGetRawAPIEndpointFromPodAnnotationWithoutRetry(t *testing.T) { - var tests = []struct { - name string - nodeName string - pods []testresources.FakeStaticPod - clientSetup func(*clientsetfake.Clientset) - expectedEndpoint string - expectedErr bool - }{ - { - name: "no pods", - nodeName: nodeName, - expectedErr: true, - }, - { - name: "exactly one pod with annotation", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234"}, - }, - }, - expectedEndpoint: "1.2.3.4:1234", - }, - { - name: "two pods: one with annotation, one missing annotation", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234"}, - }, - { - Component: constants.KubeAPIServer, - }, - }, - expectedErr: true, - }, - { - name: "two pods: different annotations", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234"}, - }, - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.5:1234"}, - }, - }, - expectedErr: true, - }, - { - name: "two pods: both missing annotation", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - }, - { - Component: constants.KubeAPIServer, - }, - }, - expectedErr: true, - }, - { - name: "exactly one pod with annotation; request fails", - nodeName: nodeName, - pods: []testresources.FakeStaticPod{ - { - Component: constants.KubeAPIServer, - Annotations: map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: "1.2.3.4:1234"}, - }, - }, - clientSetup: func(clientset *clientsetfake.Clientset) { - clientset.PrependReactor("list", "pods", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apierrors.NewInternalError(errors.New("API server down")) - }) - }, - expectedErr: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - for i, pod := range rt.pods { - pod.NodeName = rt.nodeName - if err := pod.CreateWithPodSuffix(client, strconv.Itoa(i)); err != nil { - t.Errorf("error setting up test creating pod for node %q", pod.NodeName) - return - } - } - if rt.clientSetup != nil { - rt.clientSetup(client) - } - endpoint, err := getRawAPIEndpointFromPodAnnotationWithoutRetry(client, rt.nodeName) - if err != nil && !rt.expectedErr { - t.Errorf("got error %v, but wasn't expecting any error", err) - return - } else if err == nil && rt.expectedErr { - t.Error("didn't get any error; but was expecting an error") - return - } else if err != nil && rt.expectedErr { - return - } - if endpoint != rt.expectedEndpoint { - t.Errorf("expected API endpoint: %v; got: %v", rt.expectedEndpoint, endpoint) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/config/common.go b/cmd/kubeadm/app/util/config/common.go deleted file mode 100644 index 4e9762112918b..0000000000000 --- a/cmd/kubeadm/app/util/config/common.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "bytes" - "net" - "reflect" - "strings" - - "github.com/pkg/errors" - "k8s.io/klog/v2" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - netutil "k8s.io/apimachinery/pkg/util/net" - "k8s.io/apimachinery/pkg/util/version" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -// MarshalKubeadmConfigObject marshals an Object registered in the kubeadm scheme. If the object is a InitConfiguration or ClusterConfiguration, some extra logic is run -func MarshalKubeadmConfigObject(obj runtime.Object) ([]byte, error) { - switch internalcfg := obj.(type) { - case *kubeadmapi.InitConfiguration: - return MarshalInitConfigurationToBytes(internalcfg, kubeadmapiv1beta2.SchemeGroupVersion) - default: - return kubeadmutil.MarshalToYamlForCodecs(obj, kubeadmapiv1beta2.SchemeGroupVersion, kubeadmscheme.Codecs) - } -} - -// validateSupportedVersion checks if the supplied GroupVersion is not on the lists of old unsupported or deprecated GVs. -// If it is, an error is returned. -func validateSupportedVersion(gv schema.GroupVersion, allowDeprecated bool) error { - // The support matrix will look something like this now and in the future: - // v1.10 and earlier: v1alpha1 - // v1.11: v1alpha1 read-only, writes only v1alpha2 config - // v1.12: v1alpha2 read-only, writes only v1alpha3 config. Errors if the user tries to use v1alpha1 - // v1.13: v1alpha3 read-only, writes only v1beta1 config. Errors if the user tries to use v1alpha1 or v1alpha2 - // v1.14: v1alpha3 convert only, writes only v1beta1 config. Errors if the user tries to use v1alpha1 or v1alpha2 - // v1.15: v1beta1 read-only, writes only v1beta2 config. Errors if the user tries to use v1alpha1, v1alpha2 or v1alpha3 - oldKnownAPIVersions := map[string]string{ - "kubeadm.k8s.io/v1alpha1": "v1.11", - "kubeadm.k8s.io/v1alpha2": "v1.12", - "kubeadm.k8s.io/v1alpha3": "v1.14", - } - - // Deprecated API versions are supported by us, but can only be used for migration. - deprecatedAPIVersions := map[string]struct{}{ - "kubeadm.k8s.io/v1beta1": {}, - } - - gvString := gv.String() - - if useKubeadmVersion := oldKnownAPIVersions[gvString]; useKubeadmVersion != "" { - return errors.Errorf("your configuration file uses an old API spec: %q. Please use kubeadm %s instead and run 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv.String(), useKubeadmVersion) - } - - if _, present := deprecatedAPIVersions[gvString]; present && !allowDeprecated { - klog.Warningf("your configuration file uses a deprecated API spec: %q. Please use 'kubeadm config migrate --old-config old.yaml --new-config new.yaml', which will write the new, similar spec using a newer API version.", gv) - } - - return nil -} - -// NormalizeKubernetesVersion resolves version labels, sets alternative -// image registry if requested for CI builds, and validates minimal -// version that kubeadm SetInitDynamicDefaultssupports. -func NormalizeKubernetesVersion(cfg *kubeadmapi.ClusterConfiguration) error { - // Requested version is automatic CI build, thus use KubernetesCI Image Repository for core images - if kubeadmutil.KubernetesIsCIVersion(cfg.KubernetesVersion) { - cfg.CIImageRepository = constants.DefaultCIImageRepository - } - - // Parse and validate the version argument and resolve possible CI version labels - ver, err := kubeadmutil.KubernetesReleaseVersion(cfg.KubernetesVersion) - if err != nil { - return err - } - cfg.KubernetesVersion = ver - - // Parse the given kubernetes version and make sure it's higher than the lowest supported - k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion) - if err != nil { - return errors.Wrapf(err, "couldn't parse Kubernetes version %q", cfg.KubernetesVersion) - } - if k8sVersion.LessThan(constants.MinimumControlPlaneVersion) { - return errors.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", constants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion) - } - return nil -} - -// LowercaseSANs can be used to force all SANs to be lowercase so it passes IsDNS1123Subdomain -func LowercaseSANs(sans []string) { - for i, san := range sans { - lowercase := strings.ToLower(san) - if lowercase != san { - klog.V(1).Infof("lowercasing SAN %q to %q", san, lowercase) - sans[i] = lowercase - } - } -} - -// VerifyAPIServerBindAddress can be used to verify if a bind address for the API Server is 0.0.0.0, -// in which case this address is not valid and should not be used. -func VerifyAPIServerBindAddress(address string) error { - ip := net.ParseIP(address) - if ip == nil { - return errors.Errorf("cannot parse IP address: %s", address) - } - // There are users with network setups where default routes are present, but network interfaces - // use only link-local addresses (e.g. as described in RFC5549). - // In many cases that matching global unicast IP address can be found on loopback interface, - // so kubeadm allows users to specify address=Loopback for handling supporting the scenario above. - // Nb. SetAPIEndpointDynamicDefaults will try to translate loopback to a valid address afterwards - if ip.IsLoopback() { - return nil - } - if !ip.IsGlobalUnicast() { - return errors.Errorf("cannot use %q as the bind address for the API Server", address) - } - return nil -} - -// ChooseAPIServerBindAddress is a wrapper for netutil.ResolveBindAddress that also handles -// the case where no default routes were found and an IP for the API server could not be obtained. -func ChooseAPIServerBindAddress(bindAddress net.IP) (net.IP, error) { - ip, err := netutil.ResolveBindAddress(bindAddress) - if err != nil { - if netutil.IsNoRoutesError(err) { - klog.Warningf("WARNING: could not obtain a bind address for the API Server: %v; using: %s", err, constants.DefaultAPIServerBindAddress) - defaultIP := net.ParseIP(constants.DefaultAPIServerBindAddress) - if defaultIP == nil { - return nil, errors.Errorf("cannot parse default IP address: %s", constants.DefaultAPIServerBindAddress) - } - return defaultIP, nil - } - return nil, err - } - if bindAddress != nil && !bindAddress.IsUnspecified() && !reflect.DeepEqual(ip, bindAddress) { - klog.Warningf("WARNING: overriding requested API server bind address: requested %q, actual %q", bindAddress, ip) - } - return ip, nil -} - -// MigrateOldConfig migrates an old configuration from a byte slice into a new one (returned again as a byte slice). -// Only kubeadm kinds are migrated. Others are silently ignored. -func MigrateOldConfig(oldConfig []byte) ([]byte, error) { - newConfig := [][]byte{} - - gvkmap, err := kubeadmutil.SplitYAMLDocuments(oldConfig) - if err != nil { - return []byte{}, err - } - - gvks := []schema.GroupVersionKind{} - for gvk := range gvkmap { - gvks = append(gvks, gvk) - } - - // Migrate InitConfiguration and ClusterConfiguration if there are any in the config - if kubeadmutil.GroupVersionKindsHasInitConfiguration(gvks...) || kubeadmutil.GroupVersionKindsHasClusterConfiguration(gvks...) { - o, err := documentMapToInitConfiguration(gvkmap, true) - if err != nil { - return []byte{}, err - } - b, err := MarshalKubeadmConfigObject(o) - if err != nil { - return []byte{}, err - } - newConfig = append(newConfig, b) - } - - // Migrate JoinConfiguration if there is any - if kubeadmutil.GroupVersionKindsHasJoinConfiguration(gvks...) { - o, err := documentMapToJoinConfiguration(gvkmap, true) - if err != nil { - return []byte{}, err - } - b, err := MarshalKubeadmConfigObject(o) - if err != nil { - return []byte{}, err - } - newConfig = append(newConfig, b) - } - - return bytes.Join(newConfig, []byte(constants.YAMLDocumentSeparator)), nil -} diff --git a/cmd/kubeadm/app/util/config/common_test.go b/cmd/kubeadm/app/util/config/common_test.go deleted file mode 100644 index 327f396baaaf6..0000000000000 --- a/cmd/kubeadm/app/util/config/common_test.go +++ /dev/null @@ -1,405 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - "testing" - - "github.com/lithammer/dedent" - - "k8s.io/apimachinery/pkg/runtime/schema" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const KubeadmGroupName = "kubeadm.k8s.io" - -func TestValidateSupportedVersion(t *testing.T) { - tests := []struct { - gv schema.GroupVersion - allowDeprecated bool - expectedErr bool - }{ - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1alpha1", - }, - expectedErr: true, - }, - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1alpha2", - }, - expectedErr: true, - }, - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1alpha3", - }, - expectedErr: true, - }, - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1alpha3", - }, - allowDeprecated: true, - expectedErr: true, - }, - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1beta1", - }, - allowDeprecated: true, - }, - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1beta1", - }, - }, - { - gv: schema.GroupVersion{ - Group: KubeadmGroupName, - Version: "v1beta2", - }, - }, - { - gv: schema.GroupVersion{ - Group: "foo.k8s.io", - Version: "v1", - }, - }, - } - - for _, rt := range tests { - t.Run(fmt.Sprintf("%s/allowDeprecated:%t", rt.gv, rt.allowDeprecated), func(t *testing.T) { - err := validateSupportedVersion(rt.gv, rt.allowDeprecated) - if rt.expectedErr && err == nil { - t.Error("unexpected success") - } else if !rt.expectedErr && err != nil { - t.Errorf("unexpected failure: %v", err) - } - }) - } -} - -func TestLowercaseSANs(t *testing.T) { - tests := []struct { - name string - in []string - out []string - }{ - { - name: "empty struct", - }, - { - name: "already lowercase", - in: []string{"example.k8s.io"}, - out: []string{"example.k8s.io"}, - }, - { - name: "ip addresses and uppercase", - in: []string{"EXAMPLE.k8s.io", "10.100.0.1"}, - out: []string{"example.k8s.io", "10.100.0.1"}, - }, - { - name: "punycode and uppercase", - in: []string{"xn--7gq663byk9a.xn--fiqz9s", "ANOTHEREXAMPLE.k8s.io"}, - out: []string{"xn--7gq663byk9a.xn--fiqz9s", "anotherexample.k8s.io"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := &kubeadmapiv1beta2.ClusterConfiguration{ - APIServer: kubeadmapiv1beta2.APIServer{ - CertSANs: test.in, - }, - } - - LowercaseSANs(cfg.APIServer.CertSANs) - - if len(cfg.APIServer.CertSANs) != len(test.out) { - t.Fatalf("expected %d elements, got %d", len(test.out), len(cfg.APIServer.CertSANs)) - } - - for i, expected := range test.out { - if cfg.APIServer.CertSANs[i] != expected { - t.Errorf("expected element %d to be %q, got %q", i, expected, cfg.APIServer.CertSANs[i]) - } - } - }) - } -} - -func TestVerifyAPIServerBindAddress(t *testing.T) { - tests := []struct { - name string - address string - expectedError bool - }{ - { - name: "valid address: IPV4", - address: "192.168.0.1", - }, - { - name: "valid address: IPV6", - address: "2001:db8:85a3::8a2e:370:7334", - }, - { - name: "valid address 127.0.0.1", - address: "127.0.0.1", - expectedError: false, - }, - { - name: "invalid address: not a global unicast 0.0.0.0", - address: "0.0.0.0", - expectedError: true, - }, - { - name: "invalid address: not a global unicast ::", - address: "::", - expectedError: true, - }, - { - name: "invalid address: cannot parse IPV4", - address: "255.255.255.255.255", - expectedError: true, - }, - { - name: "invalid address: cannot parse IPV6", - address: "2a00:800::2a00:800:10102a00", - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := VerifyAPIServerBindAddress(test.address); (err != nil) != test.expectedError { - t.Errorf("expected error: %v, got %v, error: %v", test.expectedError, (err != nil), err) - } - }) - } -} - -func TestMigrateOldConfigFromFile(t *testing.T) { - tests := []struct { - desc string - oldCfg string - expectedKinds []string - expectErr bool - }{ - { - desc: "empty file produces empty result", - oldCfg: "", - expectErr: false, - }, - { - desc: "bad config produces error", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - `), - expectErr: true, - }, - { - desc: "InitConfiguration only gets migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: InitConfiguration - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - }, - expectErr: false, - }, - { - desc: "ClusterConfiguration only gets migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: ClusterConfiguration - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - }, - expectErr: false, - }, - { - desc: "JoinConfiguration only gets migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - discovery: - bootstrapToken: - token: abcdef.0123456789abcdef - apiServerEndpoint: kube-apiserver:6443 - unsafeSkipCAVerification: true - `), - expectedKinds: []string{ - constants.JoinConfigurationKind, - }, - expectErr: false, - }, - { - desc: "Init + Cluster Configurations are migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: InitConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: ClusterConfiguration - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - }, - expectErr: false, - }, - { - desc: "Init + Join Configurations are migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: InitConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - discovery: - bootstrapToken: - token: abcdef.0123456789abcdef - apiServerEndpoint: kube-apiserver:6443 - unsafeSkipCAVerification: true - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - constants.JoinConfigurationKind, - }, - expectErr: false, - }, - { - desc: "Cluster + Join Configurations are migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: ClusterConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - discovery: - bootstrapToken: - token: abcdef.0123456789abcdef - apiServerEndpoint: kube-apiserver:6443 - unsafeSkipCAVerification: true - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - constants.JoinConfigurationKind, - }, - expectErr: false, - }, - { - desc: "Init + Cluster + Join Configurations are migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: InitConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: ClusterConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - discovery: - bootstrapToken: - token: abcdef.0123456789abcdef - apiServerEndpoint: kube-apiserver:6443 - unsafeSkipCAVerification: true - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - constants.JoinConfigurationKind, - }, - expectErr: false, - }, - { - desc: "component configs are not migrated", - oldCfg: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: InitConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: ClusterConfiguration - --- - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - discovery: - bootstrapToken: - token: abcdef.0123456789abcdef - apiServerEndpoint: kube-apiserver:6443 - unsafeSkipCAVerification: true - --- - apiVersion: kubeproxy.config.k8s.io/v1alpha1 - kind: KubeProxyConfiguration - --- - apiVersion: kubelet.config.k8s.io/v1beta1 - kind: KubeletConfiguration - `), - expectedKinds: []string{ - constants.InitConfigurationKind, - constants.ClusterConfigurationKind, - constants.JoinConfigurationKind, - }, - expectErr: false, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - b, err := MigrateOldConfig([]byte(test.oldCfg)) - if test.expectErr { - if err == nil { - t.Fatalf("unexpected success:\n%s", b) - } - } else { - if err != nil { - t.Fatalf("unexpected failure: %v", err) - } - gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b) - if err != nil { - t.Fatalf("unexpected error returned by GroupVersionKindsFromBytes: %v", err) - } - if len(gvks) != len(test.expectedKinds) { - t.Fatalf("length mismatch between resulting gvks and expected kinds:\n\tlen(gvks)=%d\n\tlen(expectedKinds)=%d", - len(gvks), len(test.expectedKinds)) - } - for _, expectedKind := range test.expectedKinds { - if !kubeadmutil.GroupVersionKindsHasKind(gvks, expectedKind) { - t.Fatalf("migration failed to produce config kind: %s", expectedKind) - } - } - } - }) - } -} diff --git a/cmd/kubeadm/app/util/config/initconfiguration.go b/cmd/kubeadm/app/util/config/initconfiguration.go deleted file mode 100644 index d8a37124870ca..0000000000000 --- a/cmd/kubeadm/app/util/config/initconfiguration.go +++ /dev/null @@ -1,337 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "bytes" - "fmt" - "io/ioutil" - "net" - "strconv" - - "github.com/pkg/errors" - "k8s.io/klog/v2" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - netutil "k8s.io/apimachinery/pkg/util/net" - bootstraputil "k8s.io/cluster-bootstrap/token/util" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" - kubeadmruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" -) - -// SetInitDynamicDefaults checks and sets configuration values for the InitConfiguration object -func SetInitDynamicDefaults(cfg *kubeadmapi.InitConfiguration) error { - if err := SetBootstrapTokensDynamicDefaults(&cfg.BootstrapTokens); err != nil { - return err - } - if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, true); err != nil { - return err - } - if err := SetAPIEndpointDynamicDefaults(&cfg.LocalAPIEndpoint); err != nil { - return err - } - return SetClusterDynamicDefaults(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, &cfg.NodeRegistration) -} - -// SetBootstrapTokensDynamicDefaults checks and sets configuration values for the BootstrapTokens object -func SetBootstrapTokensDynamicDefaults(cfg *[]kubeadmapi.BootstrapToken) error { - // Populate the .Token field with a random value if unset - // We do this at this layer, and not the API defaulting layer - // because of possible security concerns, and more practically - // because we can't return errors in the API object defaulting - // process but here we can. - for i, bt := range *cfg { - if bt.Token != nil && len(bt.Token.String()) > 0 { - continue - } - - tokenStr, err := bootstraputil.GenerateBootstrapToken() - if err != nil { - return errors.Wrap(err, "couldn't generate random token") - } - token, err := kubeadmapi.NewBootstrapTokenString(tokenStr) - if err != nil { - return err - } - (*cfg)[i].Token = token - } - - return nil -} - -// SetNodeRegistrationDynamicDefaults checks and sets configuration values for the NodeRegistration object -func SetNodeRegistrationDynamicDefaults(cfg *kubeadmapi.NodeRegistrationOptions, controlPlaneTaint bool) error { - var err error - cfg.Name, err = kubeadmutil.GetHostname(cfg.Name) - if err != nil { - return err - } - - // Only if the slice is nil, we should append the control-plane taint. This allows the user to specify an empty slice for no default control-plane taint - if controlPlaneTaint && cfg.Taints == nil { - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - cfg.Taints = []v1.Taint{kubeadmconstants.OldControlPlaneTaint} - } - - if cfg.CRISocket == "" { - cfg.CRISocket, err = kubeadmruntime.DetectCRISocket() - if err != nil { - return err - } - klog.V(1).Infof("detected and using CRI socket: %s", cfg.CRISocket) - } - - return nil -} - -// SetAPIEndpointDynamicDefaults checks and sets configuration values for the APIEndpoint object -func SetAPIEndpointDynamicDefaults(cfg *kubeadmapi.APIEndpoint) error { - // validate cfg.API.AdvertiseAddress. - addressIP := net.ParseIP(cfg.AdvertiseAddress) - if addressIP == nil && cfg.AdvertiseAddress != "" { - return errors.Errorf("couldn't use \"%s\" as \"apiserver-advertise-address\", must be ipv4 or ipv6 address", cfg.AdvertiseAddress) - } - - // kubeadm allows users to specify address=Loopback as a selector for global unicast IP address that can be found on loopback interface. - // e.g. This is required for network setups where default routes are present, but network interfaces use only link-local addresses (e.g. as described in RFC5549). - if addressIP.IsLoopback() { - loopbackIP, err := netutil.ChooseBindAddressForInterface(netutil.LoopbackInterfaceName) - if err != nil { - return err - } - if loopbackIP != nil { - klog.V(4).Infof("Found active IP %v on loopback interface", loopbackIP.String()) - cfg.AdvertiseAddress = loopbackIP.String() - return nil - } - return errors.New("unable to resolve link-local addresses") - } - - // This is the same logic as the API Server uses, except that if no interface is found the address is set to 0.0.0.0, which is invalid and cannot be used - // for bootstrapping a cluster. - ip, err := ChooseAPIServerBindAddress(addressIP) - if err != nil { - return err - } - cfg.AdvertiseAddress = ip.String() - - return nil -} - -// SetClusterDynamicDefaults checks and sets values for the ClusterConfiguration object -func SetClusterDynamicDefaults(cfg *kubeadmapi.ClusterConfiguration, localAPIEndpoint *kubeadmapi.APIEndpoint, nodeRegOpts *kubeadmapi.NodeRegistrationOptions) error { - // Default all the embedded ComponentConfig structs - componentconfigs.Default(cfg, localAPIEndpoint, nodeRegOpts) - - // Resolve possible version labels and validate version string - if err := NormalizeKubernetesVersion(cfg); err != nil { - return err - } - - // If ControlPlaneEndpoint is specified without a port number defaults it to - // the bindPort number of the APIEndpoint. - // This will allow join of additional control plane instances with different bindPort number - if cfg.ControlPlaneEndpoint != "" { - host, port, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint) - if err != nil { - return err - } - if port == "" { - cfg.ControlPlaneEndpoint = net.JoinHostPort(host, strconv.FormatInt(int64(localAPIEndpoint.BindPort), 10)) - } - } - - // Downcase SANs. Some domain names (like ELBs) have capitals in them. - LowercaseSANs(cfg.APIServer.CertSANs) - return nil -} - -// DefaultedInitConfiguration takes a versioned init config (often populated by flags), defaults it and converts it into internal InitConfiguration -func DefaultedInitConfiguration(versionedInitCfg *kubeadmapiv1beta2.InitConfiguration, versionedClusterCfg *kubeadmapiv1beta2.ClusterConfiguration) (*kubeadmapi.InitConfiguration, error) { - internalcfg := &kubeadmapi.InitConfiguration{} - - // Takes passed flags into account; the defaulting is executed once again enforcing assignment of - // static default values to cfg only for values not provided with flags - kubeadmscheme.Scheme.Default(versionedInitCfg) - if err := kubeadmscheme.Scheme.Convert(versionedInitCfg, internalcfg, nil); err != nil { - return nil, err - } - - kubeadmscheme.Scheme.Default(versionedClusterCfg) - if err := kubeadmscheme.Scheme.Convert(versionedClusterCfg, &internalcfg.ClusterConfiguration, nil); err != nil { - return nil, err - } - - // Applies dynamic defaults to settings not provided with flags - if err := SetInitDynamicDefaults(internalcfg); err != nil { - return nil, err - } - // Validates cfg (flags/configs + defaults + dynamic defaults) - if err := validation.ValidateInitConfiguration(internalcfg).ToAggregate(); err != nil { - return nil, err - } - return internalcfg, nil -} - -// LoadInitConfigurationFromFile loads a supported versioned InitConfiguration from a file, converts it into internal config, defaults it and verifies it. -func LoadInitConfigurationFromFile(cfgPath string) (*kubeadmapi.InitConfiguration, error) { - klog.V(1).Infof("loading configuration from %q", cfgPath) - - b, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, errors.Wrapf(err, "unable to read config from %q ", cfgPath) - } - - return BytesToInitConfiguration(b) -} - -// LoadOrDefaultInitConfiguration takes a path to a config file and a versioned configuration that can serve as the default config -// If cfgPath is specified, the versioned configs will always get overridden with the one in the file (specified by cfgPath). -// The external, versioned configuration is defaulted and converted to the internal type. -// Right thereafter, the configuration is defaulted again with dynamic values (like IP addresses of a machine, etc) -// Lastly, the internal config is validated and returned. -func LoadOrDefaultInitConfiguration(cfgPath string, versionedInitCfg *kubeadmapiv1beta2.InitConfiguration, versionedClusterCfg *kubeadmapiv1beta2.ClusterConfiguration) (*kubeadmapi.InitConfiguration, error) { - if cfgPath != "" { - // Loads configuration from config file, if provided - // Nb. --config overrides command line flags - return LoadInitConfigurationFromFile(cfgPath) - } - - return DefaultedInitConfiguration(versionedInitCfg, versionedClusterCfg) -} - -// BytesToInitConfiguration converts a byte slice to an internal, defaulted and validated InitConfiguration object. -// The map may contain many different YAML documents. These YAML documents are parsed one-by-one -// and well-known ComponentConfig GroupVersionKinds are stored inside of the internal InitConfiguration struct. -// The resulting InitConfiguration is then dynamically defaulted and validated prior to return. -func BytesToInitConfiguration(b []byte) (*kubeadmapi.InitConfiguration, error) { - gvkmap, err := kubeadmutil.SplitYAMLDocuments(b) - if err != nil { - return nil, err - } - - return documentMapToInitConfiguration(gvkmap, false) -} - -// documentMapToInitConfiguration converts a map of GVKs and YAML documents to defaulted and validated configuration object. -func documentMapToInitConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated bool) (*kubeadmapi.InitConfiguration, error) { - var initcfg *kubeadmapi.InitConfiguration - var clustercfg *kubeadmapi.ClusterConfiguration - - for gvk, fileContent := range gvkmap { - // first, check if this GVK is supported and possibly not deprecated - if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated); err != nil { - return nil, err - } - - // verify the validity of the YAML - strict.VerifyUnmarshalStrict(fileContent, gvk) - - if kubeadmutil.GroupVersionKindsHasInitConfiguration(gvk) { - // Set initcfg to an empty struct value the deserializer will populate - initcfg = &kubeadmapi.InitConfiguration{} - // Decode the bytes into the internal struct. Under the hood, the bytes will be unmarshalled into the - // right external version, defaulted, and converted into the internal version. - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), fileContent, initcfg); err != nil { - return nil, err - } - continue - } - if kubeadmutil.GroupVersionKindsHasClusterConfiguration(gvk) { - // Set clustercfg to an empty struct value the deserializer will populate - clustercfg = &kubeadmapi.ClusterConfiguration{} - // Decode the bytes into the internal struct. Under the hood, the bytes will be unmarshalled into the - // right external version, defaulted, and converted into the internal version. - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), fileContent, clustercfg); err != nil { - return nil, err - } - continue - } - - // If the group is neither a kubeadm core type or of a supported component config group, we dump a warning about it being ignored - if !componentconfigs.Scheme.IsGroupRegistered(gvk.Group) { - fmt.Printf("[config] WARNING: Ignored YAML document with GroupVersionKind %v\n", gvk) - } - } - - // Enforce that InitConfiguration and/or ClusterConfiguration has to exist among the YAML documents - if initcfg == nil && clustercfg == nil { - return nil, errors.New("no InitConfiguration or ClusterConfiguration kind was found in the YAML file") - } - - // If InitConfiguration wasn't given, default it by creating an external struct instance, default it and convert into the internal type - if initcfg == nil { - extinitcfg := &kubeadmapiv1beta2.InitConfiguration{} - kubeadmscheme.Scheme.Default(extinitcfg) - // Set initcfg to an empty struct value the deserializer will populate - initcfg = &kubeadmapi.InitConfiguration{} - if err := kubeadmscheme.Scheme.Convert(extinitcfg, initcfg, nil); err != nil { - return nil, err - } - } - // If ClusterConfiguration was given, populate it in the InitConfiguration struct - if clustercfg != nil { - initcfg.ClusterConfiguration = *clustercfg - } - - // Load any component configs - if err := componentconfigs.FetchFromDocumentMap(&initcfg.ClusterConfiguration, gvkmap); err != nil { - return nil, err - } - - // Applies dynamic defaults to settings not provided with flags - if err := SetInitDynamicDefaults(initcfg); err != nil { - return nil, err - } - - // Validates cfg (flags/configs + defaults + dynamic defaults) - if err := validation.ValidateInitConfiguration(initcfg).ToAggregate(); err != nil { - return nil, err - } - - return initcfg, nil -} - -// MarshalInitConfigurationToBytes marshals the internal InitConfiguration object to bytes. It writes the embedded -// ClusterConfiguration object with ComponentConfigs out as separate YAML documents -func MarshalInitConfigurationToBytes(cfg *kubeadmapi.InitConfiguration, gv schema.GroupVersion) ([]byte, error) { - initbytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg, gv, kubeadmscheme.Codecs) - if err != nil { - return []byte{}, err - } - allFiles := [][]byte{initbytes} - - // Exception: If the specified groupversion is targeting the internal type, don't print embedded ClusterConfiguration contents - // This is mostly used for unit testing. In a real scenario the internal version of the API is never marshalled as-is. - if gv.Version != runtime.APIVersionInternal { - clusterbytes, err := kubeadmutil.MarshalToYamlForCodecs(&cfg.ClusterConfiguration, gv, kubeadmscheme.Codecs) - if err != nil { - return []byte{}, err - } - allFiles = append(allFiles, clusterbytes) - } - return bytes.Join(allFiles, []byte(kubeadmconstants.YAMLDocumentSeparator)), nil -} diff --git a/cmd/kubeadm/app/util/config/initconfiguration_test.go b/cmd/kubeadm/app/util/config/initconfiguration_test.go deleted file mode 100644 index 435a3401e919c..0000000000000 --- a/cmd/kubeadm/app/util/config/initconfiguration_test.go +++ /dev/null @@ -1,186 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "bytes" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "sigs.k8s.io/yaml" -) - -func TestLoadInitConfigurationFromFile(t *testing.T) { - // Create temp folder for the test case - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir: %v", err) - } - defer os.RemoveAll(tmpdir) - - // cfgFiles is in cluster_test.go - var tests = []struct { - name string - fileContents []byte - expectErr bool - }{ - { - name: "v1beta1.partial1", - fileContents: cfgFiles["InitConfiguration_v1beta1"], - }, - { - name: "v1beta1.partial2", - fileContents: cfgFiles["ClusterConfiguration_v1beta1"], - }, - { - name: "v1beta1.full", - fileContents: bytes.Join([][]byte{ - cfgFiles["InitConfiguration_v1beta1"], - cfgFiles["ClusterConfiguration_v1beta1"], - cfgFiles["Kube-proxy_componentconfig"], - cfgFiles["Kubelet_componentconfig"], - }, []byte(constants.YAMLDocumentSeparator)), - }, - { - name: "v1beta2.partial1", - fileContents: cfgFiles["InitConfiguration_v1beta2"], - }, - { - name: "v1beta2.partial2", - fileContents: cfgFiles["ClusterConfiguration_v1beta2"], - }, - { - name: "v1beta2.full", - fileContents: bytes.Join([][]byte{ - cfgFiles["InitConfiguration_v1beta2"], - cfgFiles["ClusterConfiguration_v1beta2"], - cfgFiles["Kube-proxy_componentconfig"], - cfgFiles["Kubelet_componentconfig"], - }, []byte(constants.YAMLDocumentSeparator)), - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - cfgPath := filepath.Join(tmpdir, rt.name) - err := ioutil.WriteFile(cfgPath, rt.fileContents, 0644) - if err != nil { - t.Errorf("Couldn't create file: %v", err) - return - } - - obj, err := LoadInitConfigurationFromFile(cfgPath) - if rt.expectErr { - if err == nil { - t.Error("Unexpected success") - } - } else { - if err != nil { - t.Errorf("Error reading file: %v", err) - return - } - - if obj == nil { - t.Error("Unexpected nil return value") - } - } - }) - } -} - -func TestDefaultTaintsMarshaling(t *testing.T) { - tests := []struct { - desc string - cfg kubeadmapiv1beta2.InitConfiguration - expectedTaintCnt int - }{ - { - desc: "Uninitialized nodeRegistration field produces a single taint (the master one)", - cfg: kubeadmapiv1beta2.InitConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubeadm.k8s.io/v1beta2", - Kind: constants.InitConfigurationKind, - }, - }, - expectedTaintCnt: 1, - }, - { - desc: "Uninitialized taints field produces a single taint (the master one)", - cfg: kubeadmapiv1beta2.InitConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubeadm.k8s.io/v1beta2", - Kind: constants.InitConfigurationKind, - }, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{}, - }, - expectedTaintCnt: 1, - }, - { - desc: "Forsing taints to an empty slice produces no taints", - cfg: kubeadmapiv1beta2.InitConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubeadm.k8s.io/v1beta2", - Kind: constants.InitConfigurationKind, - }, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - Taints: []v1.Taint{}, - }, - }, - expectedTaintCnt: 0, - }, - { - desc: "Custom taints are used", - cfg: kubeadmapiv1beta2.InitConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubeadm.k8s.io/v1beta2", - Kind: constants.InitConfigurationKind, - }, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - Taints: []v1.Taint{ - {Key: "taint1"}, - {Key: "taint2"}, - }, - }, - }, - expectedTaintCnt: 2, - }, - } - - for _, tc := range tests { - t.Run(tc.desc, func(t *testing.T) { - b, err := yaml.Marshal(tc.cfg) - if err != nil { - t.Fatalf("unexpected error while marshalling to YAML: %v", err) - } - - cfg, err := BytesToInitConfiguration(b) - if err != nil { - t.Fatalf("unexpected error of BytesToInitConfiguration: %v\nconfig: %s", err, string(b)) - } - - if tc.expectedTaintCnt != len(cfg.NodeRegistration.Taints) { - t.Fatalf("unexpected taints count\nexpected: %d\ngot: %d\ntaints: %v", tc.expectedTaintCnt, len(cfg.NodeRegistration.Taints), cfg.NodeRegistration.Taints) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/config/joinconfiguration.go b/cmd/kubeadm/app/util/config/joinconfiguration.go deleted file mode 100644 index 531e45ebe9c9e..0000000000000 --- a/cmd/kubeadm/app/util/config/joinconfiguration.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "io/ioutil" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" -) - -// SetJoinDynamicDefaults checks and sets configuration values for the JoinConfiguration object -func SetJoinDynamicDefaults(cfg *kubeadmapi.JoinConfiguration) error { - addControlPlaneTaint := false - if cfg.ControlPlane != nil { - addControlPlaneTaint = true - } - if err := SetNodeRegistrationDynamicDefaults(&cfg.NodeRegistration, addControlPlaneTaint); err != nil { - return err - } - - return SetJoinControlPlaneDefaults(cfg.ControlPlane) -} - -// SetJoinControlPlaneDefaults checks and sets configuration values for the JoinControlPlane object -func SetJoinControlPlaneDefaults(cfg *kubeadmapi.JoinControlPlane) error { - if cfg != nil { - if err := SetAPIEndpointDynamicDefaults(&cfg.LocalAPIEndpoint); err != nil { - return err - } - } - return nil -} - -// LoadOrDefaultJoinConfiguration takes a path to a config file and a versioned configuration that can serve as the default config -// If cfgPath is specified, defaultversionedcfg will always get overridden. Otherwise, the default config (often populated by flags) will be used. -// Then the external, versioned configuration is defaulted and converted to the internal type. -// Right thereafter, the configuration is defaulted again with dynamic values (like IP addresses of a machine, etc) -// Lastly, the internal config is validated and returned. -func LoadOrDefaultJoinConfiguration(cfgPath string, defaultversionedcfg *kubeadmapiv1beta2.JoinConfiguration) (*kubeadmapi.JoinConfiguration, error) { - if cfgPath != "" { - // Loads configuration from config file, if provided - // Nb. --config overrides command line flags, TODO: fix this - return LoadJoinConfigurationFromFile(cfgPath) - } - - return DefaultedJoinConfiguration(defaultversionedcfg) -} - -// LoadJoinConfigurationFromFile loads versioned JoinConfiguration from file, converts it to internal, defaults and validates it -func LoadJoinConfigurationFromFile(cfgPath string) (*kubeadmapi.JoinConfiguration, error) { - klog.V(1).Infof("loading configuration from %q", cfgPath) - - b, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, errors.Wrapf(err, "unable to read config from %q ", cfgPath) - } - - gvkmap, err := kubeadmutil.SplitYAMLDocuments(b) - if err != nil { - return nil, err - } - - return documentMapToJoinConfiguration(gvkmap, false) -} - -// documentMapToJoinConfiguration takes a map between GVKs and YAML documents (as returned by SplitYAMLDocuments), -// finds a JoinConfiguration, decodes it, dynamically defaults it and then validates it prior to return. -func documentMapToJoinConfiguration(gvkmap kubeadmapi.DocumentMap, allowDeprecated bool) (*kubeadmapi.JoinConfiguration, error) { - joinBytes := []byte{} - for gvk, bytes := range gvkmap { - // not interested in anything other than JoinConfiguration - if gvk.Kind != constants.JoinConfigurationKind { - continue - } - - // check if this version is supported and possibly not deprecated - if err := validateSupportedVersion(gvk.GroupVersion(), allowDeprecated); err != nil { - return nil, err - } - - // verify the validity of the YAML - strict.VerifyUnmarshalStrict(bytes, gvk) - - joinBytes = bytes - } - - if len(joinBytes) == 0 { - return nil, errors.Errorf("no %s found in the supplied config", constants.JoinConfigurationKind) - } - - internalcfg := &kubeadmapi.JoinConfiguration{} - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), joinBytes, internalcfg); err != nil { - return nil, err - } - - // Applies dynamic defaults to settings not provided with flags - if err := SetJoinDynamicDefaults(internalcfg); err != nil { - return nil, err - } - // Validates cfg (flags/configs + defaults) - if err := validation.ValidateJoinConfiguration(internalcfg).ToAggregate(); err != nil { - return nil, err - } - - return internalcfg, nil -} - -// DefaultedJoinConfiguration takes a versioned JoinConfiguration (usually filled in by command line parameters), defaults it, converts it to internal and validates it -func DefaultedJoinConfiguration(defaultversionedcfg *kubeadmapiv1beta2.JoinConfiguration) (*kubeadmapi.JoinConfiguration, error) { - internalcfg := &kubeadmapi.JoinConfiguration{} - - // Takes passed flags into account; the defaulting is executed once again enforcing assignment of - // static default values to cfg only for values not provided with flags - kubeadmscheme.Scheme.Default(defaultversionedcfg) - kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil) - - // Applies dynamic defaults to settings not provided with flags - if err := SetJoinDynamicDefaults(internalcfg); err != nil { - return nil, err - } - // Validates cfg (flags/configs + defaults) - if err := validation.ValidateJoinConfiguration(internalcfg).ToAggregate(); err != nil { - return nil, err - } - - return internalcfg, nil -} diff --git a/cmd/kubeadm/app/util/config/joinconfiguration_test.go b/cmd/kubeadm/app/util/config/joinconfiguration_test.go deleted file mode 100644 index 5d52eb30011e8..0000000000000 --- a/cmd/kubeadm/app/util/config/joinconfiguration_test.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/lithammer/dedent" -) - -func TestLoadJoinConfigurationFromFile(t *testing.T) { - // Create temp folder for the test case - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir: %v", err) - } - defer os.RemoveAll(tmpdir) - - // cfgFiles is in cluster_test.go - var tests = []struct { - name string - fileContents string - expectErr bool - }{ - { - name: "empty file causes error", - expectErr: true, - }, - { - name: "Invalid v1beta1 causes error", - fileContents: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - `), - expectErr: true, - }, - { - name: "valid v1beta1 is loaded", - fileContents: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta1 - kind: JoinConfiguration - caCertPath: /etc/kubernetes/pki/ca.crt - discovery: - bootstrapToken: - apiServerEndpoint: kube-apiserver:6443 - token: abcdef.0123456789abcdef - unsafeSkipCAVerification: true - timeout: 5m0s - tlsBootstrapToken: abcdef.0123456789abcdef - `), - }, - { - name: "Invalid v1beta2 causes error", - fileContents: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: JoinConfiguration - `), - expectErr: true, - }, - { - name: "valid v1beta2 is loaded", - fileContents: dedent.Dedent(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: JoinConfiguration - caCertPath: /etc/kubernetes/pki/ca.crt - discovery: - bootstrapToken: - apiServerEndpoint: kube-apiserver:6443 - token: abcdef.0123456789abcdef - unsafeSkipCAVerification: true - timeout: 5m0s - tlsBootstrapToken: abcdef.0123456789abcdef - `), - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - cfgPath := filepath.Join(tmpdir, rt.name) - err := ioutil.WriteFile(cfgPath, []byte(rt.fileContents), 0644) - if err != nil { - t.Errorf("Couldn't create file: %v", err) - return - } - - obj, err := LoadJoinConfigurationFromFile(cfgPath) - if rt.expectErr { - if err == nil { - t.Error("Unexpected success") - } - } else { - if err != nil { - t.Errorf("Error reading file: %v", err) - return - } - - if obj == nil { - t.Error("Unexpected nil return value") - } - } - }) - } -} diff --git a/cmd/kubeadm/app/util/config/strict/BUILD b/cmd/kubeadm/app/util/config/strict/BUILD deleted file mode 100644 index 604d3f2c616f2..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["strict.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library", - "//cmd/kubeadm/app/componentconfigs:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["strict_test.go"], - data = glob(["testdata/**"]), - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", - "//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library", - "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/config/strict/strict.go b/cmd/kubeadm/app/util/config/strict/strict.go deleted file mode 100644 index e95fdc52c7c70..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/strict.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package strict - -import ( - "github.com/pkg/errors" - - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" - "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" - "sigs.k8s.io/yaml" -) - -// VerifyUnmarshalStrict takes a YAML byte slice and a GroupVersionKind and verifies if the YAML -// schema is known and if it unmarshals with strict mode. -// -// TODO(neolit123): The returned error here is currently ignored everywhere and a klog warning is thrown instead. -// We don't want to turn this into an actual error yet. Eventually this can be controlled with an optional CLI flag. -func VerifyUnmarshalStrict(bytes []byte, gvk schema.GroupVersionKind) error { - - var ( - iface interface{} - err error - ) - - iface, err = scheme.Scheme.New(gvk) - if err != nil { - iface, err = componentconfigs.Scheme.New(gvk) - if err != nil { - err := errors.Errorf("unknown configuration %#v for scheme definitions in %q and %q", - gvk, scheme.Scheme.Name(), componentconfigs.Scheme.Name()) - klog.Warning(err.Error()) - return err - } - } - - if err := yaml.UnmarshalStrict(bytes, iface); err != nil { - err := errors.Wrapf(err, "error unmarshaling configuration %#v", gvk) - klog.Warning(err.Error()) - return err - } - return nil -} diff --git a/cmd/kubeadm/app/util/config/strict/strict_test.go b/cmd/kubeadm/app/util/config/strict/strict_test.go deleted file mode 100644 index c2ebaef3d6c95..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/strict_test.go +++ /dev/null @@ -1,160 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package strict - -import ( - "io/ioutil" - "path/filepath" - "testing" - - "k8s.io/apimachinery/pkg/runtime/schema" - kubeproxyconfigv1alpha1 "k8s.io/kube-proxy/config/v1alpha1" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestVerifyUnmarshalStrict(t *testing.T) { - const ( - pathTestData = "testdata/" - ) - - var testFiles = []struct { - fileName string - kind string - groupVersion schema.GroupVersion - expectedError bool - }{ - // tests with file errors - { - fileName: "invalid_duplicate_field_clustercfg.yaml", - kind: constants.InitConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_duplicate_field_joincfg.yaml", - kind: constants.JoinConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_duplicate_field_kubeletcfg.yaml", - kind: "KubeletConfiguration", - groupVersion: kubeletconfigv1beta1.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_duplicate_field_kubeproxycfg.yaml", - kind: "KubeProxyConfiguration", - groupVersion: kubeproxyconfigv1alpha1.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_unknown_field_clustercfg.yaml", - kind: constants.ClusterConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_unknown_field_initcfg.yaml", - kind: constants.InitConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_unknown_field_joincfg.yaml", - kind: constants.JoinConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_unknown_field_kubeletcfg.yaml", - kind: "KubeletConfiguration", - groupVersion: kubeletconfigv1beta1.SchemeGroupVersion, - expectedError: true, - }, - { - fileName: "invalid_unknown_field_kubeproxycfg.yaml", - kind: "KubeProxyConfiguration", - groupVersion: kubeproxyconfigv1alpha1.SchemeGroupVersion, - expectedError: true, - }, - // test unknown groupVersion and kind - { - fileName: "valid_clustercfg.yaml", - kind: constants.ClusterConfigurationKind, - groupVersion: schema.GroupVersion{Group: "someGroup", Version: "v1"}, - expectedError: true, - }, - { - fileName: "valid_clustercfg.yaml", - kind: "SomeUnknownKind", - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: true, - }, - // valid tests - { - fileName: "valid_clustercfg.yaml", - kind: constants.ClusterConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: false, - }, - { - fileName: "valid_initcfg.yaml", - kind: constants.InitConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: false, - }, - { - fileName: "valid_joincfg.yaml", - kind: constants.JoinConfigurationKind, - groupVersion: kubeadmapiv1beta2.SchemeGroupVersion, - expectedError: false, - }, - { - fileName: "valid_kubeletcfg.yaml", - kind: "KubeletConfiguration", - groupVersion: kubeletconfigv1beta1.SchemeGroupVersion, - expectedError: false, - }, - { - fileName: "valid_kubeproxycfg.yaml", - kind: "KubeProxyConfiguration", - groupVersion: kubeproxyconfigv1alpha1.SchemeGroupVersion, - expectedError: false, - }, - } - - for _, test := range testFiles { - t.Run(test.fileName, func(t *testing.T) { - bytes, err := ioutil.ReadFile(filepath.Join(pathTestData, test.fileName)) - if err != nil { - t.Fatalf("couldn't read test data: %v", err) - } - gvk := schema.GroupVersionKind{ - Group: test.groupVersion.Group, - Version: test.groupVersion.Version, - Kind: test.kind, - } - err = VerifyUnmarshalStrict(bytes, gvk) - if (err != nil) != test.expectedError { - t.Errorf("expected error %v, got %v, error: %v", err != nil, test.expectedError, err) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_clustercfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_clustercfg.yaml deleted file mode 100644 index 7b17191c3cbde..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_clustercfg.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -controlPlaneEndpoint: test1 -controlPlaneEndpoint: test2 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_initcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_initcfg.yaml deleted file mode 100644 index 407c4593db72c..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_initcfg.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -bootstrapTokens: -- token: "9a08jv.c0izixklcxtmnze7" -bootstrapTokens: -- token: "9a08jv.c0izixklcxtmnze7" diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_joincfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_joincfg.yaml deleted file mode 100644 index 4c57c00a3974a..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_joincfg.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: JoinConfiguration -caCertPath: relativepath -caCertPath: relativepath diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeletcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeletcfg.yaml deleted file mode 100644 index 2578fb75dfa9b..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeletcfg.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -address: 1.2.3.4 -address: 1.2.3.4 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeproxycfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeproxycfg.yaml deleted file mode 100644 index 7e232dcc82328..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_duplicate_field_kubeproxycfg.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -portRange: "" -portRange: "" diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_clustercfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_clustercfg.yaml deleted file mode 100644 index 624bbcf5b9dc6..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_clustercfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -unknownField: test diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_initcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_initcfg.yaml deleted file mode 100644 index 185f02ba688ad..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_initcfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -unknownField: test diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_joincfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_joincfg.yaml deleted file mode 100644 index 2a10e38787ac1..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_joincfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: JoinConfiguration -unknownField: test diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeletcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeletcfg.yaml deleted file mode 100644 index 75e0e97b88b6f..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeletcfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -unknownField: unknown diff --git a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeproxycfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeproxycfg.yaml deleted file mode 100644 index 7529f30507fb8..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/invalid_unknown_field_kubeproxycfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -unknownField: unknown diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_clustercfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_clustercfg.yaml deleted file mode 100644 index 3b25a544fbda8..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/valid_clustercfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration -controlPlaneEndpoint: 202.0.100.1 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_initcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_initcfg.yaml deleted file mode 100644 index 97e95af2963ff..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/valid_initcfg.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration -bootstrapTokens: -- token: "9a08jv.c0izixklcxtmnze7" diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_joincfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_joincfg.yaml deleted file mode 100644 index cf39a0708b051..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/valid_joincfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: JoinConfiguration -caCertPath: relativepath diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeletcfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeletcfg.yaml deleted file mode 100644 index f85f7d52c09e1..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeletcfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -address: 1.2.3.4 diff --git a/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeproxycfg.yaml b/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeproxycfg.yaml deleted file mode 100644 index 955b5dd78e857..0000000000000 --- a/cmd/kubeadm/app/util/config/strict/testdata/valid_kubeproxycfg.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -portRange: "" diff --git a/cmd/kubeadm/app/util/copy.go b/cmd/kubeadm/app/util/copy.go deleted file mode 100644 index 6465547d2c204..0000000000000 --- a/cmd/kubeadm/app/util/copy.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "os/exec" -) - -// CopyDir copies the content of a folder -func CopyDir(src string, dst string) error { - cmd := exec.Command("cp", "-r", src, dst) - err := cmd.Run() - if err != nil { - return err - } - return nil -} diff --git a/cmd/kubeadm/app/util/crypto/BUILD b/cmd/kubeadm/app/util/crypto/BUILD deleted file mode 100644 index 8993845f279f0..0000000000000 --- a/cmd/kubeadm/app/util/crypto/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["crypto.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/crypto", - visibility = ["//visibility:public"], - deps = ["//vendor/github.com/pkg/errors:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = ["crypto_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - ], -) diff --git a/cmd/kubeadm/app/util/crypto/crypto.go b/cmd/kubeadm/app/util/crypto/crypto.go deleted file mode 100644 index c3b866bee2f94..0000000000000 --- a/cmd/kubeadm/app/util/crypto/crypto.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package crypto - -import ( - "github.com/pkg/errors" - - "crypto/aes" - "crypto/cipher" - "crypto/rand" -) - -// CreateRandBytes returns a cryptographically secure slice of random bytes with a given size -func CreateRandBytes(size uint32) ([]byte, error) { - bytes := make([]byte, size) - if _, err := rand.Read(bytes); err != nil { - return nil, err - } - return bytes, nil -} - -// EncryptBytes takes a byte slice of raw data and an encryption key and returns an encrypted byte slice of data. -// The key must be an AES key, either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256 -func EncryptBytes(data, key []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - nonce, err := CreateRandBytes(uint32(gcm.NonceSize())) - if err != nil { - return nil, err - } - return gcm.Seal(nonce, nonce, data, nil), nil -} - -// DecryptBytes takes a byte slice of encrypted data and an encryption key and returns a decrypted byte slice of data. -// The key must be an AES key, either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256 -func DecryptBytes(data, key []byte) ([]byte, error) { - block, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - gcm, err := cipher.NewGCM(block) - if err != nil { - return nil, err - } - nonceSize := gcm.NonceSize() - if len(data) < nonceSize { - return nil, errors.New("size of data is less than the nonce") - } - - nonce, out := data[:nonceSize], data[nonceSize:] - out, err = gcm.Open(nil, nonce, out, nil) - if err != nil { - return nil, err - } - return out, nil -} diff --git a/cmd/kubeadm/app/util/crypto/crypto_test.go b/cmd/kubeadm/app/util/crypto/crypto_test.go deleted file mode 100644 index 40fd29595455e..0000000000000 --- a/cmd/kubeadm/app/util/crypto/crypto_test.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package crypto - -import ( - "testing" - - "github.com/lithammer/dedent" - - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -func TestEncryptAndDecryptData(t *testing.T) { - key1, err := CreateRandBytes(kubeadmconstants.CertificateKeySize) - if err != nil { - t.Fatal(err) - } - key2, err := CreateRandBytes(kubeadmconstants.CertificateKeySize) - if err != nil { - t.Fatal(err) - } - testData := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit.") - - tests := map[string]struct { - encryptKey []byte - decryptKey []byte - data []byte - expectDecryptErr bool - }{ - "can decrypt using the correct key": { - encryptKey: key1, - decryptKey: key1, - data: testData, - expectDecryptErr: false, - }, - "can't decrypt using incorrect key": { - encryptKey: key1, - decryptKey: key2, - data: testData, - expectDecryptErr: true, - }, - "can't decrypt without a key": { - encryptKey: key1, - decryptKey: []byte{}, - data: testData, - expectDecryptErr: true, - }, - } - - for name, test := range tests { - t.Run(name, func(t2 *testing.T) { - encryptedData, err := EncryptBytes(test.data, test.encryptKey) - if err != nil { - t2.Fatalf(dedent.Dedent( - "EncryptBytes failed\nerror: %v"), - err, - ) - } - - decryptedData, err := DecryptBytes(encryptedData, test.decryptKey) - if (err != nil) != test.expectDecryptErr { - t2.Fatalf(dedent.Dedent( - "DecryptBytes failed\nexpected error: %t\n\tgot: %t\nerror: %v"), - test.expectDecryptErr, - (err != nil), - err, - ) - } - - if (string(decryptedData) != string(test.data)) && !test.expectDecryptErr { - t2.Fatalf(dedent.Dedent( - "EncryptDecryptBytes failed\nexpected decryptedData equal to data\n\tgot: data=%q decryptedData=%q"), - test.data, - string(decryptedData), - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/dryrun/BUILD b/cmd/kubeadm/app/util/dryrun/BUILD deleted file mode 100644 index 10b55121ff216..0000000000000 --- a/cmd/kubeadm/app/util/dryrun/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["dryrun.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/dryrun/dryrun.go b/cmd/kubeadm/app/util/dryrun/dryrun.go deleted file mode 100644 index aec1584e37866..0000000000000 --- a/cmd/kubeadm/app/util/dryrun/dryrun.go +++ /dev/null @@ -1,140 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package dryrun - -import ( - "fmt" - "io" - "io/ioutil" - "path/filepath" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - errorsutil "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// FileToPrint represents a temporary file on disk that might want to be aliased when printing -// Useful for things like loading a file from /tmp/ but saying to the user "Would write file foo to /etc/kubernetes/..." -type FileToPrint struct { - RealPath string - PrintPath string -} - -// NewFileToPrint makes a new instance of FileToPrint with the specified arguments -func NewFileToPrint(realPath, printPath string) FileToPrint { - return FileToPrint{ - RealPath: realPath, - PrintPath: printPath, - } -} - -// PrintDryRunFile is a helper method around PrintDryRunFiles -func PrintDryRunFile(fileName, realDir, printDir string, w io.Writer) error { - return PrintDryRunFiles([]FileToPrint{ - NewFileToPrint(filepath.Join(realDir, fileName), filepath.Join(printDir, fileName)), - }, w) -} - -// PrintDryRunFiles prints the contents of the FileToPrints given to it to the writer w -func PrintDryRunFiles(files []FileToPrint, w io.Writer) error { - errs := []error{} - for _, file := range files { - if len(file.RealPath) == 0 { - continue - } - - fileBytes, err := ioutil.ReadFile(file.RealPath) - if err != nil { - errs = append(errs, err) - continue - } - - // Make it possible to fake the path of the file; i.e. you may want to tell the user - // "Here is what would be written to /etc/kubernetes/admin.conf", although you wrote it to /tmp/kubeadm-dryrun/admin.conf and are loading it from there - // Fall back to the "real" path if PrintPath is not set - outputFilePath := file.PrintPath - if len(outputFilePath) == 0 { - outputFilePath = file.RealPath - } - - fmt.Fprintf(w, "[dryrun] Would write file %q with content:\n", outputFilePath) - apiclient.PrintBytesWithLinePrefix(w, fileBytes, "\t") - } - return errorsutil.NewAggregate(errs) -} - -// Waiter is an implementation of apiclient.Waiter that should be used for dry-running -type Waiter struct{} - -// NewWaiter returns a new Waiter object that talks to the given Kubernetes cluster -func NewWaiter() apiclient.Waiter { - return &Waiter{} -} - -// WaitForAPI just returns a dummy nil, to indicate that the program should just proceed -func (w *Waiter) WaitForAPI() error { - fmt.Println("[dryrun] Would wait for the API Server's /healthz endpoint to return 'ok'") - return nil -} - -// WaitForPodsWithLabel just returns a dummy nil, to indicate that the program should just proceed -func (w *Waiter) WaitForPodsWithLabel(kvLabel string) error { - fmt.Printf("[dryrun] Would wait for the Pods with the label %q in the %s namespace to become Running\n", kvLabel, metav1.NamespaceSystem) - return nil -} - -// WaitForPodToDisappear just returns a dummy nil, to indicate that the program should just proceed -func (w *Waiter) WaitForPodToDisappear(podName string) error { - fmt.Printf("[dryrun] Would wait for the %q Pod in the %s namespace to be deleted\n", podName, metav1.NamespaceSystem) - return nil -} - -// WaitForHealthyKubelet blocks until the kubelet /healthz endpoint returns 'ok' -func (w *Waiter) WaitForHealthyKubelet(_ time.Duration, healthzEndpoint string) error { - fmt.Printf("[dryrun] Would make sure the kubelet %q endpoint is healthy\n", healthzEndpoint) - return nil -} - -// WaitForKubeletAndFunc is a wrapper for WaitForHealthyKubelet that also blocks for a function -func (w *Waiter) WaitForKubeletAndFunc(f func() error) error { - return nil -} - -// SetTimeout is a no-op; we don't wait in this implementation -func (w *Waiter) SetTimeout(_ time.Duration) {} - -// WaitForStaticPodControlPlaneHashes returns an empty hash for all control plane images; -func (w *Waiter) WaitForStaticPodControlPlaneHashes(_ string) (map[string]string, error) { - return map[string]string{ - constants.KubeAPIServer: "", - constants.KubeControllerManager: "", - constants.KubeScheduler: "", - }, nil -} - -// WaitForStaticPodSingleHash returns an empty hash -// but the empty strings there are needed -func (w *Waiter) WaitForStaticPodSingleHash(_ string, _ string) (string, error) { - return "", nil -} - -// WaitForStaticPodHashChange returns a dummy nil error in order for the flow to just continue as we're dryrunning -func (w *Waiter) WaitForStaticPodHashChange(_, _, _ string) error { - return nil -} diff --git a/cmd/kubeadm/app/util/endpoint.go b/cmd/kubeadm/app/util/endpoint.go deleted file mode 100644 index d11060e5e2678..0000000000000 --- a/cmd/kubeadm/app/util/endpoint.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "net" - "net/url" - "strconv" - - "github.com/pkg/errors" - - "k8s.io/apimachinery/pkg/util/validation" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - utilsnet "k8s.io/utils/net" -) - -// GetControlPlaneEndpoint returns a properly formatted endpoint for the control plane built according following rules: -// - If the controlPlaneEndpoint is defined, use it. -// - if the controlPlaneEndpoint is defined but without a port number, use the controlPlaneEndpoint + localEndpoint.BindPort is used. -// - Otherwise, in case the controlPlaneEndpoint is not defined, use the localEndpoint.AdvertiseAddress + the localEndpoint.BindPort. -func GetControlPlaneEndpoint(controlPlaneEndpoint string, localEndpoint *kubeadmapi.APIEndpoint) (string, error) { - // get the URL of the local endpoint - localAPIEndpoint, err := GetLocalAPIEndpoint(localEndpoint) - if err != nil { - return "", err - } - - // if the controlplane endpoint is defined - if len(controlPlaneEndpoint) > 0 { - // parse the controlplane endpoint - var host, port string - var err error - if host, port, err = ParseHostPort(controlPlaneEndpoint); err != nil { - return "", errors.Wrapf(err, "invalid value %q given for controlPlaneEndpoint", controlPlaneEndpoint) - } - - // if a port is provided within the controlPlaneAddress warn the users we are using it, else use the bindport - localEndpointPort := strconv.Itoa(int(localEndpoint.BindPort)) - if port != "" { - if port != localEndpointPort { - fmt.Println("[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address") - } - } else { - port = localEndpointPort - } - - // overrides the control-plane url using the controlPlaneAddress (and eventually the bindport) - return formatURL(host, port).String(), nil - } - - return localAPIEndpoint, nil -} - -// GetLocalAPIEndpoint parses an APIEndpoint and returns it as a string, -// or returns and error in case it cannot be parsed. -func GetLocalAPIEndpoint(localEndpoint *kubeadmapi.APIEndpoint) (string, error) { - // get the URL of the local endpoint - localEndpointIP, localEndpointPort, err := parseAPIEndpoint(localEndpoint) - if err != nil { - return "", err - } - url := formatURL(localEndpointIP.String(), localEndpointPort) - return url.String(), nil -} - -// ParseHostPort parses a network address of the form "host:port", "ipv4:port", "[ipv6]:port" into host and port; -// ":port" can be eventually omitted. -// If the string is not a valid representation of network address, ParseHostPort returns an error. -func ParseHostPort(hostport string) (string, string, error) { - var host, port string - var err error - - // try to split host and port - if host, port, err = net.SplitHostPort(hostport); err != nil { - // if SplitHostPort returns an error, the entire hostport is considered as host - host = hostport - } - - // if port is defined, parse and validate it - if port != "" { - if _, err := ParsePort(port); err != nil { - return "", "", errors.Errorf("hostport %s: port %s must be a valid number between 1 and 65535, inclusive", hostport, port) - } - } - - // if host is a valid IP, returns it - if ip := net.ParseIP(host); ip != nil { - return host, port, nil - } - - // if host is a validate RFC-1123 subdomain, returns it - if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 { - return host, port, nil - } - - return "", "", errors.Errorf("hostport %s: host '%s' must be a valid IP address or a valid RFC-1123 DNS subdomain", hostport, host) -} - -// ParsePort parses a string representing a TCP port. -// If the string is not a valid representation of a TCP port, ParsePort returns an error. -func ParsePort(port string) (int, error) { - portInt, err := utilsnet.ParsePort(port, true) - if err == nil && (1 <= portInt && portInt <= 65535) { - return portInt, nil - } - - return 0, errors.New("port must be a valid number between 1 and 65535, inclusive") -} - -// parseAPIEndpoint parses an APIEndpoint and returns the AdvertiseAddress as net.IP and the BindPort as string. -// If the BindPort or AdvertiseAddress are invalid it returns an error. -func parseAPIEndpoint(localEndpoint *kubeadmapi.APIEndpoint) (net.IP, string, error) { - // parse the bind port - bindPortString := strconv.Itoa(int(localEndpoint.BindPort)) - if _, err := ParsePort(bindPortString); err != nil { - return nil, "", errors.Wrapf(err, "invalid value %q given for api.bindPort", localEndpoint.BindPort) - } - - // parse the AdvertiseAddress - var ip = net.ParseIP(localEndpoint.AdvertiseAddress) - if ip == nil { - return nil, "", errors.Errorf("invalid value `%s` given for api.advertiseAddress", localEndpoint.AdvertiseAddress) - } - - return ip, bindPortString, nil -} - -// formatURL takes a host and a port string and creates a net.URL using https scheme -func formatURL(host, port string) *url.URL { - return &url.URL{ - Scheme: "https", - Host: net.JoinHostPort(host, port), - } -} diff --git a/cmd/kubeadm/app/util/endpoint_test.go b/cmd/kubeadm/app/util/endpoint_test.go deleted file mode 100644 index 2227cb0c29ec3..0000000000000 --- a/cmd/kubeadm/app/util/endpoint_test.go +++ /dev/null @@ -1,391 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "testing" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestGetControlPlaneEndpoint(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.InitConfiguration - expectedEndpoint string - expectedError bool - }{ - { - name: "use ControlPlaneEndpoint (dns) if fully defined", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "cp.k8s.io:1234", - }, - }, - expectedEndpoint: "https://cp.k8s.io:1234", - }, - { - name: "use ControlPlaneEndpoint (ipv4) if fully defined", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "1.2.3.4:1234", - }, - }, - expectedEndpoint: "https://1.2.3.4:1234", - }, - { - name: "use ControlPlaneEndpoint (ipv6) if fully defined", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "[2001:db8::1]:1234", - }, - }, - expectedEndpoint: "https://[2001:db8::1]:1234", - }, - { - name: "use ControlPlaneEndpoint (dns) + BindPort if ControlPlaneEndpoint defined without port", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - - ControlPlaneEndpoint: "cp.k8s.io", - }, - }, - expectedEndpoint: "https://cp.k8s.io:4567", - }, - { - name: "use ControlPlaneEndpoint (ipv4) + BindPort if ControlPlaneEndpoint defined without port", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "1.2.3.4", - }, - }, - expectedEndpoint: "https://1.2.3.4:4567", - }, - { - name: "use ControlPlaneEndpoint (ipv6) + BindPort if ControlPlaneEndpoint defined without port", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - - ControlPlaneEndpoint: "2001:db8::1", - }, - }, - expectedEndpoint: "https://[2001:db8::1]:4567", - }, - { - name: "use AdvertiseAddress (ipv4) + BindPort if ControlPlaneEndpoint is not defined", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "4.5.6.7", - }, - }, - expectedEndpoint: "https://4.5.6.7:4567", - }, - { - name: "use AdvertiseAddress (ipv6) + BindPort if ControlPlaneEndpoint is not defined", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 4567, - AdvertiseAddress: "2001:db8::1", - }, - }, - expectedEndpoint: "https://[2001:db8::1]:4567", - }, - { - name: "fail if invalid BindPort", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - BindPort: 0, - }, - }, - expectedError: true, - }, - { - name: "fail if invalid ControlPlaneEndpoint (dns)", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "bad!!.cp.k8s.io", - }, - }, - expectedError: true, - }, - { - name: "fail if invalid ControlPlaneEndpoint (ip4)", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "1..0", - }, - }, - expectedError: true, - }, - { - name: "fail if invalid ControlPlaneEndpoint (ip6)", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "1200::AB00:1234::2552:7777:1313", - }, - }, - expectedError: true, - }, - { - name: "fail if invalid ControlPlaneEndpoint (port)", - cfg: &kubeadmapi.InitConfiguration{ - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "cp.k8s.io:0", - }, - }, - expectedError: true, - }, - { - name: "fail if invalid AdvertiseAddress (ip4)", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1..0", - BindPort: 4567, - }, - }, - expectedError: true, - }, - { - name: "fail if invalid AdvertiseAddress (ip6)", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1200::AB00:1234::2552:7777:1313", - BindPort: 4567, - }, - }, - expectedError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualEndpoint, actualError := GetControlPlaneEndpoint(rt.cfg.ControlPlaneEndpoint, &rt.cfg.LocalAPIEndpoint) - - if (actualError != nil) && !rt.expectedError { - t.Errorf("%s unexpected failure: %v", rt.name, actualError) - return - } else if (actualError == nil) && rt.expectedError { - t.Errorf("%s passed when expected to fail", rt.name) - return - } - - if actualEndpoint != rt.expectedEndpoint { - t.Errorf("%s returned invalid endpoint %s, expected %s", rt.name, actualEndpoint, rt.expectedEndpoint) - } - }) - } -} - -func TestParseHostPort(t *testing.T) { - - var tests = []struct { - name string - hostport string - expectedHost string - expectedPort string - expectedError bool - }{ - { - name: "valid dns", - hostport: "cp.k8s.io", - expectedHost: "cp.k8s.io", - expectedPort: "", - }, - { - name: "valid dns:port", - hostport: "cp.k8s.io:1234", - expectedHost: "cp.k8s.io", - expectedPort: "1234", - }, - { - name: "valid ip4", - hostport: "1.2.3.4", - expectedHost: "1.2.3.4", - expectedPort: "", - }, - { - name: "valid ipv4:port", - hostport: "1.2.3.4:1234", - expectedHost: "1.2.3.4", - expectedPort: "1234", - }, - { - name: "valid ipv6", - hostport: "2001:db8::1", - expectedHost: "2001:db8::1", - expectedPort: "", - }, - { - name: "valid ipv6:port", - hostport: "[2001:db8::1]:1234", - expectedHost: "2001:db8::1", - expectedPort: "1234", - }, - { - name: "invalid port(not a number)", - hostport: "cp.k8s.io:aaa", - expectedError: true, - }, - { - name: "invalid port(out of range, positive port number)", - hostport: "cp.k8s.io:987654321", - expectedError: true, - }, - { - name: "invalid port(out of range, negative port number)", - hostport: "cp.k8s.io:-987654321", - expectedError: true, - }, - { - name: "invalid port(out of range, negative port number)", - hostport: "cp.k8s.io:123:123", - expectedError: true, - }, - { - name: "invalid dns", - hostport: "bad!!cp.k8s.io", - expectedError: true, - }, - { - name: "invalid valid dns:port", - hostport: "bad!!cp.k8s.io:1234", - expectedError: true, - }, - { - name: "invalid ip4, but valid DNS", - hostport: "259.2.3.4", - expectedHost: "259.2.3.4", - }, - { - name: "invalid ip4", - hostport: "1..3.4", - expectedError: true, - }, - { - name: "invalid ip4(2):port", - hostport: "1..3.4:1234", - expectedError: true, - }, - { - name: "invalid ipv6", - hostport: "1200::AB00:1234::2552:7777:1313", - expectedError: true, - }, - { - name: "invalid ipv6:port", - hostport: "[1200::AB00:1234::2552:7777:1313]:1234", - expectedError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualHost, actualPort, actualError := ParseHostPort(rt.hostport) - - if (actualError != nil) && !rt.expectedError { - t.Errorf("%s unexpected failure: %v", rt.name, actualError) - return - } else if (actualError == nil) && rt.expectedError { - t.Errorf("%s passed when expected to fail", rt.name) - return - } - - if actualHost != rt.expectedHost { - t.Errorf("%s returned invalid host %s, expected %s", rt.name, actualHost, rt.expectedHost) - return - } - - if actualPort != rt.expectedPort { - t.Errorf("%s returned invalid port %s, expected %s", rt.name, actualPort, rt.expectedPort) - } - }) - } -} - -func TestParsePort(t *testing.T) { - - var tests = []struct { - name string - port string - expectedPort int - expectedError bool - }{ - { - name: "valid port", - port: "1234", - expectedPort: 1234, - }, - { - name: "invalid port (not a number)", - port: "a", - expectedError: true, - }, - { - name: "invalid port (<1)", - port: "-10", - expectedError: true, - }, - { - name: "invalid port (>65535)", - port: "66535", - expectedError: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actualPort, actualError := ParsePort(rt.port) - - if (actualError != nil) && !rt.expectedError { - t.Errorf("%s unexpected failure: %v", rt.name, actualError) - return - } else if (actualError == nil) && rt.expectedError { - t.Errorf("%s passed when expected to fail", rt.name) - return - } - - if actualPort != rt.expectedPort { - t.Errorf("%s returned invalid port %d, expected %d", rt.name, actualPort, rt.expectedPort) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/env.go b/cmd/kubeadm/app/util/env.go deleted file mode 100644 index c419627249310..0000000000000 --- a/cmd/kubeadm/app/util/env.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "os" - "strings" - - v1 "k8s.io/api/core/v1" -) - -// GetProxyEnvVars builds a list of environment variables in order to use the right proxy -func GetProxyEnvVars() []v1.EnvVar { - envs := []v1.EnvVar{} - for _, env := range os.Environ() { - pos := strings.Index(env, "=") - if pos == -1 { - // malformed environment variable, skip it. - continue - } - name := env[:pos] - value := env[pos+1:] - if strings.HasSuffix(strings.ToLower(name), "_proxy") && value != "" { - envVar := v1.EnvVar{Name: name, Value: value} - envs = append(envs, envVar) - } - } - return envs -} diff --git a/cmd/kubeadm/app/util/error.go b/cmd/kubeadm/app/util/error.go deleted file mode 100644 index 8757a3071a049..0000000000000 --- a/cmd/kubeadm/app/util/error.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "flag" - "fmt" - "os" - "strconv" - "strings" - - errorsutil "k8s.io/apimachinery/pkg/util/errors" -) - -const ( - // DefaultErrorExitCode defines exit the code for failed action generally - DefaultErrorExitCode = 1 - // PreFlightExitCode defines exit the code for preflight checks - PreFlightExitCode = 2 - // ValidationExitCode defines the exit code validation checks - ValidationExitCode = 3 -) - -// fatal prints the message if set and then exits. -func fatal(msg string, code int) { - if len(msg) > 0 { - // add newline if needed - if !strings.HasSuffix(msg, "\n") { - msg += "\n" - } - - fmt.Fprint(os.Stderr, msg) - } - os.Exit(code) -} - -// CheckErr prints a user friendly error to STDERR and exits with a non-zero -// exit code. Unrecognized errors will be printed with an "error: " prefix. -// -// This method is generic to the command in use and may be used by non-Kubectl -// commands. -func CheckErr(err error) { - checkErr(err, fatal) -} - -// preflightError allows us to know if the error is a preflight error or not -// defining the interface here avoids an import cycle of pulling in preflight into the util package -type preflightError interface { - Preflight() bool -} - -// checkErr formats a given error as a string and calls the passed handleErr -// func with that string and an exit code. -func checkErr(err error, handleErr func(string, int)) { - - var msg string - if err != nil { - msg = fmt.Sprintf("%s\nTo see the stack trace of this error execute with --v=5 or higher", err.Error()) - // check if the verbosity level in klog is high enough and print a stack trace. - f := flag.CommandLine.Lookup("v") - if f != nil { - // assume that the "v" flag contains a parseable Int32 as per klog's "Level" type alias, - // thus no error from ParseInt is handled here. - if v, e := strconv.ParseInt(f.Value.String(), 10, 32); e == nil { - // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md - // klog.V(5) - Trace level verbosity - if v > 4 { - msg = fmt.Sprintf("%+v", err) - } - } - } - } - - switch err.(type) { - case nil: - return - case preflightError: - handleErr(msg, PreFlightExitCode) - case errorsutil.Aggregate: - handleErr(msg, ValidationExitCode) - - default: - handleErr(msg, DefaultErrorExitCode) - } -} - -// FormatErrMsg returns a human-readable string describing the slice of errors passed to the function -func FormatErrMsg(errs []error) string { - var errMsg string - for _, err := range errs { - errMsg = fmt.Sprintf("%s\t- %s\n", errMsg, err.Error()) - } - return errMsg -} diff --git a/cmd/kubeadm/app/util/error_test.go b/cmd/kubeadm/app/util/error_test.go deleted file mode 100644 index 5532ae866c510..0000000000000 --- a/cmd/kubeadm/app/util/error_test.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "github.com/pkg/errors" - "testing" -) - -type pferror struct{} - -func (p *pferror) Preflight() bool { return true } -func (p *pferror) Error() string { return "" } -func TestCheckErr(t *testing.T) { - var codeReturned int - errHandle := func(err string, code int) { - codeReturned = code - } - - var tests = []struct { - name string - e error - expected int - }{ - {"error is nil", nil, 0}, - {"empty error", errors.New(""), DefaultErrorExitCode}, - {"preflight error", &pferror{}, PreFlightExitCode}, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - codeReturned = 0 - checkErr(rt.e, errHandle) - if codeReturned != rt.expected { - t.Errorf( - "failed checkErr:\n\texpected: %d\n\t actual: %d", - rt.expected, - codeReturned, - ) - } - }) - } -} - -func TestFormatErrMsg(t *testing.T) { - errMsg1 := "specified version to upgrade to v1.9.0-alpha.3 is equal to or lower than the cluster version v1.10.0-alpha.0.69+638add6ddfb6d2. Downgrades are not supported yet" - errMsg2 := "specified version to upgrade to v1.9.0-alpha.3 is higher than the kubeadm version v1.9.0-alpha.1.3121+84178212527295-dirty. Upgrade kubeadm first using the tool you used to install kubeadm" - - testCases := []struct { - name string - errs []error - expect string - }{ - { - name: "two errors", - errs: []error{ - errors.New(errMsg1), - errors.New(errMsg2), - }, - expect: "\t- " + errMsg1 + "\n" + "\t- " + errMsg2 + "\n", - }, - { - name: "one error", - errs: []error{ - errors.New(errMsg1), - }, - expect: "\t- " + errMsg1 + "\n", - }, - } - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - got := FormatErrMsg(testCase.errs) - if got != testCase.expect { - t.Errorf("FormatErrMsg error, expect: %v, got: %v", testCase.expect, got) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/etcd/BUILD b/cmd/kubeadm/app/util/etcd/BUILD deleted file mode 100644 index cf257239c832a..0000000000000 --- a/cmd/kubeadm/app/util/etcd/BUILD +++ /dev/null @@ -1,55 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "etcd.go", - "etcddata.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/go.etcd.io/etcd/clientv3:go_default_library", - "//vendor/go.etcd.io/etcd/pkg/transport:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["etcd_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/test/resources:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", - "//staging/src/k8s.io/client-go/testing:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/etcd/etcd.go b/cmd/kubeadm/app/util/etcd/etcd.go deleted file mode 100644 index b9f9c2f27633c..0000000000000 --- a/cmd/kubeadm/app/util/etcd/etcd.go +++ /dev/null @@ -1,520 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package etcd - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/url" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/pkg/errors" - "go.etcd.io/etcd/clientv3" - "go.etcd.io/etcd/pkg/transport" - "google.golang.org/grpc" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/util/config" -) - -const etcdTimeout = 2 * time.Second - -// Exponential backoff for etcd operations (up to ~200 seconds) -var etcdBackoff = wait.Backoff{ - Steps: 18, - Duration: 100 * time.Millisecond, - Factor: 1.5, - Jitter: 0.1, -} - -// ClusterInterrogator is an interface to get etcd cluster related information -type ClusterInterrogator interface { - CheckClusterHealth() error - WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) - Sync() error - ListMembers() ([]Member, error) - AddMember(name string, peerAddrs string) ([]Member, error) - GetMemberID(peerURL string) (uint64, error) - RemoveMember(id uint64) ([]Member, error) -} - -// Client provides connection parameters for an etcd cluster -type Client struct { - Endpoints []string - TLS *tls.Config -} - -// New creates a new EtcdCluster client -func New(endpoints []string, ca, cert, key string) (*Client, error) { - client := Client{Endpoints: endpoints} - - if ca != "" || cert != "" || key != "" { - tlsInfo := transport.TLSInfo{ - CertFile: cert, - KeyFile: key, - TrustedCAFile: ca, - } - tlsConfig, err := tlsInfo.ClientConfig() - if err != nil { - return nil, err - } - client.TLS = tlsConfig - } - - return &client, nil -} - -// NewFromCluster creates an etcd client for the etcd endpoints present in etcd member list. In order to compose this information, -// it will first discover at least one etcd endpoint to connect to. Once created, the client synchronizes client's endpoints with -// the known endpoints from the etcd membership API, since it is the authoritative source of truth for the list of available members. -func NewFromCluster(client clientset.Interface, certificatesDir string) (*Client, error) { - // Discover at least one etcd endpoint to connect to by inspecting the existing etcd pods - - // Get the list of etcd endpoints - endpoints, err := getEtcdEndpoints(client) - if err != nil { - return nil, err - } - klog.V(1).Infof("etcd endpoints read from pods: %s", strings.Join(endpoints, ",")) - - // Creates an etcd client - etcdClient, err := New( - endpoints, - filepath.Join(certificatesDir, constants.EtcdCACertName), - filepath.Join(certificatesDir, constants.EtcdHealthcheckClientCertName), - filepath.Join(certificatesDir, constants.EtcdHealthcheckClientKeyName), - ) - if err != nil { - return nil, errors.Wrapf(err, "error creating etcd client for %v endpoints", endpoints) - } - - // synchronizes client's endpoints with the known endpoints from the etcd membership. - err = etcdClient.Sync() - if err != nil { - return nil, errors.Wrap(err, "error syncing endpoints with etcd") - } - klog.V(1).Infof("update etcd endpoints: %s", strings.Join(etcdClient.Endpoints, ",")) - - return etcdClient, nil -} - -// getEtcdEndpoints returns the list of etcd endpoints. -func getEtcdEndpoints(client clientset.Interface) ([]string, error) { - return getEtcdEndpointsWithBackoff(client, constants.StaticPodMirroringDefaultRetry) -} - -func getEtcdEndpointsWithBackoff(client clientset.Interface, backoff wait.Backoff) ([]string, error) { - etcdEndpoints, err := getRawEtcdEndpointsFromPodAnnotation(client, backoff) - if err != nil { - // NB: this is a fallback when there is no annotation found in the etcd pods that contains - // the client URL, and so we fallback to reading the ClusterStatus struct present in the - // kubeadm-config ConfigMap. This can happen for example, when performing the first - // `kubeadm upgrade apply`. This logic will be removed when the cluster status struct - // is removed from the kubeadm-config ConfigMap. - return getRawEtcdEndpointsFromClusterStatus(client) - } - return etcdEndpoints, nil -} - -// getRawEtcdEndpointsFromPodAnnotation returns the list of endpoints as reported on etcd's pod annotations using the given backoff -func getRawEtcdEndpointsFromPodAnnotation(client clientset.Interface, backoff wait.Backoff) ([]string, error) { - etcdEndpoints := []string{} - var lastErr error - // Let's tolerate some unexpected transient failures from the API server or load balancers. Also, if - // static pods were not yet mirrored into the API server we want to wait for this propagation. - err := wait.ExponentialBackoff(backoff, func() (bool, error) { - var overallEtcdPodCount int - if etcdEndpoints, overallEtcdPodCount, lastErr = getRawEtcdEndpointsFromPodAnnotationWithoutRetry(client); lastErr != nil { - return false, nil - } - // TODO (ereslibre): this logic will need tweaking once that we get rid of the ClusterStatus, since we won't have - // the ClusterStatus safety net we will have to retry in both cases. - if len(etcdEndpoints) == 0 { - if overallEtcdPodCount == 0 { - return false, nil - } - // Fail fast scenario, to be removed once we get rid of the ClusterStatus - return true, errors.New("etcd Pods exist, but no etcd endpoint annotations were found") - } - return true, nil - }) - if err != nil { - if lastErr != nil { - return []string{}, errors.Wrap(lastErr, "could not retrieve the list of etcd endpoints") - } - return []string{}, errors.Wrap(err, "could not retrieve the list of etcd endpoints") - } - return etcdEndpoints, nil -} - -// getRawEtcdEndpointsFromPodAnnotationWithoutRetry returns the list of etcd endpoints as reported by etcd Pod annotations, -// along with the number of global etcd pods. This allows for callers to tell the difference between "no endpoints found", -// and "no endpoints found and pods were listed", so they can skip retrying. -func getRawEtcdEndpointsFromPodAnnotationWithoutRetry(client clientset.Interface) ([]string, int, error) { - klog.V(3).Infof("retrieving etcd endpoints from %q annotation in etcd Pods", constants.EtcdAdvertiseClientUrlsAnnotationKey) - podList, err := client.CoreV1().Pods(metav1.NamespaceSystem).List( - context.TODO(), - metav1.ListOptions{ - LabelSelector: fmt.Sprintf("component=%s,tier=%s", constants.Etcd, constants.ControlPlaneTier), - }, - ) - if err != nil { - return []string{}, 0, err - } - etcdEndpoints := []string{} - for _, pod := range podList.Items { - etcdEndpoint, ok := pod.ObjectMeta.Annotations[constants.EtcdAdvertiseClientUrlsAnnotationKey] - if !ok { - klog.V(3).Infof("etcd Pod %q is missing the %q annotation; cannot infer etcd advertise client URL using the Pod annotation", pod.ObjectMeta.Name, constants.EtcdAdvertiseClientUrlsAnnotationKey) - continue - } - etcdEndpoints = append(etcdEndpoints, etcdEndpoint) - } - return etcdEndpoints, len(podList.Items), nil -} - -// TODO: remove after 1.20, when the ClusterStatus struct is removed from the kubeadm-config ConfigMap. -func getRawEtcdEndpointsFromClusterStatus(client clientset.Interface) ([]string, error) { - klog.V(3).Info("retrieving etcd endpoints from the cluster status") - clusterStatus, err := config.GetClusterStatus(client) - if err != nil { - return []string{}, err - } - etcdEndpoints := []string{} - for _, e := range clusterStatus.APIEndpoints { - etcdEndpoints = append(etcdEndpoints, GetClientURLByIP(e.AdvertiseAddress)) - } - return etcdEndpoints, nil -} - -// Sync synchronizes client's endpoints with the known endpoints from the etcd membership. -func (c *Client) Sync() error { - // Syncs the list of endpoints - var cli *clientv3.Client - var lastError error - err := wait.ExponentialBackoff(etcdBackoff, func() (bool, error) { - var err error - cli, err = clientv3.New(clientv3.Config{ - Endpoints: c.Endpoints, - DialTimeout: etcdTimeout, - DialOptions: []grpc.DialOption{ - grpc.WithBlock(), // block until the underlying connection is up - }, - TLS: c.TLS, - }) - if err != nil { - lastError = err - return false, nil - } - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) - err = cli.Sync(ctx) - cancel() - if err == nil { - return true, nil - } - klog.V(5).Infof("Failed to sync etcd endpoints: %v", err) - lastError = err - return false, nil - }) - if err != nil { - return lastError - } - klog.V(1).Infof("etcd endpoints read from etcd: %s", strings.Join(cli.Endpoints(), ",")) - - c.Endpoints = cli.Endpoints() - return nil -} - -// Member struct defines an etcd member; it is used for avoiding to spread github.com/coreos/etcd dependency -// across kubeadm codebase -type Member struct { - Name string - PeerURL string -} - -func (c *Client) listMembers() (*clientv3.MemberListResponse, error) { - // Gets the member list - var lastError error - var resp *clientv3.MemberListResponse - err := wait.ExponentialBackoff(etcdBackoff, func() (bool, error) { - cli, err := clientv3.New(clientv3.Config{ - Endpoints: c.Endpoints, - DialTimeout: etcdTimeout, - DialOptions: []grpc.DialOption{ - grpc.WithBlock(), // block until the underlying connection is up - }, - TLS: c.TLS, - }) - if err != nil { - lastError = err - return false, nil - } - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) - resp, err = cli.MemberList(ctx) - cancel() - if err == nil { - return true, nil - } - klog.V(5).Infof("Failed to get etcd member list: %v", err) - lastError = err - return false, nil - }) - if err != nil { - return nil, lastError - } - return resp, nil -} - -// GetMemberID returns the member ID of the given peer URL -func (c *Client) GetMemberID(peerURL string) (uint64, error) { - resp, err := c.listMembers() - if err != nil { - return 0, err - } - - for _, member := range resp.Members { - if member.GetPeerURLs()[0] == peerURL { - return member.GetID(), nil - } - } - return 0, nil -} - -// ListMembers returns the member list. -func (c *Client) ListMembers() ([]Member, error) { - resp, err := c.listMembers() - if err != nil { - return nil, err - } - - ret := make([]Member, 0, len(resp.Members)) - for _, m := range resp.Members { - ret = append(ret, Member{Name: m.Name, PeerURL: m.PeerURLs[0]}) - } - return ret, nil -} - -// RemoveMember notifies an etcd cluster to remove an existing member -func (c *Client) RemoveMember(id uint64) ([]Member, error) { - // Remove an existing member from the cluster - var lastError error - var resp *clientv3.MemberRemoveResponse - err := wait.ExponentialBackoff(etcdBackoff, func() (bool, error) { - cli, err := clientv3.New(clientv3.Config{ - Endpoints: c.Endpoints, - DialTimeout: etcdTimeout, - DialOptions: []grpc.DialOption{ - grpc.WithBlock(), // block until the underlying connection is up - }, - TLS: c.TLS, - }) - if err != nil { - lastError = err - return false, nil - } - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) - resp, err = cli.MemberRemove(ctx, id) - cancel() - if err == nil { - return true, nil - } - klog.V(5).Infof("Failed to remove etcd member: %v", err) - lastError = err - return false, nil - }) - if err != nil { - return nil, lastError - } - - // Returns the updated list of etcd members - ret := []Member{} - for _, m := range resp.Members { - ret = append(ret, Member{Name: m.Name, PeerURL: m.PeerURLs[0]}) - } - - return ret, nil -} - -// AddMember notifies an existing etcd cluster that a new member is joining -func (c *Client) AddMember(name string, peerAddrs string) ([]Member, error) { - // Parse the peer address, required to add the client URL later to the list - // of endpoints for this client. Parsing as a first operation to make sure that - // if this fails no member addition is performed on the etcd cluster. - parsedPeerAddrs, err := url.Parse(peerAddrs) - if err != nil { - return nil, errors.Wrapf(err, "error parsing peer address %s", peerAddrs) - } - - // Adds a new member to the cluster - var lastError error - var resp *clientv3.MemberAddResponse - err = wait.ExponentialBackoff(etcdBackoff, func() (bool, error) { - cli, err := clientv3.New(clientv3.Config{ - Endpoints: c.Endpoints, - DialTimeout: etcdTimeout, - DialOptions: []grpc.DialOption{ - grpc.WithBlock(), // block until the underlying connection is up - }, - TLS: c.TLS, - }) - if err != nil { - lastError = err - return false, nil - } - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) - resp, err = cli.MemberAdd(ctx, []string{peerAddrs}) - cancel() - if err == nil { - return true, nil - } - klog.V(5).Infof("Failed to add etcd member: %v", err) - lastError = err - return false, nil - }) - if err != nil { - return nil, lastError - } - - // Returns the updated list of etcd members - ret := []Member{} - for _, m := range resp.Members { - // If the peer address matches, this is the member we are adding. - // Use the name we passed to the function. - if peerAddrs == m.PeerURLs[0] { - ret = append(ret, Member{Name: name, PeerURL: peerAddrs}) - continue - } - // Otherwise, we are processing other existing etcd members returned by AddMembers. - memberName := m.Name - // In some cases during concurrent join, some members can end up without a name. - // Use the member ID as name for those. - if len(memberName) == 0 { - memberName = strconv.FormatUint(m.ID, 16) - } - ret = append(ret, Member{Name: memberName, PeerURL: m.PeerURLs[0]}) - } - - // Add the new member client address to the list of endpoints - c.Endpoints = append(c.Endpoints, GetClientURLByIP(parsedPeerAddrs.Hostname())) - - return ret, nil -} - -// CheckClusterHealth returns nil for status Up or error for status Down -func (c *Client) CheckClusterHealth() error { - _, err := c.getClusterStatus() - return err -} - -// getClusterStatus returns nil for status Up (along with endpoint status response map) or error for status Down -func (c *Client) getClusterStatus() (map[string]*clientv3.StatusResponse, error) { - clusterStatus := make(map[string]*clientv3.StatusResponse) - for _, ep := range c.Endpoints { - // Gets the member status - var lastError error - var resp *clientv3.StatusResponse - err := wait.ExponentialBackoff(etcdBackoff, func() (bool, error) { - cli, err := clientv3.New(clientv3.Config{ - Endpoints: c.Endpoints, - DialTimeout: etcdTimeout, - DialOptions: []grpc.DialOption{ - grpc.WithBlock(), // block until the underlying connection is up - }, - TLS: c.TLS, - }) - if err != nil { - lastError = err - return false, nil - } - defer cli.Close() - - ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) - resp, err = cli.Status(ctx, ep) - cancel() - if err == nil { - return true, nil - } - klog.V(5).Infof("Failed to get etcd status for %s: %v", ep, err) - lastError = err - return false, nil - }) - if err != nil { - return nil, lastError - } - - clusterStatus[ep] = resp - } - return clusterStatus, nil -} - -// WaitForClusterAvailable returns true if all endpoints in the cluster are available after retry attempts, an error is returned otherwise -func (c *Client) WaitForClusterAvailable(retries int, retryInterval time.Duration) (bool, error) { - for i := 0; i < retries; i++ { - if i > 0 { - klog.V(1).Infof("[etcd] Waiting %v until next retry\n", retryInterval) - time.Sleep(retryInterval) - } - klog.V(2).Infof("[etcd] attempting to see if all cluster endpoints (%s) are available %d/%d", c.Endpoints, i+1, retries) - _, err := c.getClusterStatus() - if err != nil { - switch err { - case context.DeadlineExceeded: - klog.V(1).Infof("[etcd] Attempt timed out") - default: - klog.V(1).Infof("[etcd] Attempt failed with error: %v\n", err) - } - continue - } - return true, nil - } - return false, errors.New("timeout waiting for etcd cluster to be available") -} - -// GetClientURL creates an HTTPS URL that uses the configured advertise -// address and client port for the API controller -func GetClientURL(localEndpoint *kubeadmapi.APIEndpoint) string { - return "https://" + net.JoinHostPort(localEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenClientPort)) -} - -// GetPeerURL creates an HTTPS URL that uses the configured advertise -// address and peer port for the API controller -func GetPeerURL(localEndpoint *kubeadmapi.APIEndpoint) string { - return "https://" + net.JoinHostPort(localEndpoint.AdvertiseAddress, strconv.Itoa(constants.EtcdListenPeerPort)) -} - -// GetClientURLByIP creates an HTTPS URL based on an IP address -// and the client listening port. -func GetClientURLByIP(ip string) string { - return "https://" + net.JoinHostPort(ip, strconv.Itoa(constants.EtcdListenClientPort)) -} diff --git a/cmd/kubeadm/app/util/etcd/etcd_test.go b/cmd/kubeadm/app/util/etcd/etcd_test.go deleted file mode 100644 index 0e1f892f9d368..0000000000000 --- a/cmd/kubeadm/app/util/etcd/etcd_test.go +++ /dev/null @@ -1,330 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package etcd - -import ( - "fmt" - "reflect" - "strconv" - "testing" - - "github.com/pkg/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" - clientsetfake "k8s.io/client-go/kubernetes/fake" - clienttesting "k8s.io/client-go/testing" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - testresources "k8s.io/kubernetes/cmd/kubeadm/test/resources" -) - -func testGetURL(t *testing.T, getURLFunc func(*kubeadmapi.APIEndpoint) string, port int) { - portStr := strconv.Itoa(port) - var tests = []struct { - name string - advertiseAddress string - expectedURL string - }{ - { - name: "IPv4", - advertiseAddress: "10.10.10.10", - expectedURL: fmt.Sprintf("https://10.10.10.10:%s", portStr), - }, - { - name: "IPv6", - advertiseAddress: "2001:db8::2", - expectedURL: fmt.Sprintf("https://[2001:db8::2]:%s", portStr), - }, - { - name: "IPv4 localhost", - advertiseAddress: "127.0.0.1", - expectedURL: fmt.Sprintf("https://127.0.0.1:%s", portStr), - }, - { - name: "IPv6 localhost", - advertiseAddress: "::1", - expectedURL: fmt.Sprintf("https://[::1]:%s", portStr), - }, - } - - for _, test := range tests { - url := getURLFunc(&kubeadmapi.APIEndpoint{AdvertiseAddress: test.advertiseAddress}) - if url != test.expectedURL { - t.Errorf("expected %s, got %s", test.expectedURL, url) - } - } -} - -func TestGetClientURL(t *testing.T) { - testGetURL(t, GetClientURL, constants.EtcdListenClientPort) -} - -func TestGetPeerURL(t *testing.T) { - testGetURL(t, GetClientURL, constants.EtcdListenClientPort) -} - -func TestGetClientURLByIP(t *testing.T) { - portStr := strconv.Itoa(constants.EtcdListenClientPort) - var tests = []struct { - name string - ip string - expectedURL string - }{ - { - name: "IPv4", - ip: "10.10.10.10", - expectedURL: fmt.Sprintf("https://10.10.10.10:%s", portStr), - }, - { - name: "IPv6", - ip: "2001:db8::2", - expectedURL: fmt.Sprintf("https://[2001:db8::2]:%s", portStr), - }, - { - name: "IPv4 localhost", - ip: "127.0.0.1", - expectedURL: fmt.Sprintf("https://127.0.0.1:%s", portStr), - }, - { - name: "IPv6 localhost", - ip: "::1", - expectedURL: fmt.Sprintf("https://[::1]:%s", portStr), - }, - } - - for _, test := range tests { - url := GetClientURLByIP(test.ip) - if url != test.expectedURL { - t.Errorf("expected %s, got %s", test.expectedURL, url) - } - } -} - -func TestGetEtcdEndpointsWithBackoff(t *testing.T) { - var tests = []struct { - name string - pods []testresources.FakeStaticPod - configMap *testresources.FakeConfigMap - expectedEndpoints []string - expectedErr bool - }{ - { - name: "no pod annotations; no ClusterStatus", - expectedEndpoints: []string{}, - }, - { - name: "ipv4 endpoint in pod annotation; no ClusterStatus; port is preserved", - pods: []testresources.FakeStaticPod{ - { - Component: constants.Etcd, - Annotations: map[string]string{ - constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:1234", - }, - }, - }, - expectedEndpoints: []string{"https://1.2.3.4:1234"}, - }, - { - name: "no pod annotations; ClusterStatus with valid ipv4 endpoint; port is inferred", - configMap: testresources.ClusterStatusWithAPIEndpoint("cp-0", kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}), - expectedEndpoints: []string{"https://1.2.3.4:2379"}, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - for _, pod := range rt.pods { - if err := pod.Create(client); err != nil { - t.Errorf("error setting up test creating pod for node %q", pod.NodeName) - } - } - if rt.configMap != nil { - if err := rt.configMap.Create(client); err != nil { - t.Error("could not create ConfigMap") - } - } - endpoints, err := getEtcdEndpointsWithBackoff(client, wait.Backoff{Duration: 0, Jitter: 0, Steps: 1}) - if err != nil && !rt.expectedErr { - t.Errorf("got error %q; was expecting no errors", err) - return - } else if err == nil && rt.expectedErr { - t.Error("got no error; was expecting an error") - return - } else if err != nil && rt.expectedErr { - return - } - - if !reflect.DeepEqual(endpoints, rt.expectedEndpoints) { - t.Errorf("expected etcd endpoints: %v; got: %v", rt.expectedEndpoints, endpoints) - } - }) - } -} - -func TestGetRawEtcdEndpointsFromPodAnnotation(t *testing.T) { - var tests = []struct { - name string - pods []testresources.FakeStaticPod - clientSetup func(*clientsetfake.Clientset) - expectedEndpoints []string - expectedErr bool - }{ - { - name: "exactly one pod with annotation", - pods: []testresources.FakeStaticPod{ - { - NodeName: "cp-0", - Component: constants.Etcd, - Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"}, - }, - }, - expectedEndpoints: []string{"https://1.2.3.4:2379"}, - }, - { - name: "no pods with annotation", - expectedErr: true, - }, - { - name: "exactly one pod with annotation; all requests fail", - pods: []testresources.FakeStaticPod{ - { - NodeName: "cp-0", - Component: constants.Etcd, - Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"}, - }, - }, - clientSetup: func(clientset *clientsetfake.Clientset) { - clientset.PrependReactor("list", "pods", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apierrors.NewInternalError(errors.New("API server down")) - }) - }, - expectedErr: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - for i, pod := range rt.pods { - if err := pod.CreateWithPodSuffix(client, strconv.Itoa(i)); err != nil { - t.Errorf("error setting up test creating pod for node %q", pod.NodeName) - } - } - if rt.clientSetup != nil { - rt.clientSetup(client) - } - endpoints, err := getRawEtcdEndpointsFromPodAnnotation(client, wait.Backoff{Duration: 0, Jitter: 0, Steps: 1}) - if err != nil && !rt.expectedErr { - t.Errorf("got error %v, but wasn't expecting any error", err) - return - } else if err == nil && rt.expectedErr { - t.Error("didn't get any error; but was expecting an error") - return - } else if err != nil && rt.expectedErr { - return - } - if !reflect.DeepEqual(endpoints, rt.expectedEndpoints) { - t.Errorf("expected etcd endpoints: %v; got: %v", rt.expectedEndpoints, endpoints) - } - }) - } -} - -func TestGetRawEtcdEndpointsFromPodAnnotationWithoutRetry(t *testing.T) { - var tests = []struct { - name string - pods []testresources.FakeStaticPod - clientSetup func(*clientsetfake.Clientset) - expectedEndpoints []string - expectedErr bool - }{ - { - name: "no pods", - expectedEndpoints: []string{}, - }, - { - name: "exactly one pod with annotation", - pods: []testresources.FakeStaticPod{ - { - NodeName: "cp-0", - Component: constants.Etcd, - Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"}, - }, - }, - expectedEndpoints: []string{"https://1.2.3.4:2379"}, - }, - { - name: "two pods with annotation", - pods: []testresources.FakeStaticPod{ - { - NodeName: "cp-0", - Component: constants.Etcd, - Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"}, - }, - { - NodeName: "cp-1", - Component: constants.Etcd, - Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.5:2379"}, - }, - }, - expectedEndpoints: []string{"https://1.2.3.4:2379", "https://1.2.3.5:2379"}, - }, - { - name: "exactly one pod with annotation; request fails", - pods: []testresources.FakeStaticPod{ - { - NodeName: "cp-0", - Component: constants.Etcd, - Annotations: map[string]string{constants.EtcdAdvertiseClientUrlsAnnotationKey: "https://1.2.3.4:2379"}, - }, - }, - clientSetup: func(clientset *clientsetfake.Clientset) { - clientset.PrependReactor("list", "pods", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, apierrors.NewInternalError(errors.New("API server down")) - }) - }, - expectedErr: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - client := clientsetfake.NewSimpleClientset() - for _, pod := range rt.pods { - if err := pod.Create(client); err != nil { - t.Errorf("error setting up test creating pod for node %q", pod.NodeName) - return - } - } - if rt.clientSetup != nil { - rt.clientSetup(client) - } - endpoints, _, err := getRawEtcdEndpointsFromPodAnnotationWithoutRetry(client) - if err != nil && !rt.expectedErr { - t.Errorf("got error %v, but wasn't expecting any error", err) - return - } else if err == nil && rt.expectedErr { - t.Error("didn't get any error; but was expecting an error") - return - } else if err != nil && rt.expectedErr { - return - } - if !reflect.DeepEqual(endpoints, rt.expectedEndpoints) { - t.Errorf("expected etcd endpoints: %v; got: %v", rt.expectedEndpoints, endpoints) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/etcd/etcddata.go b/cmd/kubeadm/app/util/etcd/etcddata.go deleted file mode 100644 index 29538620ecd74..0000000000000 --- a/cmd/kubeadm/app/util/etcd/etcddata.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package etcd - -import ( - "os" - - "github.com/pkg/errors" -) - -// CreateDataDirectory creates the etcd data directory (commonly /var/lib/etcd) with the right permissions. -func CreateDataDirectory(dir string) error { - if err := os.MkdirAll(dir, 0700); err != nil { - return errors.Wrapf(err, "failed to create the etcd data directory: %q", dir) - } - return nil -} diff --git a/cmd/kubeadm/app/util/image/BUILD b/cmd/kubeadm/app/util/image/BUILD deleted file mode 100644 index f61adb81c4a62..0000000000000 --- a/cmd/kubeadm/app/util/image/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["image.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/image", - visibility = ["//visibility:public"], -) - -go_test( - name = "go_default_test", - srcs = ["image_test.go"], - embed = [":go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/image/image.go b/cmd/kubeadm/app/util/image/image.go deleted file mode 100644 index 0a3c2bb6cb719..0000000000000 --- a/cmd/kubeadm/app/util/image/image.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package image - -import "regexp" - -var ( - // tagMatcher is the regex used to match a tag. - // Basically we presume an image can be made of `[domain][:port][path][:tag][@sha256:digest]` - // We are obvously interested only in the tag, but for the purpose of properly matching it, we also match the digest - // (if present). All the parts before the tag we match in a single match everything (but not greedy) group. - // All matched sub-groups, except the tag one, get thrown away. Hence, in a result of FindStringSubmatch, if a tag - // matches, it's going to be the second returned element (after the full match). - tagMatcher = regexp.MustCompile(`^(?U:.*)(?::([[:word:]][[:word:].-]*))?(?:@sha256:[a-fA-F0-9]{64})?$`) -) - -// TagFromImage extracts a tag from image. An empty string is returned if no tag is discovered. -func TagFromImage(image string) string { - matches := tagMatcher.FindStringSubmatch(image) - if len(matches) >= 2 { - return matches[1] - } - return "" -} diff --git a/cmd/kubeadm/app/util/image/image_test.go b/cmd/kubeadm/app/util/image/image_test.go deleted file mode 100644 index dd9ee2779bd4a..0000000000000 --- a/cmd/kubeadm/app/util/image/image_test.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package image - -import "testing" - -func TestTagFromImage(t *testing.T) { - tests := map[string]string{ - "kindest/node": "", - "kindest/node:latest": "latest", - "kindest/node:v1.17.0": "v1.17.0", - "kindest/node:v1.17.0@sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62": "v1.17.0", - "kindest/node@sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62": "", - - "example.com/kindest/node": "", - "example.com/kindest/node:latest": "latest", - "example.com/kindest/node:v1.17.0": "v1.17.0", - "example.com/kindest/node:v1.17.0@sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62": "v1.17.0", - "example.com/kindest/node@sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62": "", - - "example.com:3000/kindest/node": "", - "example.com:3000/kindest/node:latest": "latest", - "example.com:3000/kindest/node:v1.17.0": "v1.17.0", - "example.com:3000/kindest/node:v1.17.0@sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62": "v1.17.0", - "example.com:3000/kindest/node@sha256:9512edae126da271b66b990b6fff768fbb7cd786c7d39e86bdf55906352fdf62": "", - } - for in, expected := range tests { - out := TagFromImage(in) - if out != expected { - t.Errorf("TagFromImage(%q) = %q, expected %q instead", in, out, expected) - } - } -} diff --git a/cmd/kubeadm/app/util/initsystem/BUILD b/cmd/kubeadm/app/util/initsystem/BUILD deleted file mode 100644 index 8ddbd5a67be40..0000000000000 --- a/cmd/kubeadm/app/util/initsystem/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = [ - "initsystem.go", - "initsystem_unix.go", - "initsystem_windows.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/initsystem", - deps = select({ - "@io_bazel_rules_go//go/platform:windows": [ - "//vendor/golang.org/x/sys/windows/svc:go_default_library", - "//vendor/golang.org/x/sys/windows/svc/mgr:go_default_library", - ], - "//conditions:default": [], - }), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/initsystem/initsystem.go b/cmd/kubeadm/app/util/initsystem/initsystem.go deleted file mode 100644 index 48e152f0a9be0..0000000000000 --- a/cmd/kubeadm/app/util/initsystem/initsystem.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package initsystem - -// InitSystem is the interface that describe behaviors of an init system -type InitSystem interface { - // return a string describing how to enable a service - EnableCommand(service string) string - - // ServiceStart tries to start a specific service - ServiceStart(service string) error - - // ServiceStop tries to stop a specific service - ServiceStop(service string) error - - // ServiceRestart tries to reload the environment and restart the specific service - ServiceRestart(service string) error - - // ServiceExists ensures the service is defined for this init system. - ServiceExists(service string) bool - - // ServiceIsEnabled ensures the service is enabled to start on each boot. - ServiceIsEnabled(service string) bool - - // ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) - ServiceIsActive(service string) bool -} diff --git a/cmd/kubeadm/app/util/initsystem/initsystem_unix.go b/cmd/kubeadm/app/util/initsystem/initsystem_unix.go deleted file mode 100644 index 5cbf9099e75b0..0000000000000 --- a/cmd/kubeadm/app/util/initsystem/initsystem_unix.go +++ /dev/null @@ -1,164 +0,0 @@ -// +build !windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package initsystem - -import ( - "fmt" - "os/exec" - "strings" -) - -// OpenRCInitSystem defines openrc -type OpenRCInitSystem struct{} - -// ServiceStart tries to start a specific service -func (openrc OpenRCInitSystem) ServiceStart(service string) error { - args := []string{service, "start"} - return exec.Command("rc-service", args...).Run() -} - -// ServiceStop tries to stop a specific service -func (openrc OpenRCInitSystem) ServiceStop(service string) error { - args := []string{service, "stop"} - return exec.Command("rc-service", args...).Run() -} - -// ServiceRestart tries to reload the environment and restart the specific service -func (openrc OpenRCInitSystem) ServiceRestart(service string) error { - args := []string{service, "restart"} - return exec.Command("rc-service", args...).Run() -} - -// ServiceExists ensures the service is defined for this init system. -// openrc writes to stderr if a service is not found or not enabled -// this is in contrast to systemd which only writes to stdout. -// Hence, we use the Combinedoutput, and ignore the error. -func (openrc OpenRCInitSystem) ServiceExists(service string) bool { - args := []string{service, "status"} - outBytes, _ := exec.Command("rc-service", args...).CombinedOutput() - return !strings.Contains(string(outBytes), "does not exist") -} - -// ServiceIsEnabled ensures the service is enabled to start on each boot. -func (openrc OpenRCInitSystem) ServiceIsEnabled(service string) bool { - args := []string{"show", "default"} - outBytes, _ := exec.Command("rc-update", args...).Output() - return strings.Contains(string(outBytes), service) -} - -// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) -func (openrc OpenRCInitSystem) ServiceIsActive(service string) bool { - args := []string{service, "status"} - outBytes, _ := exec.Command("rc-service", args...).CombinedOutput() - outStr := string(outBytes) - return !strings.Contains(outStr, "stopped") && !strings.Contains(outStr, "does not exist") -} - -// EnableCommand return a string describing how to enable a service -func (openrc OpenRCInitSystem) EnableCommand(service string) string { - return fmt.Sprintf("rc-update add %s default", service) -} - -// SystemdInitSystem defines systemd -type SystemdInitSystem struct{} - -// EnableCommand return a string describing how to enable a service -func (sysd SystemdInitSystem) EnableCommand(service string) string { - return fmt.Sprintf("systemctl enable %s.service", service) -} - -// reloadSystemd reloads the systemd daemon -func (sysd SystemdInitSystem) reloadSystemd() error { - if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil { - return fmt.Errorf("failed to reload systemd: %v", err) - } - return nil -} - -// ServiceStart tries to start a specific service -func (sysd SystemdInitSystem) ServiceStart(service string) error { - // Before we try to start any service, make sure that systemd is ready - if err := sysd.reloadSystemd(); err != nil { - return err - } - args := []string{"start", service} - return exec.Command("systemctl", args...).Run() -} - -// ServiceRestart tries to reload the environment and restart the specific service -func (sysd SystemdInitSystem) ServiceRestart(service string) error { - // Before we try to restart any service, make sure that systemd is ready - if err := sysd.reloadSystemd(); err != nil { - return err - } - args := []string{"restart", service} - return exec.Command("systemctl", args...).Run() -} - -// ServiceStop tries to stop a specific service -func (sysd SystemdInitSystem) ServiceStop(service string) error { - args := []string{"stop", service} - return exec.Command("systemctl", args...).Run() -} - -// ServiceExists ensures the service is defined for this init system. -func (sysd SystemdInitSystem) ServiceExists(service string) bool { - args := []string{"status", service} - outBytes, _ := exec.Command("systemctl", args...).Output() - output := string(outBytes) - return !strings.Contains(output, "Loaded: not-found") -} - -// ServiceIsEnabled ensures the service is enabled to start on each boot. -func (sysd SystemdInitSystem) ServiceIsEnabled(service string) bool { - args := []string{"is-enabled", service} - err := exec.Command("systemctl", args...).Run() - return err == nil -} - -// ServiceIsActive will check is the service is "active". In the case of -// crash looping services (kubelet in our case) status will return as -// "activating", so we will consider this active as well. -func (sysd SystemdInitSystem) ServiceIsActive(service string) bool { - args := []string{"is-active", service} - // Ignoring error here, command returns non-0 if in "activating" status: - outBytes, _ := exec.Command("systemctl", args...).Output() - output := strings.TrimSpace(string(outBytes)) - if output == "active" || output == "activating" { - return true - } - return false -} - -// GetInitSystem returns an InitSystem for the current system, or nil -// if we cannot detect a supported init system. -// This indicates we will skip init system checks, not an error. -func GetInitSystem() (InitSystem, error) { - // Assume existence of systemctl in path implies this is a systemd system: - _, err := exec.LookPath("systemctl") - if err == nil { - return &SystemdInitSystem{}, nil - } - _, err = exec.LookPath("openrc") - if err == nil { - return &OpenRCInitSystem{}, nil - } - - return nil, fmt.Errorf("no supported init system detected, skipping checking for services") -} diff --git a/cmd/kubeadm/app/util/initsystem/initsystem_windows.go b/cmd/kubeadm/app/util/initsystem/initsystem_windows.go deleted file mode 100644 index 394272ddcb334..0000000000000 --- a/cmd/kubeadm/app/util/initsystem/initsystem_windows.go +++ /dev/null @@ -1,245 +0,0 @@ -// +build windows - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package initsystem - -import ( - "fmt" - "time" - - "golang.org/x/sys/windows/svc" - "golang.org/x/sys/windows/svc/mgr" -) - -// WindowsInitSystem is the windows implementation of InitSystem -type WindowsInitSystem struct{} - -// EnableCommand return a string describing how to enable a service -func (sysd WindowsInitSystem) EnableCommand(service string) string { - return fmt.Sprintf("Set-Service '%s' -StartupType Automatic", service) -} - -// ServiceStart tries to start a specific service -// Following Windows documentation: https://docs.microsoft.com/en-us/windows/desktop/Services/starting-a-service -func (sysd WindowsInitSystem) ServiceStart(service string) error { - m, err := mgr.Connect() - if err != nil { - return err - } - defer m.Disconnect() - - s, err := m.OpenService(service) - if err != nil { - return fmt.Errorf("could not access service %s: %v", service, err) - } - defer s.Close() - - // Check if service is already started - status, err := s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - - if status.State != svc.Stopped && status.State != svc.StopPending { - return nil - } - - timeout := time.Now().Add(10 * time.Second) - for status.State != svc.Stopped { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to stop", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - - // Start the service - err = s.Start("is", "manual-started") - if err != nil { - return fmt.Errorf("could not start service %s: %v", service, err) - } - - // Check that the start was successful - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - timeout = time.Now().Add(10 * time.Second) - for status.State != svc.Running { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to start", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - return nil -} - -// ServiceRestart tries to reload the environment and restart the specific service -func (sysd WindowsInitSystem) ServiceRestart(service string) error { - if err := sysd.ServiceStop(service); err != nil { - return fmt.Errorf("couldn't stop service %s: %v", service, err) - } - if err := sysd.ServiceStart(service); err != nil { - return fmt.Errorf("couldn't start service %s: %v", service, err) - } - - return nil -} - -// ServiceStop tries to stop a specific service -// Following Windows documentation: https://docs.microsoft.com/en-us/windows/desktop/Services/stopping-a-service -func (sysd WindowsInitSystem) ServiceStop(service string) error { - m, err := mgr.Connect() - if err != nil { - return err - } - defer m.Disconnect() - - s, err := m.OpenService(service) - if err != nil { - return fmt.Errorf("could not access service %s: %v", service, err) - } - defer s.Close() - - // Check if service is already stopped - status, err := s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - - if status.State == svc.Stopped { - return nil - } - - // If StopPending, check that service eventually stops - if status.State == svc.StopPending { - timeout := time.Now().Add(10 * time.Second) - for status.State != svc.Stopped { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to stop", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - return nil - } - - // Stop the service - status, err = s.Control(svc.Stop) - if err != nil { - return fmt.Errorf("could not stop service %s: %v", service, err) - } - - // Check that the stop was successful - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not query service %s: %v", service, err) - } - timeout := time.Now().Add(10 * time.Second) - for status.State != svc.Stopped { - if timeout.Before(time.Now()) { - return fmt.Errorf("timeout waiting for %s service to stop", service) - } - time.Sleep(300 * time.Millisecond) - status, err = s.Query() - if err != nil { - return fmt.Errorf("could not retrieve %s service status: %v", service, err) - } - } - return nil -} - -// ServiceExists ensures the service is defined for this init system. -func (sysd WindowsInitSystem) ServiceExists(service string) bool { - m, err := mgr.Connect() - if err != nil { - return false - } - defer m.Disconnect() - s, err := m.OpenService(service) - if err != nil { - return false - } - defer s.Close() - - return true -} - -// ServiceIsEnabled ensures the service is enabled to start on each boot. -func (sysd WindowsInitSystem) ServiceIsEnabled(service string) bool { - m, err := mgr.Connect() - if err != nil { - return false - } - defer m.Disconnect() - - s, err := m.OpenService(service) - if err != nil { - return false - } - defer s.Close() - - c, err := s.Config() - if err != nil { - return false - } - - return c.StartType != mgr.StartDisabled -} - -// ServiceIsActive ensures the service is running, or attempting to run. (crash looping in the case of kubelet) -func (sysd WindowsInitSystem) ServiceIsActive(service string) bool { - m, err := mgr.Connect() - if err != nil { - return false - } - defer m.Disconnect() - s, err := m.OpenService(service) - if err != nil { - return false - } - defer s.Close() - - status, err := s.Query() - if err != nil { - return false - } - return status.State == svc.Running -} - -// GetInitSystem returns an InitSystem for the current system, or nil -// if we cannot detect a supported init system. -// This indicates we will skip init system checks, not an error. -func GetInitSystem() (InitSystem, error) { - m, err := mgr.Connect() - if err != nil { - return nil, fmt.Errorf("no supported init system detected: %v", err) - } - defer m.Disconnect() - return &WindowsInitSystem{}, nil -} diff --git a/cmd/kubeadm/app/util/kubeconfig/BUILD b/cmd/kubeadm/app/util/kubeconfig/BUILD deleted file mode 100644 index ff126b502a353..0000000000000 --- a/cmd/kubeadm/app/util/kubeconfig/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["kubeconfig_test.go"], - embed = [":go_default_library"], - deps = ["//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["kubeconfig.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig", - deps = [ - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go deleted file mode 100644 index cc03c085d57b9..0000000000000 --- a/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeconfig - -import ( - "fmt" - "io/ioutil" - - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - - "github.com/pkg/errors" -) - -// CreateBasic creates a basic, general KubeConfig object that then can be extended -func CreateBasic(serverURL, clusterName, userName string, caCert []byte) *clientcmdapi.Config { - // Use the cluster and the username as the context name - contextName := fmt.Sprintf("%s@%s", userName, clusterName) - - return &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - clusterName: { - Server: serverURL, - CertificateAuthorityData: caCert, - }, - }, - Contexts: map[string]*clientcmdapi.Context{ - contextName: { - Cluster: clusterName, - AuthInfo: userName, - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{}, - CurrentContext: contextName, - } -} - -// CreateWithCerts creates a KubeConfig object with access to the API server with client certificates -func CreateWithCerts(serverURL, clusterName, userName string, caCert []byte, clientKey []byte, clientCert []byte) *clientcmdapi.Config { - config := CreateBasic(serverURL, clusterName, userName, caCert) - config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ - ClientKeyData: clientKey, - ClientCertificateData: clientCert, - } - return config -} - -// CreateWithToken creates a KubeConfig object with access to the API server with a token -func CreateWithToken(serverURL, clusterName, userName string, caCert []byte, token string) *clientcmdapi.Config { - config := CreateBasic(serverURL, clusterName, userName, caCert) - config.AuthInfos[userName] = &clientcmdapi.AuthInfo{ - Token: token, - } - return config -} - -// ClientSetFromFile returns a ready-to-use client from a kubeconfig file -func ClientSetFromFile(path string) (*clientset.Clientset, error) { - config, err := clientcmd.LoadFromFile(path) - if err != nil { - return nil, errors.Wrap(err, "failed to load admin kubeconfig") - } - return ToClientSet(config) -} - -// ToClientSet converts a KubeConfig object to a client -func ToClientSet(config *clientcmdapi.Config) (*clientset.Clientset, error) { - overrides := clientcmd.ConfigOverrides{Timeout: "10s"} - clientConfig, err := clientcmd.NewDefaultClientConfig(*config, &overrides).ClientConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to create API client configuration from kubeconfig") - } - - client, err := clientset.NewForConfig(clientConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to create API client") - } - return client, nil -} - -// WriteToDisk writes a KubeConfig object down to disk with mode 0600 -func WriteToDisk(filename string, kubeconfig *clientcmdapi.Config) error { - err := clientcmd.WriteToFile(*kubeconfig, filename) - if err != nil { - return err - } - - return nil -} - -// GetClusterFromKubeConfig returns the default Cluster of the specified KubeConfig -func GetClusterFromKubeConfig(config *clientcmdapi.Config) *clientcmdapi.Cluster { - // If there is an unnamed cluster object, use it - if config.Clusters[""] != nil { - return config.Clusters[""] - } - if config.Contexts[config.CurrentContext] != nil { - return config.Clusters[config.Contexts[config.CurrentContext].Cluster] - } - return nil -} - -// HasAuthenticationCredentials returns true if the current user has valid authentication credentials for -// token authentication, basic authentication or X509 authentication -func HasAuthenticationCredentials(config *clientcmdapi.Config) bool { - authInfo := getCurrentAuthInfo(config) - if authInfo == nil { - return false - } - - // token authentication - if len(authInfo.Token) != 0 { - return true - } - - // basic authentication - if len(authInfo.Username) != 0 && len(authInfo.Password) != 0 { - return true - } - - // X509 authentication - if (len(authInfo.ClientCertificate) != 0 || len(authInfo.ClientCertificateData) != 0) && - (len(authInfo.ClientKey) != 0 || len(authInfo.ClientKeyData) != 0) { - return true - } - - return false -} - -// EnsureAuthenticationInfoAreEmbedded check if some authentication info are provided as external key/certificate -// files, and eventually embeds such files into the kubeconfig file -func EnsureAuthenticationInfoAreEmbedded(config *clientcmdapi.Config) error { - authInfo := getCurrentAuthInfo(config) - if authInfo == nil { - return errors.New("invalid kubeconfig file. AuthInfo is not defined for the current user") - } - - if len(authInfo.ClientCertificateData) == 0 && len(authInfo.ClientCertificate) != 0 { - clientCert, err := ioutil.ReadFile(authInfo.ClientCertificate) - if err != nil { - return errors.Wrap(err, "error while reading client cert file defined in kubeconfig") - } - authInfo.ClientCertificateData = clientCert - authInfo.ClientCertificate = "" - } - if len(authInfo.ClientKeyData) == 0 && len(authInfo.ClientKey) != 0 { - clientKey, err := ioutil.ReadFile(authInfo.ClientKey) - if err != nil { - return errors.Wrap(err, "error while reading client key file defined in kubeconfig") - } - authInfo.ClientKeyData = clientKey - authInfo.ClientKey = "" - } - - return nil -} - -// EnsureCertificateAuthorityIsEmbedded check if the certificate authority is provided as an external -// file and eventually embeds it into the kubeconfig -func EnsureCertificateAuthorityIsEmbedded(cluster *clientcmdapi.Cluster) error { - if cluster == nil { - return errors.New("received nil value for Cluster") - } - - if len(cluster.CertificateAuthorityData) == 0 && len(cluster.CertificateAuthority) != 0 { - ca, err := ioutil.ReadFile(cluster.CertificateAuthority) - if err != nil { - return errors.Wrap(err, "error while reading certificate authority file defined in kubeconfig") - } - cluster.CertificateAuthorityData = ca - cluster.CertificateAuthority = "" - } - - return nil -} - -// getCurrentAuthInfo returns current authInfo, if defined -func getCurrentAuthInfo(config *clientcmdapi.Config) *clientcmdapi.AuthInfo { - if config == nil || config.CurrentContext == "" || - len(config.Contexts) == 0 || config.Contexts[config.CurrentContext] == nil { - return nil - } - user := config.Contexts[config.CurrentContext].AuthInfo - - if user == "" || len(config.AuthInfos) == 0 || config.AuthInfos[user] == nil { - return nil - } - - return config.AuthInfos[user] -} diff --git a/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go deleted file mode 100644 index 44cbbaa679537..0000000000000 --- a/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go +++ /dev/null @@ -1,330 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeconfig - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "testing" - - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -const ( - configOut1 = `apiVersion: v1 -clusters: -- cluster: - server: "" - name: k8s -contexts: -- context: - cluster: k8s - user: user1 - name: user1@k8s -current-context: user1@k8s -kind: Config -preferences: {} -users: -- name: user1 - user: - token: abc -` - configOut2 = `apiVersion: v1 -clusters: -- cluster: - server: localhost:8080 - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: user2 - name: user2@kubernetes -current-context: user2@kubernetes -kind: Config -preferences: {} -users: -- name: user2 - user: - token: cba -` -) - -type configClient struct { - clusterName string - userName string - serverURL string - caCert []byte -} - -type configClientWithCerts struct { - clientKey []byte - clientCert []byte -} - -type configClientWithToken struct { - token string -} - -func TestCreateWithCerts(t *testing.T) { - var createBasicTest = []struct { - name string - cc configClient - ccWithCerts configClientWithCerts - expected string - }{ - {"empty config", configClient{}, configClientWithCerts{}, ""}, - {"clusterName kubernetes", configClient{clusterName: "kubernetes"}, configClientWithCerts{}, ""}, - } - for _, rt := range createBasicTest { - t.Run(rt.name, func(t *testing.T) { - cwc := CreateWithCerts( - rt.cc.serverURL, - rt.cc.clusterName, - rt.cc.userName, - rt.cc.caCert, - rt.ccWithCerts.clientKey, - rt.ccWithCerts.clientCert, - ) - if cwc.Kind != rt.expected { - t.Errorf( - "failed CreateWithCerts:\n\texpected: %s\n\t actual: %s", - rt.expected, - cwc.Kind, - ) - } - }) - } -} - -func TestCreateWithToken(t *testing.T) { - var createBasicTest = []struct { - name string - cc configClient - ccWithToken configClientWithToken - expected string - }{ - {"empty config", configClient{}, configClientWithToken{}, ""}, - {"clusterName kubernetes", configClient{clusterName: "kubernetes"}, configClientWithToken{}, ""}, - } - for _, rt := range createBasicTest { - t.Run(rt.name, func(t *testing.T) { - cwc := CreateWithToken( - rt.cc.serverURL, - rt.cc.clusterName, - rt.cc.userName, - rt.cc.caCert, - rt.ccWithToken.token, - ) - if cwc.Kind != rt.expected { - t.Errorf( - "failed CreateWithToken:\n\texpected: %s\n\t actual: %s", - rt.expected, - cwc.Kind, - ) - } - }) - } -} - -func TestWriteKubeconfigToDisk(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - var writeConfig = []struct { - name string - cc configClient - ccWithToken configClientWithToken - expected error - file []byte - }{ - {"test1", configClient{clusterName: "k8s", userName: "user1"}, configClientWithToken{token: "abc"}, nil, []byte(configOut1)}, - {"test2", configClient{clusterName: "kubernetes", userName: "user2", serverURL: "localhost:8080"}, configClientWithToken{token: "cba"}, nil, []byte(configOut2)}, - } - for _, rt := range writeConfig { - t.Run(rt.name, func(t *testing.T) { - c := CreateWithToken( - rt.cc.serverURL, - rt.cc.clusterName, - rt.cc.userName, - rt.cc.caCert, - rt.ccWithToken.token, - ) - configPath := fmt.Sprintf("%s/etc/kubernetes/%s.conf", tmpdir, rt.name) - err := WriteToDisk(configPath, c) - if err != rt.expected { - t.Errorf( - "failed WriteToDisk with an error:\n\texpected: %s\n\t actual: %s", - rt.expected, - err, - ) - } - newFile, _ := ioutil.ReadFile(configPath) - if !bytes.Equal(newFile, rt.file) { - t.Errorf( - "failed WriteToDisk config write:\n\texpected: %s\n\t actual: %s", - rt.file, - newFile, - ) - } - }) - } -} - -func TestGetCurrentAuthInfo(t *testing.T) { - var testCases = []struct { - name string - config *clientcmdapi.Config - expected bool - }{ - { - name: "nil context", - config: nil, - expected: false, - }, - { - name: "no CurrentContext value", - config: &clientcmdapi.Config{}, - expected: false, - }, - { - name: "no CurrentContext object", - config: &clientcmdapi.Config{CurrentContext: "kubernetes"}, - expected: false, - }, - { - name: "CurrentContext object with bad contents", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"NOTkubernetes": {}}, - }, - expected: false, - }, - { - name: "no AuthInfo value", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {}}, - }, - expected: false, - }, - { - name: "no AuthInfo object", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - }, - expected: false, - }, - { - name: "AuthInfo object with bad contents", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - AuthInfos: map[string]*clientcmdapi.AuthInfo{"NOTkubernetes": {}}, - }, - expected: false, - }, - { - name: "valid AuthInfo", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - AuthInfos: map[string]*clientcmdapi.AuthInfo{"kubernetes": {}}, - }, - expected: true, - }, - } - for _, rt := range testCases { - t.Run(rt.name, func(t *testing.T) { - r := getCurrentAuthInfo(rt.config) - if rt.expected != (r != nil) { - t.Errorf( - "failed TestHasCredentials:\n\texpected: %v\n\t actual: %v", - rt.expected, - r, - ) - } - }) - } -} - -func TestHasCredentials(t *testing.T) { - var testCases = []struct { - name string - config *clientcmdapi.Config - expected bool - }{ - { - name: "no authInfo", - config: nil, - expected: false, - }, - { - name: "no credentials", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - AuthInfos: map[string]*clientcmdapi.AuthInfo{"kubernetes": {}}, - }, - expected: false, - }, - { - name: "token authentication credentials", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - AuthInfos: map[string]*clientcmdapi.AuthInfo{"kubernetes": {Token: "123"}}, - }, - expected: true, - }, - { - name: "basic authentication credentials", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - AuthInfos: map[string]*clientcmdapi.AuthInfo{"kubernetes": {Username: "A", Password: "B"}}, - }, - expected: true, - }, - { - name: "X509 authentication credentials", - config: &clientcmdapi.Config{ - CurrentContext: "kubernetes", - Contexts: map[string]*clientcmdapi.Context{"kubernetes": {AuthInfo: "kubernetes"}}, - AuthInfos: map[string]*clientcmdapi.AuthInfo{"kubernetes": {ClientKey: "A", ClientCertificate: "B"}}, - }, - expected: true, - }, - } - for _, rt := range testCases { - t.Run(rt.name, func(t *testing.T) { - r := HasAuthenticationCredentials(rt.config) - if rt.expected != r { - t.Errorf( - "failed TestHasCredentials:\n\texpected: %v\n\t actual: %v", - rt.expected, - r, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/marshal.go b/cmd/kubeadm/app/util/marshal.go deleted file mode 100644 index 0eaa82efdc12c..0000000000000 --- a/cmd/kubeadm/app/util/marshal.go +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bufio" - "bytes" - "io" - - "github.com/pkg/errors" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - yamlserializer "k8s.io/apimachinery/pkg/runtime/serializer/yaml" - errorsutil "k8s.io/apimachinery/pkg/util/errors" - utilyaml "k8s.io/apimachinery/pkg/util/yaml" - clientsetscheme "k8s.io/client-go/kubernetes/scheme" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// MarshalToYaml marshals an object into yaml. -func MarshalToYaml(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) { - return MarshalToYamlForCodecs(obj, gv, clientsetscheme.Codecs) -} - -// MarshalToYamlForCodecs marshals an object into yaml using the specified codec -// TODO: Is specifying the gv really needed here? -// TODO: Can we support json out of the box easily here? -func MarshalToYamlForCodecs(obj runtime.Object, gv schema.GroupVersion, codecs serializer.CodecFactory) ([]byte, error) { - const mediaType = runtime.ContentTypeYAML - info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) - if !ok { - return []byte{}, errors.Errorf("unsupported media type %q", mediaType) - } - - encoder := codecs.EncoderForVersion(info.Serializer, gv) - return runtime.Encode(encoder, obj) -} - -// UnmarshalFromYaml unmarshals yaml into an object. -func UnmarshalFromYaml(buffer []byte, gv schema.GroupVersion) (runtime.Object, error) { - return UnmarshalFromYamlForCodecs(buffer, gv, clientsetscheme.Codecs) -} - -// UnmarshalFromYamlForCodecs unmarshals yaml into an object using the specified codec -// TODO: Is specifying the gv really needed here? -// TODO: Can we support json out of the box easily here? -func UnmarshalFromYamlForCodecs(buffer []byte, gv schema.GroupVersion, codecs serializer.CodecFactory) (runtime.Object, error) { - const mediaType = runtime.ContentTypeYAML - info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) - if !ok { - return nil, errors.Errorf("unsupported media type %q", mediaType) - } - - decoder := codecs.DecoderToVersion(info.Serializer, gv) - return runtime.Decode(decoder, buffer) -} - -// SplitYAMLDocuments reads the YAML bytes per-document, unmarshals the TypeMeta information from each document -// and returns a map between the GroupVersionKind of the document and the document bytes -func SplitYAMLDocuments(yamlBytes []byte) (kubeadmapi.DocumentMap, error) { - gvkmap := kubeadmapi.DocumentMap{} - knownKinds := map[string]bool{} - errs := []error{} - buf := bytes.NewBuffer(yamlBytes) - reader := utilyaml.NewYAMLReader(bufio.NewReader(buf)) - for { - // Read one YAML document at a time, until io.EOF is returned - b, err := reader.Read() - if err == io.EOF { - break - } else if err != nil { - return nil, err - } - if len(b) == 0 { - break - } - // Deserialize the TypeMeta information of this byte slice - gvk, err := yamlserializer.DefaultMetaFactory.Interpret(b) - if err != nil { - return nil, err - } - if len(gvk.Group) == 0 || len(gvk.Version) == 0 || len(gvk.Kind) == 0 { - return nil, errors.Errorf("invalid configuration for GroupVersionKind %+v: kind and apiVersion is mandatory information that must be specified", gvk) - } - - // Check whether the kind has been registered before. If it has, throw an error - if known := knownKinds[gvk.Kind]; known { - errs = append(errs, errors.Errorf("invalid configuration: kind %q is specified twice in YAML file", gvk.Kind)) - continue - } - knownKinds[gvk.Kind] = true - - // Save the mapping between the gvk and the bytes that object consists of - gvkmap[*gvk] = b - } - if err := errorsutil.NewAggregate(errs); err != nil { - return nil, err - } - return gvkmap, nil -} - -// GroupVersionKindsFromBytes parses the bytes and returns a gvk slice -func GroupVersionKindsFromBytes(b []byte) ([]schema.GroupVersionKind, error) { - gvkmap, err := SplitYAMLDocuments(b) - if err != nil { - return nil, err - } - gvks := []schema.GroupVersionKind{} - for gvk := range gvkmap { - gvks = append(gvks, gvk) - } - return gvks, nil -} - -// GroupVersionKindsHasKind returns whether the following gvk slice contains the kind given as a parameter -func GroupVersionKindsHasKind(gvks []schema.GroupVersionKind, kind string) bool { - for _, gvk := range gvks { - if gvk.Kind == kind { - return true - } - } - return false -} - -// GroupVersionKindsHasClusterConfiguration returns whether the following gvk slice contains a ClusterConfiguration object -func GroupVersionKindsHasClusterConfiguration(gvks ...schema.GroupVersionKind) bool { - return GroupVersionKindsHasKind(gvks, constants.ClusterConfigurationKind) -} - -// GroupVersionKindsHasInitConfiguration returns whether the following gvk slice contains a InitConfiguration object -func GroupVersionKindsHasInitConfiguration(gvks ...schema.GroupVersionKind) bool { - return GroupVersionKindsHasKind(gvks, constants.InitConfigurationKind) -} - -// GroupVersionKindsHasJoinConfiguration returns whether the following gvk slice contains a JoinConfiguration object -func GroupVersionKindsHasJoinConfiguration(gvks ...schema.GroupVersionKind) bool { - return GroupVersionKindsHasKind(gvks, constants.JoinConfigurationKind) -} diff --git a/cmd/kubeadm/app/util/marshal_test.go b/cmd/kubeadm/app/util/marshal_test.go deleted file mode 100644 index 7b3f58c272816..0000000000000 --- a/cmd/kubeadm/app/util/marshal_test.go +++ /dev/null @@ -1,402 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "reflect" - "sort" - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -var files = map[string][]byte{ - "foo": []byte(` -kind: Foo -apiVersion: foo.k8s.io/v1 -fooField: foo -`), - "bar": []byte(` -apiVersion: bar.k8s.io/v2 -barField: bar -kind: Bar -`), - "baz": []byte(` -apiVersion: baz.k8s.io/v1 -kind: Baz -baz: - foo: bar -`), - "nokind": []byte(` -apiVersion: baz.k8s.io/v1 -foo: foo -bar: bar -`), - "noapiversion": []byte(` -kind: Bar -foo: foo -bar: bar -`), -} - -func TestMarshalUnmarshalYaml(t *testing.T) { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "someName", - Namespace: "testNamespace", - Labels: map[string]string{ - "test": "yes", - }, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyAlways, - }, - } - - bytes, err := MarshalToYaml(pod, corev1.SchemeGroupVersion) - if err != nil { - t.Fatalf("unexpected error marshalling: %v", err) - } - - t.Logf("\n%s", bytes) - - obj2, err := UnmarshalFromYaml(bytes, corev1.SchemeGroupVersion) - if err != nil { - t.Fatalf("unexpected error marshalling: %v", err) - } - - pod2, ok := obj2.(*corev1.Pod) - if !ok { - t.Fatal("did not get a Pod") - } - - if pod2.Name != pod.Name { - t.Errorf("expected %q, got %q", pod.Name, pod2.Name) - } - - if pod2.Namespace != pod.Namespace { - t.Errorf("expected %q, got %q", pod.Namespace, pod2.Namespace) - } - - if !reflect.DeepEqual(pod2.Labels, pod.Labels) { - t.Errorf("expected %v, got %v", pod.Labels, pod2.Labels) - } - - if pod2.Spec.RestartPolicy != pod.Spec.RestartPolicy { - t.Errorf("expected %q, got %q", pod.Spec.RestartPolicy, pod2.Spec.RestartPolicy) - } -} - -func TestMarshalUnmarshalToYamlForCodecs(t *testing.T) { - cfg := &kubeadmapiv1beta2.InitConfiguration{ - TypeMeta: metav1.TypeMeta{ - Kind: constants.InitConfigurationKind, - APIVersion: kubeadmapiv1beta2.SchemeGroupVersion.String(), - }, - NodeRegistration: kubeadmapiv1beta2.NodeRegistrationOptions{ - Name: "testNode", - CRISocket: "/var/run/cri.sock", - }, - BootstrapTokens: []kubeadmapiv1beta2.BootstrapToken{ - { - Token: &kubeadmapiv1beta2.BootstrapTokenString{ID: "abcdef", Secret: "abcdef0123456789"}, - }, - }, - // NOTE: Using MarshalToYamlForCodecs and UnmarshalFromYamlForCodecs for ClusterConfiguration fields here won't work - // by design. This is because we have a `json:"-"` annotation in order to avoid struct duplication. See the comment - // at the kubeadmapiv1beta2.InitConfiguration definition. - } - - kubeadmapiv1beta2.SetDefaults_InitConfiguration(cfg) - scheme := runtime.NewScheme() - if err := kubeadmapiv1beta2.AddToScheme(scheme); err != nil { - t.Fatal(err) - } - codecs := serializer.NewCodecFactory(scheme) - - bytes, err := MarshalToYamlForCodecs(cfg, kubeadmapiv1beta2.SchemeGroupVersion, codecs) - if err != nil { - t.Fatalf("unexpected error marshalling InitConfiguration: %v", err) - } - t.Logf("\n%s", bytes) - - obj, err := UnmarshalFromYamlForCodecs(bytes, kubeadmapiv1beta2.SchemeGroupVersion, codecs) - if err != nil { - t.Fatalf("unexpected error unmarshalling InitConfiguration: %v", err) - } - - cfg2, ok := obj.(*kubeadmapiv1beta2.InitConfiguration) - if !ok || cfg2 == nil { - t.Fatal("did not get InitConfiguration back") - } - if !reflect.DeepEqual(*cfg, *cfg2) { - t.Errorf("expected %v, got %v", *cfg, *cfg2) - } -} - -func TestSplitYAMLDocuments(t *testing.T) { - var tests = []struct { - name string - fileContents []byte - gvkmap kubeadmapi.DocumentMap - expectedErr bool - }{ - { - name: "FooOnly", - fileContents: files["foo"], - gvkmap: kubeadmapi.DocumentMap{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}: files["foo"], - }, - }, - { - name: "FooBar", - fileContents: bytes.Join([][]byte{files["foo"], files["bar"]}, []byte(constants.YAMLDocumentSeparator)), - gvkmap: kubeadmapi.DocumentMap{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}: files["foo"], - {Group: "bar.k8s.io", Version: "v2", Kind: "Bar"}: files["bar"], - }, - }, - { - name: "FooTwiceInvalid", - fileContents: bytes.Join([][]byte{files["foo"], files["bar"], files["foo"]}, []byte(constants.YAMLDocumentSeparator)), - expectedErr: true, - }, - { - name: "InvalidBaz", - fileContents: bytes.Join([][]byte{files["foo"], files["baz"]}, []byte(constants.YAMLDocumentSeparator)), - expectedErr: true, - }, - { - name: "InvalidNoKind", - fileContents: files["nokind"], - expectedErr: true, - }, - { - name: "InvalidNoAPIVersion", - fileContents: files["noapiversion"], - expectedErr: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - - gvkmap, err := SplitYAMLDocuments(rt.fileContents) - if (err != nil) != rt.expectedErr { - t2.Errorf("expected error: %t, actual: %t", rt.expectedErr, err != nil) - } - - if !reflect.DeepEqual(gvkmap, rt.gvkmap) { - t2.Errorf("expected gvkmap: %s\n\tactual: %s\n", rt.gvkmap, gvkmap) - } - }) - } -} - -func TestGroupVersionKindsFromBytes(t *testing.T) { - var tests = []struct { - name string - fileContents []byte - gvks []string - expectedErr bool - }{ - { - name: "FooOnly", - fileContents: files["foo"], - gvks: []string{ - "foo.k8s.io/v1, Kind=Foo", - }, - }, - { - name: "FooBar", - fileContents: bytes.Join([][]byte{files["foo"], files["bar"]}, []byte(constants.YAMLDocumentSeparator)), - gvks: []string{ - "foo.k8s.io/v1, Kind=Foo", - "bar.k8s.io/v2, Kind=Bar", - }, - }, - { - name: "FooTwiceInvalid", - fileContents: bytes.Join([][]byte{files["foo"], files["bar"], files["foo"]}, []byte(constants.YAMLDocumentSeparator)), - gvks: []string{}, - expectedErr: true, - }, - { - name: "InvalidBaz", - fileContents: bytes.Join([][]byte{files["foo"], files["baz"]}, []byte(constants.YAMLDocumentSeparator)), - gvks: []string{}, - expectedErr: true, - }, - { - name: "InvalidNoKind", - fileContents: files["nokind"], - gvks: []string{}, - expectedErr: true, - }, - { - name: "InvalidNoAPIVersion", - fileContents: files["noapiversion"], - gvks: []string{}, - expectedErr: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - - gvks, err := GroupVersionKindsFromBytes(rt.fileContents) - if (err != nil) != rt.expectedErr { - t2.Errorf("expected error: %t, actual: %t", rt.expectedErr, err != nil) - } - - strgvks := []string{} - for _, gvk := range gvks { - strgvks = append(strgvks, gvk.String()) - } - sort.Strings(strgvks) - sort.Strings(rt.gvks) - - if !reflect.DeepEqual(strgvks, rt.gvks) { - t2.Errorf("expected gvks: %s\n\tactual: %s\n", rt.gvks, strgvks) - } - }) - } -} - -func TestGroupVersionKindsHasKind(t *testing.T) { - var tests = []struct { - name string - gvks []schema.GroupVersionKind - kind string - expected bool - }{ - { - name: "FooOnly", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - }, - kind: "Foo", - expected: true, - }, - { - name: "FooBar", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - {Group: "bar.k8s.io", Version: "v2", Kind: "Bar"}, - }, - kind: "Bar", - expected: true, - }, - { - name: "FooBazNoBaz", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - {Group: "bar.k8s.io", Version: "v2", Kind: "Bar"}, - }, - kind: "Baz", - expected: false, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - - actual := GroupVersionKindsHasKind(rt.gvks, rt.kind) - if rt.expected != actual { - t2.Errorf("expected gvks has kind: %t\n\tactual: %t\n", rt.expected, actual) - } - }) - } -} - -func TestGroupVersionKindsHasInitConfiguration(t *testing.T) { - var tests = []struct { - name string - gvks []schema.GroupVersionKind - kind string - expected bool - }{ - { - name: "NoInitConfiguration", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - }, - expected: false, - }, - { - name: "InitConfigurationFound", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - {Group: "bar.k8s.io", Version: "v2", Kind: "InitConfiguration"}, - }, - expected: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - - actual := GroupVersionKindsHasInitConfiguration(rt.gvks...) - if rt.expected != actual { - t2.Errorf("expected gvks has InitConfiguration: %t\n\tactual: %t\n", rt.expected, actual) - } - }) - } -} - -func TestGroupVersionKindsHasJoinConfiguration(t *testing.T) { - var tests = []struct { - name string - gvks []schema.GroupVersionKind - kind string - expected bool - }{ - { - name: "NoJoinConfiguration", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - }, - expected: false, - }, - { - name: "JoinConfigurationFound", - gvks: []schema.GroupVersionKind{ - {Group: "foo.k8s.io", Version: "v1", Kind: "Foo"}, - {Group: "bar.k8s.io", Version: "v2", Kind: "JoinConfiguration"}, - }, - expected: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t2 *testing.T) { - - actual := GroupVersionKindsHasJoinConfiguration(rt.gvks...) - if rt.expected != actual { - t2.Errorf("expected gvks has JoinConfiguration: %t\n\tactual: %t\n", rt.expected, actual) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/net.go b/cmd/kubeadm/app/util/net.go deleted file mode 100644 index 54ebcb690b9b3..0000000000000 --- a/cmd/kubeadm/app/util/net.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "os" - "strings" - - "github.com/pkg/errors" -) - -// GetHostname returns OS's hostname if 'hostnameOverride' is empty; otherwise, return 'hostnameOverride' -// NOTE: This function copied from pkg/util/node package to avoid external kubeadm dependency -func GetHostname(hostnameOverride string) (string, error) { - hostName := hostnameOverride - if len(hostName) == 0 { - nodeName, err := os.Hostname() - if err != nil { - return "", errors.Wrap(err, "couldn't determine hostname") - } - hostName = nodeName - } - - // Trim whitespaces first to avoid getting an empty hostname - // For linux, the hostname is read from file /proc/sys/kernel/hostname directly - hostName = strings.TrimSpace(hostName) - if len(hostName) == 0 { - return "", errors.New("empty hostname is invalid") - } - - return strings.ToLower(hostName), nil -} diff --git a/cmd/kubeadm/app/util/net_test.go b/cmd/kubeadm/app/util/net_test.go deleted file mode 100644 index 8d5888349a20e..0000000000000 --- a/cmd/kubeadm/app/util/net_test.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "errors" - "os" - "strings" - "testing" -) - -func TestGetHostname(t *testing.T) { - hostname, err := os.Hostname() - - testCases := []struct { - desc string - hostname string - result string - expectedErr error - }{ - { - desc: "overridden hostname", - hostname: "overridden", - result: "overridden", - expectedErr: nil, - }, - { - desc: "overridden hostname uppercase", - hostname: "OVERRIDDEN", - result: "overridden", - expectedErr: nil, - }, - { - desc: "hostname contains only spaces", - hostname: " ", - result: "", - expectedErr: errors.New("empty hostname is invalid"), - }, - { - desc: "empty parameter", - hostname: "", - result: strings.ToLower(hostname), - expectedErr: err, - }, - } - - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - result, err := GetHostname(tc.hostname) - - if err != nil && tc.expectedErr == nil { - t.Errorf("unexpected error: %v", err) - } - - if err == nil && tc.expectedErr != nil { - t.Errorf("expected error %v, got nil", tc.expectedErr) - } - - if tc.result != result { - t.Errorf("unexpected result: %s, expected: %s", result, tc.result) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/output/BUILD b/cmd/kubeadm/app/util/output/BUILD deleted file mode 100644 index 6a3c10e758ac9..0000000000000 --- a/cmd/kubeadm/app/util/output/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["output.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/output", - visibility = ["//visibility:public"], - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/printers:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/output/output.go b/cmd/kubeadm/app/util/output/output.go deleted file mode 100644 index fdaf8afc22d10..0000000000000 --- a/cmd/kubeadm/app/util/output/output.go +++ /dev/null @@ -1,188 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package output - -import ( - "fmt" - "io" - "strings" - - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/printers" -) - -// TextOutput describes the plain text output -const TextOutput = "text" - -// TextPrintFlags is an interface to handle custom text output -type TextPrintFlags interface { - ToPrinter(outputFormat string) (Printer, error) -} - -// PrintFlags composes common printer flag structs -// used across kubeadm commands, and provides a method -// of retrieving a known printer based on flag values provided. -type PrintFlags struct { - // JSONYamlPrintFlags provides default flags necessary for json/yaml printing. - JSONYamlPrintFlags *genericclioptions.JSONYamlPrintFlags - // KubeTemplatePrintFlags composes print flags that provide both a JSONPath and a go-template printer. - KubeTemplatePrintFlags *genericclioptions.KubeTemplatePrintFlags - // JSONYamlPrintFlags provides default flags necessary for kubeadm text printing. - TextPrintFlags TextPrintFlags - // TypeSetterPrinter is an implementation of ResourcePrinter that wraps another printer with types set on the objects - TypeSetterPrinter *printers.TypeSetterPrinter - // OutputFormat contains currently set output format - OutputFormat *string -} - -// AllowedFormats returns list of allowed output formats -func (pf *PrintFlags) AllowedFormats() []string { - ret := []string{TextOutput} - ret = append(ret, pf.JSONYamlPrintFlags.AllowedFormats()...) - ret = append(ret, pf.KubeTemplatePrintFlags.AllowedFormats()...) - - return ret -} - -// ToPrinter receives an outputFormat and returns a printer capable of -// handling format printing. -// Returns error if the specified outputFormat does not match supported formats. -func (pf *PrintFlags) ToPrinter() (Printer, error) { - outputFormat := "" - if pf.OutputFormat != nil { - outputFormat = *pf.OutputFormat - } - - if pf.TextPrintFlags != nil { - if p, err := pf.TextPrintFlags.ToPrinter(outputFormat); !genericclioptions.IsNoCompatiblePrinterError(err) { - return p, err - } - } - - if pf.JSONYamlPrintFlags != nil { - if p, err := pf.JSONYamlPrintFlags.ToPrinter(outputFormat); !genericclioptions.IsNoCompatiblePrinterError(err) { - return NewResourcePrinterWrapper(pf.TypeSetterPrinter.WrapToPrinter(p, err)) - } - } - - if pf.KubeTemplatePrintFlags != nil { - if p, err := pf.KubeTemplatePrintFlags.ToPrinter(outputFormat); !genericclioptions.IsNoCompatiblePrinterError(err) { - return NewResourcePrinterWrapper(pf.TypeSetterPrinter.WrapToPrinter(p, err)) - } - } - - return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: pf.OutputFormat, AllowedFormats: pf.AllowedFormats()} -} - -// AddFlags receives a *cobra.Command reference and binds -// flags related to Kubeadm printing to it -func (pf *PrintFlags) AddFlags(cmd *cobra.Command) { - pf.JSONYamlPrintFlags.AddFlags(cmd) - pf.KubeTemplatePrintFlags.AddFlags(cmd) - cmd.Flags().StringVarP(pf.OutputFormat, "experimental-output", "o", *pf.OutputFormat, fmt.Sprintf("Output format. One of: %s.", strings.Join(pf.AllowedFormats(), "|"))) -} - -// WithDefaultOutput sets a default output format if one is not provided through a flag value -func (pf *PrintFlags) WithDefaultOutput(outputFormat string) *PrintFlags { - pf.OutputFormat = &outputFormat - return pf -} - -// WithTypeSetter sets a wrapper than will surround the returned printer with a printer to type resources -func (pf *PrintFlags) WithTypeSetter(scheme *runtime.Scheme) *PrintFlags { - pf.TypeSetterPrinter = printers.NewTypeSetter(scheme) - return pf -} - -// NewOutputFlags creates new KubeadmOutputFlags -func NewOutputFlags(textPrintFlags TextPrintFlags) *PrintFlags { - outputFormat := "" - - pf := &PrintFlags{ - OutputFormat: &outputFormat, - - JSONYamlPrintFlags: genericclioptions.NewJSONYamlPrintFlags(), - KubeTemplatePrintFlags: genericclioptions.NewKubeTemplatePrintFlags(), - TextPrintFlags: textPrintFlags, - } - - // disable deprecated --template option - pf.KubeTemplatePrintFlags.TemplateArgument = nil - - return pf -} - -// Printer is a common printing interface in Kubeadm -type Printer interface { - PrintObj(obj runtime.Object, writer io.Writer) error - Fprintf(writer io.Writer, format string, args ...interface{}) (n int, err error) - Printf(format string, args ...interface{}) (n int, err error) -} - -// TextPrinter implements Printer interface for generic text output -type TextPrinter struct { -} - -// PrintObj is an implementation of ResourcePrinter.PrintObj that prints object -func (tp *TextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { - _, err := fmt.Fprintf(writer, "%+v\n", obj) - return err -} - -// Fprintf is a wrapper around fmt.Fprintf -func (tp *TextPrinter) Fprintf(writer io.Writer, format string, args ...interface{}) (n int, err error) { - return fmt.Fprintf(writer, format, args...) -} - -// Printf is a wrapper around fmt.Printf -func (tp *TextPrinter) Printf(format string, args ...interface{}) (n int, err error) { - return fmt.Printf(format, args...) -} - -// ResourcePrinterWrapper wraps ResourcePrinter and implements Printer interface -type ResourcePrinterWrapper struct { - Printer printers.ResourcePrinter -} - -// NewResourcePrinterWrapper creates new ResourcePrinter object -func NewResourcePrinterWrapper(resourcePrinter printers.ResourcePrinter, err error) (Printer, error) { - if err != nil { - return nil, err - } - return &ResourcePrinterWrapper{Printer: resourcePrinter}, nil -} - -// PrintObj is an implementation of ResourcePrinter.PrintObj that calls underlying printer API -func (rpw *ResourcePrinterWrapper) PrintObj(obj runtime.Object, writer io.Writer) error { - return rpw.Printer.PrintObj(obj, writer) -} - -// Fprintf is an empty method to satisfy Printer interface -// and silent info printing for structured output -// This method is usually redefined for the text output -func (rpw *ResourcePrinterWrapper) Fprintf(writer io.Writer, format string, args ...interface{}) (n int, err error) { - return 0, nil -} - -// Printf is an empty method to satisfy Printer interface -// and silent info printing for structured output -// This method is usually redefined for the text output -func (rpw *ResourcePrinterWrapper) Printf(format string, args ...interface{}) (n int, err error) { - return 0, nil -} diff --git a/cmd/kubeadm/app/util/patches/BUILD b/cmd/kubeadm/app/util/patches/BUILD deleted file mode 100644 index 298bc3aa12155..0000000000000 --- a/cmd/kubeadm/app/util/patches/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = ["patches.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/patches", - visibility = ["//visibility:public"], - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library", - "//vendor/github.com/evanphx/json-patch:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = ["patches_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/patches/patches.go b/cmd/kubeadm/app/util/patches/patches.go deleted file mode 100644 index 3b1ad23d0e3e0..0000000000000 --- a/cmd/kubeadm/app/util/patches/patches.go +++ /dev/null @@ -1,342 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package patches - -import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - "sync" - - jsonpatch "github.com/evanphx/json-patch" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - utilyaml "k8s.io/apimachinery/pkg/util/yaml" - "sigs.k8s.io/yaml" -) - -// PatchTarget defines a target to be patched, such as a control-plane static Pod. -type PatchTarget struct { - // Name must be the name of a known target. In the case of Kubernetes objects - // this is likely to match the ObjectMeta.Name of a target. - Name string - - // StrategicMergePatchObject is only used for strategic merge patches. - // It represents the underlying object type that is patched - e.g. "v1.Pod" - StrategicMergePatchObject interface{} - - // Data must contain the bytes that will be patched. - Data []byte -} - -// PatchManager defines an object that can apply patches. -type PatchManager struct { - patchSets []*patchSet - knownTargets []string - output io.Writer -} - -// patchSet defines a set of patches of a certain type that can patch a PatchTarget. -type patchSet struct { - targetName string - patchType types.PatchType - patches []string -} - -// String() is used for unit-testing. -func (ps *patchSet) String() string { - return fmt.Sprintf( - "{%q, %q, %#v}", - ps.targetName, - ps.patchType, - ps.patches, - ) -} - -var ( - pathLock = &sync.RWMutex{} - pathCache = map[string]*PatchManager{} - - patchTypes = map[string]types.PatchType{ - "json": types.JSONPatchType, - "merge": types.MergePatchType, - "strategic": types.StrategicMergePatchType, - "": types.StrategicMergePatchType, // Default - } - patchTypeList = []string{"json", "merge", "strategic"} - patchTypesJoined = strings.Join(patchTypeList, "|") - knownExtensions = []string{"json", "yaml"} - - regExtension = regexp.MustCompile(`.+\.(` + strings.Join(knownExtensions, "|") + `)$`) -) - -// GetPatchManagerForPath creates a patch manager that can be used to apply patches to "knownTargets". -// "path" should contain patches that can be used to patch the "knownTargets". -// If "output" is non-nil, messages about actions performed by the manager would go on this io.Writer. -func GetPatchManagerForPath(path string, knownTargets []string, output io.Writer) (*PatchManager, error) { - pathLock.RLock() - if pm, known := pathCache[path]; known { - pathLock.RUnlock() - return pm, nil - } - pathLock.RUnlock() - - if output == nil { - output = ioutil.Discard - } - - fmt.Fprintf(output, "[patches] Reading patches from path %q\n", path) - - // Get the files in the path. - patchSets, patchFiles, ignoredFiles, err := getPatchSetsFromPath(path, knownTargets, output) - if err != nil { - return nil, err - } - - if len(patchFiles) > 0 { - fmt.Fprintf(output, "[patches] Found the following patch files: %v\n", patchFiles) - } - if len(ignoredFiles) > 0 { - fmt.Fprintf(output, "[patches] Ignored the following files: %v\n", ignoredFiles) - } - - pm := &PatchManager{ - patchSets: patchSets, - knownTargets: knownTargets, - output: output, - } - pathLock.Lock() - pathCache[path] = pm - pathLock.Unlock() - - return pm, nil -} - -// ApplyPatchesToTarget takes a patch target and patches its "Data" using the patches -// stored in the patch manager. The resulted "Data" is always converted to JSON. -func (pm *PatchManager) ApplyPatchesToTarget(patchTarget *PatchTarget) error { - var err error - var patchedData []byte - - var found bool - for _, pt := range pm.knownTargets { - if pt == patchTarget.Name { - found = true - break - } - } - if !found { - return errors.Errorf("unknown patch target name %q, must be one of %v", patchTarget.Name, pm.knownTargets) - } - - // Always convert the target data to JSON. - patchedData, err = yaml.YAMLToJSON(patchTarget.Data) - if err != nil { - return err - } - - // Iterate over the patchSets. - for _, patchSet := range pm.patchSets { - if patchSet.targetName != patchTarget.Name { - continue - } - - // Iterate over the patches in the patchSets. - for _, patch := range patchSet.patches { - patchBytes := []byte(patch) - - // Patch based on the patch type. - switch patchSet.patchType { - - // JSON patch. - case types.JSONPatchType: - var patchObj jsonpatch.Patch - patchObj, err = jsonpatch.DecodePatch(patchBytes) - if err == nil { - patchedData, err = patchObj.Apply(patchedData) - } - - // Merge patch. - case types.MergePatchType: - patchedData, err = jsonpatch.MergePatch(patchedData, patchBytes) - - // Strategic merge patch. - case types.StrategicMergePatchType: - patchedData, err = strategicpatch.StrategicMergePatch( - patchedData, - patchBytes, - patchTarget.StrategicMergePatchObject, - ) - } - - if err != nil { - return errors.Wrapf(err, "could not apply the following patch of type %q to target %q:\n%s\n", - patchSet.patchType, - patchTarget.Name, - patch) - } - fmt.Fprintf(pm.output, "[patches] Applied patch of type %q to target %q\n", patchSet.patchType, patchTarget.Name) - } - - // Update the data for this patch target. - patchTarget.Data = patchedData - } - - return nil -} - -// parseFilename validates a file name and retrieves the encoded target name and patch type. -// - On unknown extension or target name it returns a warning -// - On unknown patch type it returns an error -// - On success it returns a target name and patch type -func parseFilename(fileName string, knownTargets []string) (string, types.PatchType, error, error) { - // Return a warning if the extension cannot be matched. - if !regExtension.MatchString(fileName) { - return "", "", errors.Errorf("the file extension must be one of %v", knownExtensions), nil - } - - regFileNameSplit := regexp.MustCompile( - fmt.Sprintf(`^(%s)([^.+\n]*)?(\+)?(%s)?`, strings.Join(knownTargets, "|"), patchTypesJoined), - ) - // Extract the target name and patch type. The resulting sub-string slice would look like this: - // [full-match, targetName, suffix, +, patchType] - sub := regFileNameSplit.FindStringSubmatch(fileName) - if sub == nil { - return "", "", errors.Errorf("unknown target, must be one of %v", knownTargets), nil - } - targetName := sub[1] - - if len(sub[3]) > 0 && len(sub[4]) == 0 { - return "", "", nil, errors.Errorf("unknown or missing patch type after '+', must be one of %v", patchTypeList) - } - patchType := patchTypes[sub[4]] - - return targetName, patchType, nil, nil -} - -// createPatchSet creates a patchSet object, by splitting the given "data" by "\n---". -func createPatchSet(targetName string, patchType types.PatchType, data string) (*patchSet, error) { - var patches []string - - // Split the patches and convert them to JSON. - // Data that is already JSON will not cause an error. - buf := bytes.NewBuffer([]byte(data)) - reader := utilyaml.NewYAMLReader(bufio.NewReader(buf)) - for { - patch, err := reader.Read() - if err == io.EOF { - break - } else if err != nil { - return nil, errors.Wrapf(err, "could not split patches for data:\n%s\n", data) - } - - patch = bytes.TrimSpace(patch) - if len(patch) == 0 { - continue - } - - patchJSON, err := yaml.YAMLToJSON(patch) - if err != nil { - return nil, errors.Wrapf(err, "could not convert patch to JSON:\n%s\n", patch) - } - patches = append(patches, string(patchJSON)) - } - - return &patchSet{ - targetName: targetName, - patchType: patchType, - patches: patches, - }, nil -} - -// getPatchSetsFromPath walks a path, ignores sub-directories and non-patch files, and -// returns a list of patchFile objects. -func getPatchSetsFromPath(targetPath string, knownTargets []string, output io.Writer) ([]*patchSet, []string, []string, error) { - patchFiles := []string{} - ignoredFiles := []string{} - patchSets := []*patchSet{} - - // Check if targetPath is a directory. - info, err := os.Lstat(targetPath) - if err != nil { - goto return_path_error - } - if !info.IsDir() { - err = errors.New("not a directory") - goto return_path_error - } - - err = filepath.Walk(targetPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Sub-directories and "." are ignored. - if info.IsDir() { - return nil - } - - baseName := info.Name() - - // Parse the filename and retrieve the target and patch type - targetName, patchType, warn, err := parseFilename(baseName, knownTargets) - if err != nil { - return err - } - if warn != nil { - fmt.Fprintf(output, "[patches] Ignoring file %q: %v\n", baseName, warn) - ignoredFiles = append(ignoredFiles, baseName) - return nil - } - - // Read the patch file. - data, err := ioutil.ReadFile(path) - if err != nil { - return errors.Wrapf(err, "could not read the file %q", path) - } - - if len(data) == 0 { - fmt.Fprintf(output, "[patches] Ignoring empty file: %q\n", baseName) - ignoredFiles = append(ignoredFiles, baseName) - return nil - } - - // Create a patchSet object. - patchSet, err := createPatchSet(targetName, patchType, string(data)) - if err != nil { - return err - } - - patchFiles = append(patchFiles, baseName) - patchSets = append(patchSets, patchSet) - return nil - }) - -return_path_error: - if err != nil { - return nil, nil, nil, errors.Wrapf(err, "could not list patch files for path %q", targetPath) - } - - return patchSets, patchFiles, ignoredFiles, nil -} diff --git a/cmd/kubeadm/app/util/patches/patches_test.go b/cmd/kubeadm/app/util/patches/patches_test.go deleted file mode 100644 index 7d2e8f18a672b..0000000000000 --- a/cmd/kubeadm/app/util/patches/patches_test.go +++ /dev/null @@ -1,413 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package patches - -import ( - "bytes" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "testing" - - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" -) - -var testKnownTargets = []string{ - "etcd", - "kube-apiserver", - "kube-controller-manager", - "kube-scheduler", -} - -const testDirPattern = "patch-files" - -func TestParseFilename(t *testing.T) { - tests := []struct { - name string - fileName string - expectedTargetName string - expectedPatchType types.PatchType - expectedWarning bool - expectedError bool - }{ - { - name: "valid: known target and patch type", - fileName: "etcd+merge.json", - expectedTargetName: "etcd", - expectedPatchType: types.MergePatchType, - }, - { - name: "valid: known target and default patch type", - fileName: "etcd0.yaml", - expectedTargetName: "etcd", - expectedPatchType: types.StrategicMergePatchType, - }, - { - name: "valid: known target and custom patch type", - fileName: "etcd0+merge.yaml", - expectedTargetName: "etcd", - expectedPatchType: types.MergePatchType, - }, - { - name: "invalid: unknown target", - fileName: "foo.yaml", - expectedWarning: true, - }, - { - name: "invalid: unknown extension", - fileName: "etcd.foo", - expectedWarning: true, - }, - { - name: "invalid: missing extension", - fileName: "etcd", - expectedWarning: true, - }, - { - name: "invalid: unknown patch type", - fileName: "etcd+foo.json", - expectedError: true, - }, - { - name: "invalid: missing patch type", - fileName: "etcd+.json", - expectedError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - targetName, patchType, warn, err := parseFilename(tc.fileName, testKnownTargets) - if (err != nil) != tc.expectedError { - t.Errorf("expected error: %v, got: %v, error: %v", tc.expectedError, err != nil, err) - } - if (warn != nil) != tc.expectedWarning { - t.Errorf("expected warning: %v, got: %v, warning: %v", tc.expectedWarning, warn != nil, warn) - } - if targetName != tc.expectedTargetName { - t.Errorf("expected target name: %v, got: %v", tc.expectedTargetName, targetName) - } - if patchType != tc.expectedPatchType { - t.Errorf("expected patch type: %v, got: %v", tc.expectedPatchType, patchType) - } - }) - } -} - -func TestCreatePatchSet(t *testing.T) { - tests := []struct { - name string - targetName string - patchType types.PatchType - expectedPatchSet *patchSet - data string - }{ - { - - name: "valid: YAML patches are separated and converted to JSON", - targetName: "etcd", - patchType: types.StrategicMergePatchType, - data: "foo: bar\n---\nfoo: baz\n", - expectedPatchSet: &patchSet{ - targetName: "etcd", - patchType: types.StrategicMergePatchType, - patches: []string{`{"foo":"bar"}`, `{"foo":"baz"}`}, - }, - }, - { - name: "valid: JSON patches are separated", - targetName: "etcd", - patchType: types.StrategicMergePatchType, - data: `{"foo":"bar"}` + "\n---\n" + `{"foo":"baz"}`, - expectedPatchSet: &patchSet{ - targetName: "etcd", - patchType: types.StrategicMergePatchType, - patches: []string{`{"foo":"bar"}`, `{"foo":"baz"}`}, - }, - }, - { - name: "valid: empty patches are ignored", - targetName: "etcd", - patchType: types.StrategicMergePatchType, - data: `{"foo":"bar"}` + "\n---\n ---\n" + `{"foo":"baz"}`, - expectedPatchSet: &patchSet{ - targetName: "etcd", - patchType: types.StrategicMergePatchType, - patches: []string{`{"foo":"bar"}`, `{"foo":"baz"}`}, - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - ps, _ := createPatchSet(tc.targetName, tc.patchType, tc.data) - if !reflect.DeepEqual(ps, tc.expectedPatchSet) { - t.Fatalf("expected patch set:\n%+v\ngot:\n%+v\n", tc.expectedPatchSet, ps) - } - }) - } -} - -func TestGetPatchSetsForPathMustBeDirectory(t *testing.T) { - tempFile, err := ioutil.TempFile("", "test-file") - if err != nil { - t.Errorf("error creating temporary file: %v", err) - } - defer os.Remove(tempFile.Name()) - - _, _, _, err = getPatchSetsFromPath(tempFile.Name(), testKnownTargets, ioutil.Discard) - if err == nil { - t.Fatalf("expected error for non-directory path %q", tempFile.Name()) - } -} - -func TestGetPatchSetsForPath(t *testing.T) { - const patchData = `{"foo":"bar"}` - - tests := []struct { - name string - filesToWrite []string - expectedPatchSets []*patchSet - expectedPatchFiles []string - expectedIgnoredFiles []string - expectedError bool - patchData string - }{ - { - name: "valid: patch files are sorted and non-patch files are ignored", - filesToWrite: []string{"kube-scheduler+merge.json", "kube-apiserver+json.yaml", "etcd.yaml", "foo", "bar.json"}, - patchData: patchData, - expectedPatchSets: []*patchSet{ - { - targetName: "etcd", - patchType: types.StrategicMergePatchType, - patches: []string{patchData}, - }, - { - targetName: "kube-apiserver", - patchType: types.JSONPatchType, - patches: []string{patchData}, - }, - { - targetName: "kube-scheduler", - patchType: types.MergePatchType, - patches: []string{patchData}, - }, - }, - expectedPatchFiles: []string{"etcd.yaml", "kube-apiserver+json.yaml", "kube-scheduler+merge.json"}, - expectedIgnoredFiles: []string{"bar.json", "foo"}, - }, - { - name: "valid: empty files are ignored", - patchData: "", - filesToWrite: []string{"kube-scheduler.json"}, - expectedPatchFiles: []string{}, - expectedIgnoredFiles: []string{"kube-scheduler.json"}, - expectedPatchSets: []*patchSet{}, - }, - { - name: "invalid: bad patch type in filename returns and error", - filesToWrite: []string{"kube-scheduler+foo.json"}, - expectedError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tempDir, err := ioutil.TempDir("", testDirPattern) - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempDir) - - for _, file := range tc.filesToWrite { - filePath := filepath.Join(tempDir, file) - err := ioutil.WriteFile(filePath, []byte(tc.patchData), 0644) - if err != nil { - t.Fatalf("could not write temporary file %q", filePath) - } - } - - patchSets, patchFiles, ignoredFiles, err := getPatchSetsFromPath(tempDir, testKnownTargets, ioutil.Discard) - if (err != nil) != tc.expectedError { - t.Fatalf("expected error: %v, got: %v, error: %v", tc.expectedError, err != nil, err) - } - - if !reflect.DeepEqual(tc.expectedPatchFiles, patchFiles) { - t.Fatalf("expected patch files:\n%+v\ngot:\n%+v", tc.expectedPatchFiles, patchFiles) - } - if !reflect.DeepEqual(tc.expectedIgnoredFiles, ignoredFiles) { - t.Fatalf("expected ignored files:\n%+v\ngot:\n%+v", tc.expectedIgnoredFiles, ignoredFiles) - } - if !reflect.DeepEqual(tc.expectedPatchSets, patchSets) { - t.Fatalf("expected patch sets:\n%+v\ngot:\n%+v", tc.expectedPatchSets, patchSets) - } - }) - } -} - -func TestGetPatchManagerForPath(t *testing.T) { - type file struct { - name string - data string - } - - tests := []struct { - name string - files []*file - patchTarget *PatchTarget - expectedData []byte - expectedError bool - }{ - { - name: "valid: patch a kube-apiserver target using merge patch; json patch is applied first", - patchTarget: &PatchTarget{ - Name: "kube-apiserver", - StrategicMergePatchObject: v1.Pod{}, - Data: []byte("foo: bar\nbaz: qux\n"), - }, - expectedData: []byte(`{"baz":"qux","foo":"patched"}`), - files: []*file{ - { - name: "kube-apiserver+merge.yaml", - data: "foo: patched", - }, - { - name: "kube-apiserver+json.json", - data: `[{"op": "replace", "path": "/foo", "value": "zzz"}]`, - }, - }, - }, - { - name: "valid: kube-apiserver target is patched with json patch", - patchTarget: &PatchTarget{ - Name: "kube-apiserver", - StrategicMergePatchObject: v1.Pod{}, - Data: []byte("foo: bar\n"), - }, - expectedData: []byte(`{"foo":"zzz"}`), - files: []*file{ - { - name: "kube-apiserver+json.json", - data: `[{"op": "replace", "path": "/foo", "value": "zzz"}]`, - }, - }, - }, - { - name: "valid: kube-apiserver target is patched with strategic merge patch", - patchTarget: &PatchTarget{ - Name: "kube-apiserver", - StrategicMergePatchObject: v1.Pod{}, - Data: []byte("foo: bar\n"), - }, - expectedData: []byte(`{"foo":"zzz"}`), - files: []*file{ - { - name: "kube-apiserver+strategic.json", - data: `{"foo":"zzz"}`, - }, - }, - }, - { - name: "valid: etcd target is not changed because there are no patches for it", - patchTarget: &PatchTarget{ - Name: "etcd", - StrategicMergePatchObject: v1.Pod{}, - Data: []byte("foo: bar\n"), - }, - expectedData: []byte("foo: bar\n"), - files: []*file{ - { - name: "kube-apiserver+merge.yaml", - data: "foo: patched", - }, - }, - }, - { - name: "invalid: cannot patch etcd target due to malformed json patch", - patchTarget: &PatchTarget{ - Name: "etcd", - StrategicMergePatchObject: v1.Pod{}, - Data: []byte("foo: bar\n"), - }, - files: []*file{ - { - name: "etcd+json.json", - data: `{"foo":"zzz"}`, - }, - }, - expectedError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tempDir, err := ioutil.TempDir("", testDirPattern) - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempDir) - - for _, file := range tc.files { - filePath := filepath.Join(tempDir, file.name) - err := ioutil.WriteFile(filePath, []byte(file.data), 0644) - if err != nil { - t.Fatalf("could not write temporary file %q", filePath) - } - } - - pm, err := GetPatchManagerForPath(tempDir, testKnownTargets, nil) - if err != nil { - t.Fatal(err) - } - - err = pm.ApplyPatchesToTarget(tc.patchTarget) - if (err != nil) != tc.expectedError { - t.Fatalf("expected error: %v, got: %v, error: %v", tc.expectedError, err != nil, err) - } - if err != nil { - return - } - - if !bytes.Equal(tc.patchTarget.Data, tc.expectedData) { - t.Fatalf("expected result:\n%s\ngot:\n%s", tc.expectedData, tc.patchTarget.Data) - } - }) - } -} - -func TestGetPatchManagerForPathCache(t *testing.T) { - tempDir, err := ioutil.TempDir("", testDirPattern) - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempDir) - - pmOld, err := GetPatchManagerForPath(tempDir, testKnownTargets, nil) - if err != nil { - t.Fatal(err) - } - pmNew, err := GetPatchManagerForPath(tempDir, testKnownTargets, nil) - if err != nil { - t.Fatal(err) - } - if pmOld != pmNew { - t.Logf("path %q was not cached, expected pointer: %p, got: %p", tempDir, pmOld, pmNew) - } -} diff --git a/cmd/kubeadm/app/util/pkiutil/BUILD b/cmd/kubeadm/app/util/pkiutil/BUILD deleted file mode 100644 index 9c77a4e9c5e91..0000000000000 --- a/cmd/kubeadm/app/util/pkiutil/BUILD +++ /dev/null @@ -1,47 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["pki_helpers_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = ["pki_helpers.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/features:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go deleted file mode 100644 index aa8a76da1316a..0000000000000 --- a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go +++ /dev/null @@ -1,626 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pkiutil - -import ( - "crypto" - "crypto/ecdsa" - "crypto/elliptic" - cryptorand "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "fmt" - "io/ioutil" - "math" - "math/big" - "net" - "os" - "path/filepath" - "time" - - "github.com/pkg/errors" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/validation" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/keyutil" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -const ( - // PrivateKeyBlockType is a possible value for pem.Block.Type. - PrivateKeyBlockType = "PRIVATE KEY" - // PublicKeyBlockType is a possible value for pem.Block.Type. - PublicKeyBlockType = "PUBLIC KEY" - // CertificateBlockType is a possible value for pem.Block.Type. - CertificateBlockType = "CERTIFICATE" - // RSAPrivateKeyBlockType is a possible value for pem.Block.Type. - RSAPrivateKeyBlockType = "RSA PRIVATE KEY" - rsaKeySize = 2048 -) - -// CertConfig is a wrapper around certutil.Config extending it with PublicKeyAlgorithm. -type CertConfig struct { - certutil.Config - PublicKeyAlgorithm x509.PublicKeyAlgorithm -} - -// NewCertificateAuthority creates new certificate and private key for the certificate authority -func NewCertificateAuthority(config *CertConfig) (*x509.Certificate, crypto.Signer, error) { - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key while generating CA certificate") - } - - cert, err := certutil.NewSelfSignedCACert(config.Config, key) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create self-signed CA certificate") - } - - return cert, key, nil -} - -// NewCertAndKey creates new certificate and key by passing the certificate authority certificate and key -func NewCertAndKey(caCert *x509.Certificate, caKey crypto.Signer, config *CertConfig) (*x509.Certificate, crypto.Signer, error) { - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key") - } - - cert, err := NewSignedCert(config, key, caCert, caKey) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to sign certificate") - } - - return cert, key, nil -} - -// NewCSRAndKey generates a new key and CSR and that could be signed to create the given certificate -func NewCSRAndKey(config *CertConfig) (*x509.CertificateRequest, crypto.Signer, error) { - key, err := NewPrivateKey(config.PublicKeyAlgorithm) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to create private key") - } - - csr, err := NewCSR(*config, key) - if err != nil { - return nil, nil, errors.Wrap(err, "unable to generate CSR") - } - - return csr, key, nil -} - -// HasServerAuth returns true if the given certificate is a ServerAuth -func HasServerAuth(cert *x509.Certificate) bool { - for i := range cert.ExtKeyUsage { - if cert.ExtKeyUsage[i] == x509.ExtKeyUsageServerAuth { - return true - } - } - return false -} - -// WriteCertAndKey stores certificate and key at the specified location -func WriteCertAndKey(pkiPath string, name string, cert *x509.Certificate, key crypto.Signer) error { - if err := WriteKey(pkiPath, name, key); err != nil { - return errors.Wrap(err, "couldn't write key") - } - - return WriteCert(pkiPath, name, cert) -} - -// WriteCert stores the given certificate at the given location -func WriteCert(pkiPath, name string, cert *x509.Certificate) error { - if cert == nil { - return errors.New("certificate cannot be nil when writing to file") - } - - certificatePath := pathForCert(pkiPath, name) - if err := certutil.WriteCert(certificatePath, EncodeCertPEM(cert)); err != nil { - return errors.Wrapf(err, "unable to write certificate to file %s", certificatePath) - } - - return nil -} - -// WriteKey stores the given key at the given location -func WriteKey(pkiPath, name string, key crypto.Signer) error { - if key == nil { - return errors.New("private key cannot be nil when writing to file") - } - - privateKeyPath := pathForKey(pkiPath, name) - encoded, err := keyutil.MarshalPrivateKeyToPEM(key) - if err != nil { - return errors.Wrapf(err, "unable to marshal private key to PEM") - } - if err := keyutil.WriteKey(privateKeyPath, encoded); err != nil { - return errors.Wrapf(err, "unable to write private key to file %s", privateKeyPath) - } - - return nil -} - -// WriteCSR writes the pem-encoded CSR data to csrPath. -// The CSR file will be created with file mode 0600. -// If the CSR file already exists, it will be overwritten. -// The parent directory of the csrPath will be created as needed with file mode 0700. -func WriteCSR(csrDir, name string, csr *x509.CertificateRequest) error { - if csr == nil { - return errors.New("certificate request cannot be nil when writing to file") - } - - csrPath := pathForCSR(csrDir, name) - if err := os.MkdirAll(filepath.Dir(csrPath), os.FileMode(0700)); err != nil { - return errors.Wrapf(err, "failed to make directory %s", filepath.Dir(csrPath)) - } - - if err := ioutil.WriteFile(csrPath, EncodeCSRPEM(csr), os.FileMode(0600)); err != nil { - return errors.Wrapf(err, "unable to write CSR to file %s", csrPath) - } - - return nil -} - -// WritePublicKey stores the given public key at the given location -func WritePublicKey(pkiPath, name string, key crypto.PublicKey) error { - if key == nil { - return errors.New("public key cannot be nil when writing to file") - } - - publicKeyBytes, err := EncodePublicKeyPEM(key) - if err != nil { - return err - } - publicKeyPath := pathForPublicKey(pkiPath, name) - if err := keyutil.WriteKey(publicKeyPath, publicKeyBytes); err != nil { - return errors.Wrapf(err, "unable to write public key to file %s", publicKeyPath) - } - - return nil -} - -// CertOrKeyExist returns a boolean whether the cert or the key exists -func CertOrKeyExist(pkiPath, name string) bool { - certificatePath, privateKeyPath := PathsForCertAndKey(pkiPath, name) - - _, certErr := os.Stat(certificatePath) - _, keyErr := os.Stat(privateKeyPath) - if os.IsNotExist(certErr) && os.IsNotExist(keyErr) { - // The cert and the key do not exist - return false - } - - // Both files exist or one of them - return true -} - -// CSROrKeyExist returns true if one of the CSR or key exists -func CSROrKeyExist(csrDir, name string) bool { - csrPath := pathForCSR(csrDir, name) - keyPath := pathForKey(csrDir, name) - - _, csrErr := os.Stat(csrPath) - _, keyErr := os.Stat(keyPath) - - return !(os.IsNotExist(csrErr) && os.IsNotExist(keyErr)) -} - -// TryLoadCertAndKeyFromDisk tries to load a cert and a key from the disk and validates that they are valid -func TryLoadCertAndKeyFromDisk(pkiPath, name string) (*x509.Certificate, crypto.Signer, error) { - cert, err := TryLoadCertFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load certificate") - } - - key, err := TryLoadKeyFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to load key") - } - - return cert, key, nil -} - -// TryLoadCertFromDisk tries to load the cert from the disk -func TryLoadCertFromDisk(pkiPath, name string) (*x509.Certificate, error) { - certificatePath := pathForCert(pkiPath, name) - - certs, err := certutil.CertsFromFile(certificatePath) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load the certificate file %s", certificatePath) - } - - // We are only putting one certificate in the certificate pem file, so it's safe to just pick the first one - // TODO: Support multiple certs here in order to be able to rotate certs - cert := certs[0] - - return cert, nil -} - -// TryLoadKeyFromDisk tries to load the key from the disk and validates that it is valid -func TryLoadKeyFromDisk(pkiPath, name string) (crypto.Signer, error) { - privateKeyPath := pathForKey(pkiPath, name) - - // Parse the private key from a file - privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath) - if err != nil { - return nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath) - } - - // Allow RSA and ECDSA formats only - var key crypto.Signer - switch k := privKey.(type) { - case *rsa.PrivateKey: - key = k - case *ecdsa.PrivateKey: - key = k - default: - return nil, errors.Errorf("the private key file %s is neither in RSA nor ECDSA format", privateKeyPath) - } - - return key, nil -} - -// TryLoadCSRAndKeyFromDisk tries to load the CSR and key from the disk -func TryLoadCSRAndKeyFromDisk(pkiPath, name string) (*x509.CertificateRequest, crypto.Signer, error) { - csr, err := TryLoadCSRFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "could not load CSR file") - } - - key, err := TryLoadKeyFromDisk(pkiPath, name) - if err != nil { - return nil, nil, errors.Wrap(err, "could not load key file") - } - - return csr, key, nil -} - -// TryLoadPrivatePublicKeyFromDisk tries to load the key from the disk and validates that it is valid -func TryLoadPrivatePublicKeyFromDisk(pkiPath, name string) (*rsa.PrivateKey, *rsa.PublicKey, error) { - privateKeyPath := pathForKey(pkiPath, name) - - // Parse the private key from a file - privKey, err := keyutil.PrivateKeyFromFile(privateKeyPath) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't load the private key file %s", privateKeyPath) - } - - publicKeyPath := pathForPublicKey(pkiPath, name) - - // Parse the public key from a file - pubKeys, err := keyutil.PublicKeysFromFile(publicKeyPath) - if err != nil { - return nil, nil, errors.Wrapf(err, "couldn't load the public key file %s", publicKeyPath) - } - - // Allow RSA format only - k, ok := privKey.(*rsa.PrivateKey) - if !ok { - return nil, nil, errors.Errorf("the private key file %s isn't in RSA format", privateKeyPath) - } - - p := pubKeys[0].(*rsa.PublicKey) - - return k, p, nil -} - -// TryLoadCSRFromDisk tries to load the CSR from the disk -func TryLoadCSRFromDisk(pkiPath, name string) (*x509.CertificateRequest, error) { - csrPath := pathForCSR(pkiPath, name) - - csr, err := CertificateRequestFromFile(csrPath) - if err != nil { - return nil, errors.Wrapf(err, "could not load the CSR %s", csrPath) - } - - return csr, nil -} - -// PathsForCertAndKey returns the paths for the certificate and key given the path and basename. -func PathsForCertAndKey(pkiPath, name string) (string, string) { - return pathForCert(pkiPath, name), pathForKey(pkiPath, name) -} - -func pathForCert(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name)) -} - -func pathForKey(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name)) -} - -func pathForPublicKey(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s.pub", name)) -} - -func pathForCSR(pkiPath, name string) string { - return filepath.Join(pkiPath, fmt.Sprintf("%s.csr", name)) -} - -// GetAPIServerAltNames builds an AltNames object for to be used when generating apiserver certificate -func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) { - // advertise address - advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) - if advertiseAddress == nil { - return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", - cfg.LocalAPIEndpoint.AdvertiseAddress) - } - - internalAPIServerVirtualIP, err := kubeadmconstants.GetAPIServerVirtualIP(cfg.Networking.ServiceSubnet, features.Enabled(cfg.FeatureGates, features.IPv6DualStack)) - if err != nil { - return nil, errors.Wrapf(err, "unable to get first IP address from the given CIDR: %v", cfg.Networking.ServiceSubnet) - } - - // create AltNames with defaults DNSNames/IPs - altNames := &certutil.AltNames{ - DNSNames: []string{ - cfg.NodeRegistration.Name, - "kubernetes", - "kubernetes.default", - "kubernetes.default.svc", - fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain), - }, - IPs: []net.IP{ - internalAPIServerVirtualIP, - advertiseAddress, - }, - } - - // add cluster controlPlaneEndpoint if present (dns or ip) - if len(cfg.ControlPlaneEndpoint) > 0 { - if host, _, err := kubeadmutil.ParseHostPort(cfg.ControlPlaneEndpoint); err == nil { - if ip := net.ParseIP(host); ip != nil { - altNames.IPs = append(altNames.IPs, ip) - } else { - altNames.DNSNames = append(altNames.DNSNames, host) - } - } else { - return nil, errors.Wrapf(err, "error parsing cluster controlPlaneEndpoint %q", cfg.ControlPlaneEndpoint) - } - } - - appendSANsToAltNames(altNames, cfg.APIServer.CertSANs, kubeadmconstants.APIServerCertName) - - return altNames, nil -} - -// GetEtcdAltNames builds an AltNames object for generating the etcd server certificate. -// `advertise address` and localhost are included in the SAN since this is the interfaces the etcd static pod listens on. -// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.ServerCertSANs`. -func GetEtcdAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) { - return getAltNames(cfg, kubeadmconstants.EtcdServerCertName) -} - -// GetEtcdPeerAltNames builds an AltNames object for generating the etcd peer certificate. -// Hostname and `API.AdvertiseAddress` are included if the user chooses to promote the single node etcd cluster into a multi-node one (stacked etcd). -// The user can override the listen address with `Etcd.ExtraArgs` and add SANs with `Etcd.PeerCertSANs`. -func GetEtcdPeerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames, error) { - return getAltNames(cfg, kubeadmconstants.EtcdPeerCertName) -} - -// getAltNames builds an AltNames object with the cfg and certName. -func getAltNames(cfg *kubeadmapi.InitConfiguration, certName string) (*certutil.AltNames, error) { - // advertise address - advertiseAddress := net.ParseIP(cfg.LocalAPIEndpoint.AdvertiseAddress) - if advertiseAddress == nil { - return nil, errors.Errorf("error parsing LocalAPIEndpoint AdvertiseAddress %v: is not a valid textual representation of an IP address", - cfg.LocalAPIEndpoint.AdvertiseAddress) - } - - // create AltNames with defaults DNSNames/IPs - altNames := &certutil.AltNames{ - DNSNames: []string{cfg.NodeRegistration.Name, "localhost"}, - IPs: []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback}, - } - - if cfg.Etcd.Local != nil { - if certName == kubeadmconstants.EtcdServerCertName { - appendSANsToAltNames(altNames, cfg.Etcd.Local.ServerCertSANs, kubeadmconstants.EtcdServerCertName) - } else if certName == kubeadmconstants.EtcdPeerCertName { - appendSANsToAltNames(altNames, cfg.Etcd.Local.PeerCertSANs, kubeadmconstants.EtcdPeerCertName) - } - } - return altNames, nil -} - -// appendSANsToAltNames parses SANs from as list of strings and adds them to altNames for use on a specific cert -// altNames is passed in with a pointer, and the struct is modified -// valid IP address strings are parsed and added to altNames.IPs as net.IP's -// RFC-1123 compliant DNS strings are added to altNames.DNSNames as strings -// RFC-1123 compliant wildcard DNS strings are added to altNames.DNSNames as strings -// certNames is used to print user facing warnings and should be the name of the cert the altNames will be used for -func appendSANsToAltNames(altNames *certutil.AltNames, SANs []string, certName string) { - for _, altname := range SANs { - if ip := net.ParseIP(altname); ip != nil { - altNames.IPs = append(altNames.IPs, ip) - } else if len(validation.IsDNS1123Subdomain(altname)) == 0 { - altNames.DNSNames = append(altNames.DNSNames, altname) - } else if len(validation.IsWildcardDNS1123Subdomain(altname)) == 0 { - altNames.DNSNames = append(altNames.DNSNames, altname) - } else { - fmt.Printf( - "[certificates] WARNING: '%s' was not added to the '%s' SAN, because it is not a valid IP or RFC-1123 compliant DNS entry\n", - altname, - certName, - ) - } - } -} - -// EncodeCSRPEM returns PEM-encoded CSR data -func EncodeCSRPEM(csr *x509.CertificateRequest) []byte { - block := pem.Block{ - Type: certutil.CertificateRequestBlockType, - Bytes: csr.Raw, - } - return pem.EncodeToMemory(&block) -} - -func parseCSRPEM(pemCSR []byte) (*x509.CertificateRequest, error) { - block, _ := pem.Decode(pemCSR) - if block == nil { - return nil, errors.New("data doesn't contain a valid certificate request") - } - - if block.Type != certutil.CertificateRequestBlockType { - return nil, errors.Errorf("expected block type %q, but PEM had type %q", certutil.CertificateRequestBlockType, block.Type) - } - - return x509.ParseCertificateRequest(block.Bytes) -} - -// CertificateRequestFromFile returns the CertificateRequest from a given PEM-encoded file. -// Returns an error if the file could not be read or if the CSR could not be parsed. -func CertificateRequestFromFile(file string) (*x509.CertificateRequest, error) { - pemBlock, err := ioutil.ReadFile(file) - if err != nil { - return nil, errors.Wrap(err, "failed to read file") - } - - csr, err := parseCSRPEM(pemBlock) - if err != nil { - return nil, errors.Wrapf(err, "error reading certificate request file %s", file) - } - return csr, nil -} - -// NewCSR creates a new CSR -func NewCSR(cfg CertConfig, key crypto.Signer) (*x509.CertificateRequest, error) { - template := &x509.CertificateRequest{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: cfg.Organization, - }, - DNSNames: cfg.AltNames.DNSNames, - IPAddresses: cfg.AltNames.IPs, - } - - csrBytes, err := x509.CreateCertificateRequest(cryptorand.Reader, template, key) - - if err != nil { - return nil, errors.Wrap(err, "failed to create a CSR") - } - - return x509.ParseCertificateRequest(csrBytes) -} - -// EncodeCertPEM returns PEM-endcoded certificate data -func EncodeCertPEM(cert *x509.Certificate) []byte { - block := pem.Block{ - Type: CertificateBlockType, - Bytes: cert.Raw, - } - return pem.EncodeToMemory(&block) -} - -// EncodePublicKeyPEM returns PEM-encoded public data -func EncodePublicKeyPEM(key crypto.PublicKey) ([]byte, error) { - der, err := x509.MarshalPKIXPublicKey(key) - if err != nil { - return []byte{}, err - } - block := pem.Block{ - Type: PublicKeyBlockType, - Bytes: der, - } - return pem.EncodeToMemory(&block), nil -} - -// NewPrivateKey creates an RSA private key -func NewPrivateKey(keyType x509.PublicKeyAlgorithm) (crypto.Signer, error) { - if keyType == x509.ECDSA { - return ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader) - } - - return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) -} - -// NewSignedCert creates a signed certificate using the given CA certificate and key -func NewSignedCert(cfg *CertConfig, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { - serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) - if err != nil { - return nil, err - } - if len(cfg.CommonName) == 0 { - return nil, errors.New("must specify a CommonName") - } - if len(cfg.Usages) == 0 { - return nil, errors.New("must specify at least one ExtKeyUsage") - } - - RemoveDuplicateAltNames(&cfg.AltNames) - - certTmpl := x509.Certificate{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: cfg.Organization, - }, - DNSNames: cfg.AltNames.DNSNames, - IPAddresses: cfg.AltNames.IPs, - SerialNumber: serial, - NotBefore: caCert.NotBefore, - NotAfter: time.Now().Add(kubeadmconstants.CertificateValidity).UTC(), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: cfg.Usages, - } - certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) - if err != nil { - return nil, err - } - return x509.ParseCertificate(certDERBytes) -} - -// RemoveDuplicateAltNames removes duplicate items in altNames. -func RemoveDuplicateAltNames(altNames *certutil.AltNames) { - if altNames == nil { - return - } - - if altNames.DNSNames != nil { - altNames.DNSNames = sets.NewString(altNames.DNSNames...).List() - } - - ipsKeys := make(map[string]struct{}) - var ips []net.IP - for _, one := range altNames.IPs { - if _, ok := ipsKeys[one.String()]; !ok { - ipsKeys[one.String()] = struct{}{} - ips = append(ips, one) - } - } - altNames.IPs = ips -} - -// ValidateCertPeriod checks if the certificate is valid relative to the current time -// (+/- offset) -func ValidateCertPeriod(cert *x509.Certificate, offset time.Duration) error { - period := fmt.Sprintf("NotBefore: %v, NotAfter: %v", cert.NotBefore, cert.NotAfter) - now := time.Now().Add(offset) - if now.Before(cert.NotBefore) { - return errors.Errorf("the certificate is not valid yet: %s", period) - } - if now.After(cert.NotAfter) { - return errors.Errorf("the certificate has expired: %s", period) - } - return nil -} diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go deleted file mode 100644 index ca411adf45f93..0000000000000 --- a/cmd/kubeadm/app/util/pkiutil/pki_helpers_test.go +++ /dev/null @@ -1,806 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pkiutil - -import ( - "crypto" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "io/ioutil" - "net" - "os" - "reflect" - "testing" - - certutil "k8s.io/client-go/util/cert" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" -) - -func TestNewCertificateAuthority(t *testing.T) { - cert, key, err := NewCertificateAuthority(&CertConfig{ - Config: certutil.Config{CommonName: "kubernetes"}, - }) - - if cert == nil { - t.Error("failed NewCertificateAuthority, cert == nil") - } else if !cert.IsCA { - t.Error("cert is not a valida CA") - } - - if key == nil { - t.Error("failed NewCertificateAuthority, key == nil") - } - - if err != nil { - t.Errorf("failed NewCertificateAuthority with an error: %+v", err) - } -} - -func TestNewCertAndKey(t *testing.T) { - var tests = []struct { - name string - keyGenFunc func() (crypto.Signer, error) - expected bool - }{ - { - name: "RSA key too small", - keyGenFunc: func() (crypto.Signer, error) { - return rsa.GenerateKey(rand.Reader, 128) - }, - expected: false, - }, - { - name: "RSA should succeed", - keyGenFunc: func() (crypto.Signer, error) { - return rsa.GenerateKey(rand.Reader, 2048) - }, - expected: true, - }, - { - name: "ECDSA should succeed", - keyGenFunc: func() (crypto.Signer, error) { - return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - }, - expected: true, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - caKey, err := rt.keyGenFunc() - if err != nil { - t.Fatalf("Couldn't create Private Key") - } - caCert := &x509.Certificate{} - config := &CertConfig{ - Config: certutil.Config{ - CommonName: "test", - Organization: []string{"test"}, - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - } - _, _, actual := NewCertAndKey(caCert, caKey, config) - if (actual == nil) != rt.expected { - t.Errorf( - "failed NewCertAndKey:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) - } - }) - } -} - -func TestHasServerAuth(t *testing.T) { - caCert, caKey, _ := NewCertificateAuthority(&CertConfig{Config: certutil.Config{CommonName: "kubernetes"}}) - - var tests = []struct { - name string - config CertConfig - expected bool - }{ - { - name: "has ServerAuth", - config: CertConfig{ - Config: certutil.Config{ - CommonName: "test", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - }, - expected: true, - }, - { - name: "has ServerAuth ECDSA", - config: CertConfig{ - Config: certutil.Config{ - CommonName: "test", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - PublicKeyAlgorithm: x509.ECDSA, - }, - expected: true, - }, - { - name: "doesn't have ServerAuth", - config: CertConfig{ - Config: certutil.Config{ - CommonName: "test", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - }, - }, - expected: false, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - cert, _, err := NewCertAndKey(caCert, caKey, &rt.config) - if err != nil { - t.Fatalf("Couldn't create cert: %v", err) - } - actual := HasServerAuth(cert) - if actual != rt.expected { - t.Errorf( - "failed HasServerAuth:\n\texpected: %t\n\t actual: %t", - rt.expected, - actual, - ) - } - }) - } -} - -func TestWriteCertAndKey(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatalf("Couldn't create rsa Private Key") - } - caCert := &x509.Certificate{} - actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey) - if actual != nil { - t.Errorf( - "failed WriteCertAndKey with an error: %v", - actual, - ) - } -} - -func TestWriteCert(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caCert := &x509.Certificate{} - actual := WriteCert(tmpdir, "foo", caCert) - if actual != nil { - t.Errorf( - "failed WriteCertAndKey with an error: %v", - actual, - ) - } -} - -func TestWriteKey(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatalf("Couldn't create rsa Private Key") - } - actual := WriteKey(tmpdir, "foo", caKey) - if actual != nil { - t.Errorf( - "failed WriteCertAndKey with an error: %v", - actual, - ) - } -} - -func TestWritePublicKey(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatalf("Couldn't create rsa Private Key") - } - actual := WritePublicKey(tmpdir, "foo", &caKey.PublicKey) - if actual != nil { - t.Errorf( - "failed WriteCertAndKey with an error: %v", - actual, - ) - } -} - -func TestCertOrKeyExist(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - t.Fatalf("Couldn't create rsa Private Key") - } - caCert := &x509.Certificate{} - actual := WriteCertAndKey(tmpdir, "foo", caCert, caKey) - if actual != nil { - t.Errorf( - "failed WriteCertAndKey with an error: %v", - actual, - ) - } - - var tests = []struct { - desc string - path string - name string - expected bool - }{ - { - desc: "empty path and name", - path: "", - name: "", - expected: false, - }, - { - desc: "valid path and name", - path: tmpdir, - name: "foo", - expected: true, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := CertOrKeyExist(rt.path, rt.name) - if actual != rt.expected { - t.Errorf( - "failed CertOrKeyExist:\n\texpected: %t\n\t actual: %t", - rt.expected, - actual, - ) - } - }) - } -} - -func TestTryLoadCertAndKeyFromDisk(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caCert, caKey, err := NewCertificateAuthority(&CertConfig{ - Config: certutil.Config{CommonName: "kubernetes"}, - }) - if err != nil { - t.Errorf( - "failed to create cert and key with an error: %v", - err, - ) - } - err = WriteCertAndKey(tmpdir, "foo", caCert, caKey) - if err != nil { - t.Errorf( - "failed to write cert and key with an error: %v", - err, - ) - } - - var tests = []struct { - desc string - path string - name string - expected bool - }{ - { - desc: "empty path and name", - path: "", - name: "", - expected: false, - }, - { - desc: "valid path and name", - path: tmpdir, - name: "foo", - expected: true, - }, - } - for _, rt := range tests { - t.Run(rt.desc, func(t *testing.T) { - _, _, actual := TryLoadCertAndKeyFromDisk(rt.path, rt.name) - if (actual == nil) != rt.expected { - t.Errorf( - "failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) - } - }) - } -} - -func TestTryLoadCertFromDisk(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caCert, _, err := NewCertificateAuthority(&CertConfig{ - Config: certutil.Config{CommonName: "kubernetes"}, - }) - if err != nil { - t.Errorf( - "failed to create cert and key with an error: %v", - err, - ) - } - err = WriteCert(tmpdir, "foo", caCert) - if err != nil { - t.Errorf( - "failed to write cert and key with an error: %v", - err, - ) - } - - var tests = []struct { - desc string - path string - name string - expected bool - }{ - { - desc: "empty path and name", - path: "", - name: "", - expected: false, - }, - { - desc: "valid path and name", - path: tmpdir, - name: "foo", - expected: true, - }, - } - for _, rt := range tests { - t.Run(rt.desc, func(t *testing.T) { - _, actual := TryLoadCertFromDisk(rt.path, rt.name) - if (actual == nil) != rt.expected { - t.Errorf( - "failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) - } - }) - } -} - -func TestTryLoadKeyFromDisk(t *testing.T) { - - var tests = []struct { - desc string - pathSuffix string - name string - keyGenFunc func() (crypto.Signer, error) - expected bool - }{ - { - desc: "empty path and name", - pathSuffix: "somegarbage", - name: "", - keyGenFunc: func() (crypto.Signer, error) { - return rsa.GenerateKey(rand.Reader, 2048) - }, - expected: false, - }, - { - desc: "RSA valid path and name", - pathSuffix: "", - name: "foo", - keyGenFunc: func() (crypto.Signer, error) { - return rsa.GenerateKey(rand.Reader, 2048) - }, - expected: true, - }, - { - desc: "ECDSA valid path and name", - pathSuffix: "", - name: "foo", - keyGenFunc: func() (crypto.Signer, error) { - return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - }, - expected: true, - }, - } - for _, rt := range tests { - t.Run(rt.desc, func(t *testing.T) { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - defer os.RemoveAll(tmpdir) - - caKey, err := rt.keyGenFunc() - if err != nil { - t.Errorf( - "failed to create key with an error: %v", - err, - ) - } - - err = WriteKey(tmpdir, "foo", caKey) - if err != nil { - t.Errorf( - "failed to write key with an error: %v", - err, - ) - } - _, actual := TryLoadKeyFromDisk(tmpdir+rt.pathSuffix, rt.name) - if (actual == nil) != rt.expected { - t.Errorf( - "failed TryLoadCertAndKeyFromDisk:\n\texpected: %t\n\t actual: %t", - rt.expected, - (actual == nil), - ) - } - }) - } -} - -func TestPathsForCertAndKey(t *testing.T) { - crtPath, keyPath := PathsForCertAndKey("/foo", "bar") - if crtPath != "/foo/bar.crt" { - t.Errorf("unexpected certificate path: %s", crtPath) - } - if keyPath != "/foo/bar.key" { - t.Errorf("unexpected key path: %s", keyPath) - } -} - -func TestPathForCert(t *testing.T) { - crtPath := pathForCert("/foo", "bar") - if crtPath != "/foo/bar.crt" { - t.Errorf("unexpected certificate path: %s", crtPath) - } -} - -func TestPathForKey(t *testing.T) { - keyPath := pathForKey("/foo", "bar") - if keyPath != "/foo/bar.key" { - t.Errorf("unexpected certificate path: %s", keyPath) - } -} - -func TestPathForPublicKey(t *testing.T) { - pubPath := pathForPublicKey("/foo", "bar") - if pubPath != "/foo/bar.pub" { - t.Errorf("unexpected certificate path: %s", pubPath) - } -} - -func TestPathForCSR(t *testing.T) { - csrPath := pathForCSR("/foo", "bar") - if csrPath != "/foo/bar.csr" { - t.Errorf("unexpected certificate path: %s", csrPath) - } -} - -func TestGetAPIServerAltNames(t *testing.T) { - - var tests = []struct { - desc string - name string - cfg *kubeadmapi.InitConfiguration - expectedDNSNames []string - expectedIPAddresses []string - }{ - { - desc: "empty name", - name: "", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "api.k8s.io:6443", - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - APIServer: kubeadmapi.APIServer{ - CertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"}, - }, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"}, - }, - expectedDNSNames: []string{"valid-hostname", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", "api.k8s.io"}, - expectedIPAddresses: []string{"10.96.0.1", "1.2.3.4", "10.1.245.94", "10.1.245.95"}, - }, - { - desc: "ControlPlaneEndpoint IP", - name: "ControlPlaneEndpoint IP", - cfg: &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: "1.2.3.4"}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - ControlPlaneEndpoint: "4.5.6.7:6443", - Networking: kubeadmapi.Networking{ServiceSubnet: "10.96.0.0/12", DNSDomain: "cluster.local"}, - APIServer: kubeadmapi.APIServer{ - CertSANs: []string{"10.1.245.94", "10.1.245.95", "1.2.3.L", "invalid,commas,in,DNS"}, - }, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: "valid-hostname"}, - }, - expectedDNSNames: []string{"valid-hostname", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local"}, - expectedIPAddresses: []string{"10.96.0.1", "1.2.3.4", "10.1.245.94", "10.1.245.95", "4.5.6.7"}, - }, - } - - for _, rt := range tests { - t.Run(rt.desc, func(t *testing.T) { - altNames, err := GetAPIServerAltNames(rt.cfg) - if err != nil { - t.Fatalf("failed calling GetAPIServerAltNames: %s: %v", rt.name, err) - } - - for _, DNSName := range rt.expectedDNSNames { - found := false - for _, val := range altNames.DNSNames { - if val == DNSName { - found = true - break - } - } - - if !found { - t.Errorf("%s: altNames does not contain DNSName %s but %v", rt.name, DNSName, altNames.DNSNames) - } - } - - for _, IPAddress := range rt.expectedIPAddresses { - found := false - for _, val := range altNames.IPs { - if val.Equal(net.ParseIP(IPAddress)) { - found = true - break - } - } - - if !found { - t.Errorf("%s: altNames does not contain IPAddress %s but %v", rt.name, IPAddress, altNames.IPs) - } - } - }) - } -} - -func TestGetEtcdAltNames(t *testing.T) { - proxy := "user-etcd-proxy" - proxyIP := "10.10.10.100" - cfg := &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{ - AdvertiseAddress: "1.2.3.4", - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{ - Name: "myNode", - }, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ServerCertSANs: []string{ - proxy, - proxyIP, - "1.2.3.L", - "invalid,commas,in,DNS", - }, - }, - }, - }, - } - - altNames, err := GetEtcdAltNames(cfg) - if err != nil { - t.Fatalf("failed calling GetEtcdAltNames: %v", err) - } - - expectedDNSNames := []string{"myNode", "localhost", proxy} - for _, DNSName := range expectedDNSNames { - t.Run(DNSName, func(t *testing.T) { - found := false - for _, val := range altNames.DNSNames { - if val == DNSName { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain DNSName %s", DNSName) - } - }) - } - - expectedIPAddresses := []string{"1.2.3.4", "127.0.0.1", net.IPv6loopback.String(), proxyIP} - for _, IPAddress := range expectedIPAddresses { - t.Run(IPAddress, func(t *testing.T) { - found := false - for _, val := range altNames.IPs { - if val.Equal(net.ParseIP(IPAddress)) { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain IPAddress %s", IPAddress) - } - }) - } -} - -func TestGetEtcdPeerAltNames(t *testing.T) { - hostname := "valid-hostname" - proxy := "user-etcd-proxy" - proxyIP := "10.10.10.100" - advertiseIP := "1.2.3.4" - cfg := &kubeadmapi.InitConfiguration{ - LocalAPIEndpoint: kubeadmapi.APIEndpoint{AdvertiseAddress: advertiseIP}, - ClusterConfiguration: kubeadmapi.ClusterConfiguration{ - Etcd: kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - PeerCertSANs: []string{ - proxy, - proxyIP, - "1.2.3.L", - "invalid,commas,in,DNS", - }, - }, - }, - }, - NodeRegistration: kubeadmapi.NodeRegistrationOptions{Name: hostname}, - } - - altNames, err := GetEtcdPeerAltNames(cfg) - if err != nil { - t.Fatalf("failed calling GetEtcdPeerAltNames: %v", err) - } - - expectedDNSNames := []string{hostname, proxy} - for _, DNSName := range expectedDNSNames { - t.Run(DNSName, func(t *testing.T) { - found := false - for _, val := range altNames.DNSNames { - if val == DNSName { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain DNSName %s", DNSName) - } - - expectedIPAddresses := []string{advertiseIP, proxyIP} - for _, IPAddress := range expectedIPAddresses { - found := false - for _, val := range altNames.IPs { - if val.Equal(net.ParseIP(IPAddress)) { - found = true - break - } - } - - if !found { - t.Errorf("altNames does not contain IPAddress %s", IPAddress) - } - } - }) - } -} - -func TestAppendSANsToAltNames(t *testing.T) { - var tests = []struct { - sans []string - expected int - }{ - {[]string{}, 0}, - {[]string{"abc"}, 1}, - {[]string{"*.abc"}, 1}, - {[]string{"**.abc"}, 0}, - {[]string{"a.*.bc"}, 0}, - {[]string{"a.*.bc", "abc.def"}, 1}, - {[]string{"a*.bc", "abc.def"}, 1}, - } - for _, rt := range tests { - altNames := certutil.AltNames{} - appendSANsToAltNames(&altNames, rt.sans, "foo") - actual := len(altNames.DNSNames) - if actual != rt.expected { - t.Errorf( - "failed AppendSANsToAltNames Numbers:\n\texpected: %d\n\t actual: %d", - rt.expected, - actual, - ) - } - } - -} - -func TestRemoveDuplicateAltNames(t *testing.T) { - tests := []struct { - args *certutil.AltNames - want *certutil.AltNames - }{ - { - &certutil.AltNames{}, - &certutil.AltNames{}, - }, - { - &certutil.AltNames{ - DNSNames: []string{"a", "a"}, - IPs: []net.IP{{127, 0, 0, 1}}, - }, - &certutil.AltNames{ - DNSNames: []string{"a"}, - IPs: []net.IP{{127, 0, 0, 1}}, - }, - }, - { - &certutil.AltNames{ - DNSNames: []string{"a"}, - IPs: []net.IP{{127, 0, 0, 1}, {127, 0, 0, 1}}, - }, - &certutil.AltNames{ - DNSNames: []string{"a"}, - IPs: []net.IP{{127, 0, 0, 1}}, - }, - }, - { - &certutil.AltNames{ - DNSNames: []string{"a", "a"}, - IPs: []net.IP{{127, 0, 0, 1}, {127, 0, 0, 1}}, - }, - &certutil.AltNames{ - DNSNames: []string{"a"}, - IPs: []net.IP{{127, 0, 0, 1}}, - }, - }, - } - for _, tt := range tests { - RemoveDuplicateAltNames(tt.args) - if !reflect.DeepEqual(tt.args, tt.want) { - t.Errorf("Wanted %v, got %v", tt.want, tt.args) - } - } -} diff --git a/cmd/kubeadm/app/util/pubkeypin/BUILD b/cmd/kubeadm/app/util/pubkeypin/BUILD deleted file mode 100644 index 65b88d8d9fe73..0000000000000 --- a/cmd/kubeadm/app/util/pubkeypin/BUILD +++ /dev/null @@ -1,33 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["pubkeypin_test.go"], - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["pubkeypin.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin", - deps = ["//vendor/github.com/pkg/errors:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go b/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go deleted file mode 100644 index fb157160d6a80..0000000000000 --- a/cmd/kubeadm/app/util/pubkeypin/pubkeypin.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package pubkeypin provides primitives for x509 public key pinning in the -// style of RFC7469. -package pubkeypin - -import ( - "crypto/sha256" - "crypto/x509" - "encoding/hex" - "strings" - - "github.com/pkg/errors" -) - -const ( - // formatSHA256 is the prefix for pins that are full-length SHA-256 hashes encoded in base 16 (hex) - formatSHA256 = "sha256" -) - -// Set is a set of pinned x509 public keys. -type Set struct { - sha256Hashes map[string]bool -} - -// NewSet returns a new, empty PubKeyPinSet -func NewSet() *Set { - return &Set{make(map[string]bool)} -} - -// Allow adds an allowed public key hash to the Set -func (s *Set) Allow(pubKeyHashes ...string) error { - for _, pubKeyHash := range pubKeyHashes { - parts := strings.Split(pubKeyHash, ":") - if len(parts) != 2 { - return errors.New("invalid public key hash, expected \"format:value\"") - } - format, value := parts[0], parts[1] - - switch strings.ToLower(format) { - case "sha256": - return s.allowSHA256(value) - default: - return errors.Errorf("unknown hash format %q", format) - } - } - return nil -} - -// CheckAny checks if at least one certificate matches one of the public keys in the set -func (s *Set) CheckAny(certificates []*x509.Certificate) error { - var hashes []string - - for _, certificate := range certificates { - if s.checkSHA256(certificate) { - return nil - } - - hashes = append(hashes, Hash(certificate)) - } - return errors.Errorf("none of the public keys %q are pinned", strings.Join(hashes, ":")) -} - -// Empty returns true if the Set contains no pinned public keys. -func (s *Set) Empty() bool { - return len(s.sha256Hashes) == 0 -} - -// Hash calculates the SHA-256 hash of the Subject Public Key Information (SPKI) -// object in an x509 certificate (in DER encoding). It returns the full hash as a -// hex encoded string (suitable for passing to Set.Allow). -func Hash(certificate *x509.Certificate) string { - spkiHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) - return formatSHA256 + ":" + strings.ToLower(hex.EncodeToString(spkiHash[:])) -} - -// allowSHA256 validates a "sha256" format hash and adds a canonical version of it into the Set -func (s *Set) allowSHA256(hash string) error { - // validate that the hash is the right length to be a full SHA-256 hash - hashLength := hex.DecodedLen(len(hash)) - if hashLength != sha256.Size { - return errors.Errorf("expected a %d byte SHA-256 hash, found %d bytes", sha256.Size, hashLength) - } - - // validate that the hash is valid hex - _, err := hex.DecodeString(hash) - if err != nil { - return err - } - - // in the end, just store the original hex string in memory (in lowercase) - s.sha256Hashes[strings.ToLower(hash)] = true - return nil -} - -// checkSHA256 returns true if the certificate's "sha256" hash is pinned in the Set -func (s *Set) checkSHA256(certificate *x509.Certificate) bool { - actualHash := sha256.Sum256(certificate.RawSubjectPublicKeyInfo) - actualHashHex := strings.ToLower(hex.EncodeToString(actualHash[:])) - return s.sha256Hashes[actualHashHex] -} diff --git a/cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go b/cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go deleted file mode 100644 index 8ca2c6cf77640..0000000000000 --- a/cmd/kubeadm/app/util/pubkeypin/pubkeypin_test.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pubkeypin - -import ( - "crypto/x509" - "encoding/pem" - "strings" - "testing" -) - -// testCertPEM is a simple self-signed test certificate issued with the openssl CLI: -// openssl req -new -newkey rsa:2048 -days 36500 -nodes -x509 -keyout /dev/null -out test.crt -const testCertPEM = ` ------BEGIN CERTIFICATE----- -MIIDRDCCAiygAwIBAgIJAJgVaCXvC6HkMA0GCSqGSIb3DQEBBQUAMB8xHTAbBgNV -BAMTFGt1YmVhZG0ta2V5cGlucy10ZXN0MCAXDTE3MDcwNTE3NDMxMFoYDzIxMTcw -NjExMTc0MzEwWjAfMR0wGwYDVQQDExRrdWJlYWRtLWtleXBpbnMtdGVzdDCCASIw -DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0ba8mHU9UtYlzM1Own2Fk/XGjR -J4uJQvSeGLtz1hID1IA0dLwruvgLCPadXEOw/f/IWIWcmT+ZmvIHZKa/woq2iHi5 -+HLhXs7aG4tjKGLYhag1hLjBI7icqV7ovkjdGAt9pWkxEzhIYClFMXDjKpMSynu+ -YX6nZ9tic1cOkHmx2yiZdMkuriRQnpTOa7bb03OC1VfGl7gHlOAIYaj4539WCOr8 -+ACTUMJUFEHcRZ2o8a/v6F9GMK+7SC8SJUI+GuroXqlMAdhEv4lX5Co52enYaClN -+D9FJLRpBv2YfiCQdJRaiTvCBSxEFz6BN+PtP5l2Hs703ZWEkOqCByM6HV8CAwEA -AaOBgDB+MB0GA1UdDgQWBBRQgUX8MhK2rWBWQiPHWcKzoWDH5DBPBgNVHSMESDBG -gBRQgUX8MhK2rWBWQiPHWcKzoWDH5KEjpCEwHzEdMBsGA1UEAxMUa3ViZWFkbS1r -ZXlwaW5zLXRlc3SCCQCYFWgl7wuh5DAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEB -BQUAA4IBAQCaAUif7Pfx3X0F08cxhx8/Hdx4jcJw6MCq6iq6rsXM32ge43t8OHKC -pJW08dk58a3O1YQSMMvD6GJDAiAfXzfwcwY6j258b1ZlI9Ag0VokvhMl/XfdCsdh -AWImnL1t4hvU5jLaImUUMlYxMcSfHBGAm7WJIZ2LdEfg6YWfZh+WGbg1W7uxLxk6 -y4h5rWdNnzBHWAGf7zJ0oEDV6W6RSwNXtC0JNnLaeIUm/6xdSddJlQPwUv8YH4jX -c1vuFqTnJBPcb7W//R/GI2Paicm1cmns9NLnPR35exHxFTy+D1yxmGokpoPMdife -aH+sfuxT8xeTPb3kjzF9eJTlnEquUDLM ------END CERTIFICATE-----` - -// expectedHash can be verified using the openssl CLI. -const expectedHash = `sha256:345959acb2c3b2feb87d281961c893f62a314207ef02599f1cc4a5fb255480b3` - -// testCert2PEM is a second test cert generated the same way as testCertPEM -const testCert2PEM = ` ------BEGIN CERTIFICATE----- -MIID9jCCAt6gAwIBAgIJAN5MXZDic7qYMA0GCSqGSIb3DQEBBQUAMFkxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMTCXRlc3RDZXJ0MjAgFw0xNzA3MjQxNjA0 -MDFaGA8yMTE3MDYzMDE2MDQwMVowWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNv -bWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAG -A1UEAxMJdGVzdENlcnQyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA -0brwpJYN2ytPWzRBtZSVc3dhkQlA59AzxzqeLLkano0Pxo9NIc3T/y58nnRI8uaS -I1P7BzUfJTiUEvmAtX8NggqKK4ld/gPrU+IRww1CUYS4KCkA/0d0ctPy0JwBCjD+ -b57G3rmNE8c+0jns6J96ZzNtqmv6N+ZlFBAXm1p4S+k0kGi5+hoQ6H7SYXjk2lG+ -r/8jPQEjy/NSdw1dcCA0Nc6o+hPr32927dS6J9KOhBeXNYUNdbuDDmroM9/gN2e/ -YMSA1olLeDPQ7Xvhk0PIyEDnHh83AffPCx5yM3htVRGddjIsPAVUJEL3z5leJtxe -fzyPghOhHJY0PXqznDQTcwIDAQABo4G+MIG7MB0GA1UdDgQWBBRP0IJqv/5rQ4Uf -SByl77dJeEapRDCBiwYDVR0jBIGDMIGAgBRP0IJqv/5rQ4UfSByl77dJeEapRKFd -pFswWTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoT -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAxMJdGVzdENlcnQyggkA -3kxdkOJzupgwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEA0RIMHc10 -wHHPMh9UflqBgDMF7gfbOL0juJfGloAOcohWWfMZBBJ0CQKMy3xRyoK3HmbW1eeb -iATjesw7t4VEAwf7mgKAd+eTfWYB952uq5qYJ2TI28mSofEq1Wz3RmrNkC1KCBs1 -u+YMFGwyl6necV9zKCeiju4jeovI1GA38TvH7MgYln6vMJ+FbgOXj7XCpek7dQiY -KGaeSSH218mGNQaWRQw2Sm3W6cFdANoCJUph4w18s7gjtFpfV63s80hXRps+vEyv -jEQMEQpG8Ss7HGJLGLBw/xAmG0e//XS/o2dDonbGbvzToFByz8OGxjMhk6yV6hdd -+iyvsLAw/MYMSA== ------END CERTIFICATE----- -` - -// testCert is a small helper to get a test x509.Certificate from the PEM constants -func testCert(t *testing.T, pemString string) *x509.Certificate { - // Decode the example certificate from a PEM file into a PEM block - pemBlock, _ := pem.Decode([]byte(pemString)) - if pemBlock == nil { - t.Fatal("failed to parse test certificate PEM") - return nil - } - - // Parse the PEM block into an x509.Certificate - result, err := x509.ParseCertificate(pemBlock.Bytes) - if err != nil { - t.Fatalf("failed to parse test certificate: %v", err) - return nil - } - return result -} - -func TestSet(t *testing.T) { - s := NewSet() - if !s.Empty() { - t.Error("expected a new set to be empty") - return - } - err := s.Allow("xyz") - if err == nil || !s.Empty() { - t.Error("expected allowing junk to fail") - return - } - - err = s.Allow("0011223344") - if err == nil || !s.Empty() { - t.Error("expected allowing something too short to fail") - return - } - - err = s.Allow(expectedHash + expectedHash) - if err == nil || !s.Empty() { - t.Error("expected allowing something too long to fail") - return - } - - err = s.CheckAny([]*x509.Certificate{testCert(t, testCertPEM)}) - if err == nil { - t.Error("expected test cert to not be allowed (yet)") - return - } - - err = s.Allow(strings.ToUpper(expectedHash)) - if err != nil || s.Empty() { - t.Error("expected allowing uppercase expectedHash to succeed") - return - } - - err = s.CheckAny([]*x509.Certificate{testCert(t, testCertPEM)}) - if err != nil { - t.Errorf("expected test cert to be allowed, but got back: %v", err) - return - } - - err = s.CheckAny([]*x509.Certificate{testCert(t, testCert2PEM)}) - if err == nil { - t.Error("expected the second test cert to be disallowed") - return - } -} - -func TestHash(t *testing.T) { - actualHash := Hash(testCert(t, testCertPEM)) - if actualHash != expectedHash { - t.Errorf( - "failed to Hash() to the expected value\n\texpected: %q\n\t actual: %q", - expectedHash, - actualHash, - ) - } -} diff --git a/cmd/kubeadm/app/util/runtime/BUILD b/cmd/kubeadm/app/util/runtime/BUILD deleted file mode 100644 index e320f461cf034..0000000000000 --- a/cmd/kubeadm/app/util/runtime/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "runtime.go", - "runtime_unix.go", - "runtime_windows.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:windows": [ - "//vendor/github.com/Microsoft/go-winio:go_default_library", - ], - "//conditions:default": [], - }), -) - -go_test( - name = "go_default_test", - srcs = ["runtime_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/app/util/runtime/runtime.go b/cmd/kubeadm/app/util/runtime/runtime.go deleted file mode 100644 index 56e7d4abad204..0000000000000 --- a/cmd/kubeadm/app/util/runtime/runtime.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "path/filepath" - goruntime "runtime" - "strings" - - "github.com/pkg/errors" - - errorsutil "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - utilsexec "k8s.io/utils/exec" -) - -// ContainerRuntime is an interface for working with container runtimes -type ContainerRuntime interface { - IsDocker() bool - IsRunning() error - ListKubeContainers() ([]string, error) - RemoveContainers(containers []string) error - PullImage(image string) error - ImageExists(image string) (bool, error) -} - -// CRIRuntime is a struct that interfaces with the CRI -type CRIRuntime struct { - exec utilsexec.Interface - criSocket string -} - -// DockerRuntime is a struct that interfaces with the Docker daemon -type DockerRuntime struct { - exec utilsexec.Interface -} - -// NewContainerRuntime sets up and returns a ContainerRuntime struct -func NewContainerRuntime(execer utilsexec.Interface, criSocket string) (ContainerRuntime, error) { - var toolName string - var runtime ContainerRuntime - - if criSocket != constants.DefaultDockerCRISocket { - toolName = "crictl" - // !!! temporary work around crictl warning: - // Using "/var/run/crio/crio.sock" as endpoint is deprecated, - // please consider using full url format "unix:///var/run/crio/crio.sock" - if filepath.IsAbs(criSocket) && goruntime.GOOS != "windows" { - criSocket = "unix://" + criSocket - } - runtime = &CRIRuntime{execer, criSocket} - } else { - toolName = "docker" - runtime = &DockerRuntime{execer} - } - - if _, err := execer.LookPath(toolName); err != nil { - return nil, errors.Wrapf(err, "%s is required for container runtime", toolName) - } - - return runtime, nil -} - -// IsDocker returns true if the runtime is docker -func (runtime *CRIRuntime) IsDocker() bool { - return false -} - -// IsDocker returns true if the runtime is docker -func (runtime *DockerRuntime) IsDocker() bool { - return true -} - -// IsRunning checks if runtime is running -func (runtime *CRIRuntime) IsRunning() error { - if out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "info").CombinedOutput(); err != nil { - return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out)) - } - return nil -} - -// IsRunning checks if runtime is running -func (runtime *DockerRuntime) IsRunning() error { - if out, err := runtime.exec.Command("docker", "info").CombinedOutput(); err != nil { - return errors.Wrapf(err, "container runtime is not running: output: %s, error", string(out)) - } - return nil -} - -// ListKubeContainers lists running k8s CRI pods -func (runtime *CRIRuntime) ListKubeContainers() ([]string, error) { - out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "pods", "-q").CombinedOutput() - if err != nil { - return nil, errors.Wrapf(err, "output: %s, error", string(out)) - } - pods := []string{} - pods = append(pods, strings.Fields(string(out))...) - return pods, nil -} - -// ListKubeContainers lists running k8s containers -func (runtime *DockerRuntime) ListKubeContainers() ([]string, error) { - output, err := runtime.exec.Command("docker", "ps", "-a", "--filter", "name=k8s_", "-q").CombinedOutput() - return strings.Fields(string(output)), err -} - -// RemoveContainers removes running k8s pods -func (runtime *CRIRuntime) RemoveContainers(containers []string) error { - errs := []error{} - for _, container := range containers { - out, err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "stopp", container).CombinedOutput() - if err != nil { - // don't stop on errors, try to remove as many containers as possible - errs = append(errs, errors.Wrapf(err, "failed to stop running pod %s: output: %s, error", container, string(out))) - } else { - out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "rmp", container).CombinedOutput() - if err != nil { - errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out))) - } - } - } - return errorsutil.NewAggregate(errs) -} - -// RemoveContainers removes running containers -func (runtime *DockerRuntime) RemoveContainers(containers []string) error { - errs := []error{} - for _, container := range containers { - out, err := runtime.exec.Command("docker", "stop", container).CombinedOutput() - if err != nil { - // don't stop on errors, try to remove as many containers as possible - errs = append(errs, errors.Wrapf(err, "failed to stop running container %s: output: %s, error", container, string(out))) - } else { - out, err = runtime.exec.Command("docker", "rm", "--volumes", container).CombinedOutput() - if err != nil { - errs = append(errs, errors.Wrapf(err, "failed to remove running container %s: output: %s, error", container, string(out))) - } - } - } - return errorsutil.NewAggregate(errs) -} - -// PullImage pulls the image -func (runtime *CRIRuntime) PullImage(image string) error { - var err error - var out []byte - for i := 0; i < constants.PullImageRetry; i++ { - out, err = runtime.exec.Command("crictl", "-r", runtime.criSocket, "pull", image).CombinedOutput() - if err == nil { - return nil - } - } - return errors.Wrapf(err, "output: %s, error", out) -} - -// PullImage pulls the image -func (runtime *DockerRuntime) PullImage(image string) error { - var err error - var out []byte - for i := 0; i < constants.PullImageRetry; i++ { - out, err = runtime.exec.Command("docker", "pull", image).CombinedOutput() - if err == nil { - return nil - } - } - return errors.Wrapf(err, "output: %s, error", out) -} - -// ImageExists checks to see if the image exists on the system -func (runtime *CRIRuntime) ImageExists(image string) (bool, error) { - err := runtime.exec.Command("crictl", "-r", runtime.criSocket, "inspecti", image).Run() - return err == nil, nil -} - -// ImageExists checks to see if the image exists on the system -func (runtime *DockerRuntime) ImageExists(image string) (bool, error) { - err := runtime.exec.Command("docker", "inspect", image).Run() - return err == nil, nil -} - -// detectCRISocketImpl is separated out only for test purposes, DON'T call it directly, use DetectCRISocket instead -func detectCRISocketImpl(isSocket func(string) bool) (string, error) { - foundCRISockets := []string{} - knownCRISockets := []string{ - // Docker and containerd sockets are special cased below, hence not to be included here - "/var/run/crio/crio.sock", - } - - if isSocket(dockerSocket) { - // the path in dockerSocket is not CRI compatible, hence we should replace it with a CRI compatible socket - foundCRISockets = append(foundCRISockets, constants.DefaultDockerCRISocket) - } else if isSocket(containerdSocket) { - // Docker 18.09 gets bundled together with containerd, thus having both dockerSocket and containerdSocket present. - // For compatibility reasons, we use the containerd socket only if Docker is not detected. - foundCRISockets = append(foundCRISockets, containerdSocket) - } - - for _, socket := range knownCRISockets { - if isSocket(socket) { - foundCRISockets = append(foundCRISockets, socket) - } - } - - switch len(foundCRISockets) { - case 0: - // Fall back to Docker if no CRI is detected, we can error out later on if we need it - return constants.DefaultDockerCRISocket, nil - case 1: - // Precisely one CRI found, use that - return foundCRISockets[0], nil - default: - // Multiple CRIs installed? - return "", errors.Errorf("Found multiple CRI sockets, please use --cri-socket to select one: %s", strings.Join(foundCRISockets, ", ")) - } -} - -// DetectCRISocket uses a list of known CRI sockets to detect one. If more than one or none is discovered, an error is returned. -func DetectCRISocket() (string, error) { - return detectCRISocketImpl(isExistingSocket) -} diff --git a/cmd/kubeadm/app/util/runtime/runtime_test.go b/cmd/kubeadm/app/util/runtime/runtime_test.go deleted file mode 100644 index df92e6917051f..0000000000000 --- a/cmd/kubeadm/app/util/runtime/runtime_test.go +++ /dev/null @@ -1,462 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "io/ioutil" - "net" - "os" - "reflect" - "runtime" - "testing" - - "github.com/pkg/errors" - - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -func TestNewContainerRuntime(t *testing.T) { - execLookPathOK := fakeexec.FakeExec{ - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - execLookPathErr := fakeexec.FakeExec{ - LookPathFunc: func(cmd string) (string, error) { return "", errors.Errorf("%s not found", cmd) }, - } - cases := []struct { - name string - execer fakeexec.FakeExec - criSocket string - isDocker bool - isError bool - }{ - {"valid: default cri socket", execLookPathOK, constants.DefaultDockerCRISocket, true, false}, - {"valid: cri-o socket url", execLookPathOK, "unix:///var/run/crio/crio.sock", false, false}, - {"valid: cri-o socket path", execLookPathOK, "/var/run/crio/crio.sock", false, false}, - {"invalid: no crictl", execLookPathErr, "unix:///var/run/crio/crio.sock", false, true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&tc.execer, tc.criSocket) - if err != nil { - if !tc.isError { - t.Fatalf("unexpected NewContainerRuntime error. criSocket: %s, error: %v", tc.criSocket, err) - } - return // expected error occurs, impossible to test runtime further - } - if tc.isError && err == nil { - t.Fatalf("unexpected NewContainerRuntime success. criSocket: %s", tc.criSocket) - } - isDocker := runtime.IsDocker() - if tc.isDocker != isDocker { - t.Fatalf("unexpected isDocker() result %v for the criSocket %s", isDocker, tc.criSocket) - } - }) - } -} - -func genFakeActions(fcmd *fakeexec.FakeCmd, num int) []fakeexec.FakeCommandAction { - var actions []fakeexec.FakeCommandAction - for i := 0; i < num; i++ { - actions = append(actions, func(cmd string, args ...string) exec.Cmd { - return fakeexec.InitFakeCmd(fcmd, cmd, args...) - }) - } - return actions -} - -func TestIsRunning(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - - criExecer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - dockerExecer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil }, - } - - cases := []struct { - name string - criSocket string - execer fakeexec.FakeExec - isError bool - }{ - {"valid: CRI-O is running", "unix:///var/run/crio/crio.sock", criExecer, false}, - {"invalid: CRI-O is not running", "unix:///var/run/crio/crio.sock", criExecer, true}, - {"valid: docker is running", constants.DefaultDockerCRISocket, dockerExecer, false}, - {"invalid: docker is not running", constants.DefaultDockerCRISocket, dockerExecer, true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&tc.execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) - } - isRunning := runtime.IsRunning() - if tc.isError && isRunning == nil { - t.Error("unexpected IsRunning() success") - } - if !tc.isError && isRunning != nil { - t.Error("unexpected IsRunning() error") - } - }) - } -} - -func TestListKubeContainers(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return []byte("k8s_p1\nk8s_p2"), nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("k8s_p1\nk8s_p2"), nil, nil }, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - isError bool - }{ - {"valid: list containers using CRI socket url", "unix:///var/run/crio/crio.sock", false}, - {"invalid: list containers using CRI socket url", "unix:///var/run/crio/crio.sock", true}, - {"valid: list containers using docker", constants.DefaultDockerCRISocket, false}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v", err) - } - - containers, err := runtime.ListKubeContainers() - if tc.isError { - if err == nil { - t.Errorf("unexpected ListKubeContainers success") - } - return - } else if err != nil { - t.Errorf("unexpected ListKubeContainers error: %v", err) - } - - if !reflect.DeepEqual(containers, []string{"k8s_p1", "k8s_p2"}) { - t.Errorf("unexpected ListKubeContainers output: %v", containers) - } - }) - } -} - -func TestRemoveContainers(t *testing.T) { - fakeOK := func() ([]byte, []byte, error) { return nil, nil, nil } - fakeErr := func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} } - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, // Test case 1 - fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK, - fakeErr, fakeOK, fakeOK, fakeErr, fakeOK, - fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, fakeOK, - fakeOK, fakeOK, fakeOK, fakeErr, fakeOK, fakeOK, - fakeErr, fakeOK, fakeOK, fakeErr, fakeOK, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - containers []string - isError bool - }{ - {"valid: remove containers using CRI", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, false}, // Test case 1 - {"invalid: CRI rmp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, - {"invalid: CRI stopp failure", "unix:///var/run/crio/crio.sock", []string{"k8s_p1", "k8s_p2", "k8s_p3"}, true}, - {"valid: remove containers using docker", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, false}, - {"invalid: docker rm failure", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true}, - {"invalid: docker stop failure", constants.DefaultDockerCRISocket, []string{"k8s_c1", "k8s_c2", "k8s_c3"}, true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) - } - - err = runtime.RemoveContainers(tc.containers) - if !tc.isError && err != nil { - t.Errorf("unexpected RemoveContainers errors: %v, criSocket: %s, containers: %v", err, tc.criSocket, tc.containers) - } - if tc.isError && err == nil { - t.Errorf("unexpected RemoveContainers success, criSocket: %s, containers: %v", tc.criSocket, tc.containers) - } - }) - } -} - -func TestPullImage(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - CombinedOutputScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - // If the pull fails, it will be retried 5 times (see PullImageRetry in constants/constants.go) - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return []byte("error"), nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.CombinedOutputScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - image string - isError bool - }{ - {"valid: pull image using CRI", "unix:///var/run/crio/crio.sock", "image1", false}, - {"invalid: CRI pull error", "unix:///var/run/crio/crio.sock", "image2", true}, - {"valid: pull image using docker", constants.DefaultDockerCRISocket, "image1", false}, - {"invalid: docker pull error", constants.DefaultDockerCRISocket, "image2", true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) - } - - err = runtime.PullImage(tc.image) - if !tc.isError && err != nil { - t.Errorf("unexpected PullImage error: %v, criSocket: %s, image: %s", err, tc.criSocket, tc.image) - } - if tc.isError && err == nil { - t.Errorf("unexpected PullImage success, criSocket: %s, image: %s", tc.criSocket, tc.image) - } - }) - } -} - -func TestImageExists(t *testing.T) { - fcmd := fakeexec.FakeCmd{ - RunScript: []fakeexec.FakeAction{ - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - func() ([]byte, []byte, error) { return nil, nil, nil }, - func() ([]byte, []byte, error) { return nil, nil, &fakeexec.FakeExitError{Status: 1} }, - }, - } - execer := fakeexec.FakeExec{ - CommandScript: genFakeActions(&fcmd, len(fcmd.RunScript)), - LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/crictl", nil }, - } - - cases := []struct { - name string - criSocket string - image string - result bool - }{ - {"valid: test if image exists using CRI", "unix:///var/run/crio/crio.sock", "image1", false}, - {"invalid: CRI inspecti failure", "unix:///var/run/crio/crio.sock", "image2", true}, - {"valid: test if image exists using docker", constants.DefaultDockerCRISocket, "image1", false}, - {"invalid: docker inspect failure", constants.DefaultDockerCRISocket, "image2", true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runtime, err := NewContainerRuntime(&execer, tc.criSocket) - if err != nil { - t.Fatalf("unexpected NewContainerRuntime error: %v, criSocket: %s", err, tc.criSocket) - } - - result, err := runtime.ImageExists(tc.image) - if !tc.result != result { - t.Errorf("unexpected ImageExists result: %t, criSocket: %s, image: %s, expected result: %t", err, tc.criSocket, tc.image, tc.result) - } - }) - } -} - -func TestIsExistingSocket(t *testing.T) { - // this test is not expected to work on Windows - if runtime.GOOS == "windows" { - return - } - - const tempPrefix = "test.kubeadm.runtime.isExistingSocket." - tests := []struct { - name string - proc func(*testing.T) - }{ - { - name: "Valid domain socket is detected as such", - proc: func(t *testing.T) { - tmpFile, err := ioutil.TempFile("", tempPrefix) - if err != nil { - t.Fatalf("unexpected error by TempFile: %v", err) - } - theSocket := tmpFile.Name() - os.Remove(theSocket) - tmpFile.Close() - - con, err := net.Listen("unix", theSocket) - if err != nil { - t.Fatalf("unexpected error while dialing a socket: %v", err) - } - defer con.Close() - - if !isExistingSocket(theSocket) { - t.Fatalf("isExistingSocket(%q) gave unexpected result. Should have been true, instead of false", theSocket) - } - }, - }, - { - name: "Regular file is not a domain socket", - proc: func(t *testing.T) { - tmpFile, err := ioutil.TempFile("", tempPrefix) - if err != nil { - t.Fatalf("unexpected error by TempFile: %v", err) - } - theSocket := tmpFile.Name() - defer os.Remove(theSocket) - tmpFile.Close() - - if isExistingSocket(theSocket) { - t.Fatalf("isExistingSocket(%q) gave unexpected result. Should have been false, instead of true", theSocket) - } - }, - }, - { - name: "Non existent socket is not a domain socket", - proc: func(t *testing.T) { - const theSocket = "/non/existent/socket" - if isExistingSocket(theSocket) { - t.Fatalf("isExistingSocket(%q) gave unexpected result. Should have been false, instead of true", theSocket) - } - }, - }, - } - - for _, test := range tests { - t.Run(test.name, test.proc) - } -} - -func TestDetectCRISocketImpl(t *testing.T) { - tests := []struct { - name string - existingSockets []string - expectedError bool - expectedSocket string - }{ - { - name: "No existing sockets, use Docker", - existingSockets: []string{}, - expectedError: false, - expectedSocket: constants.DefaultDockerCRISocket, - }, - { - name: "One valid CRI socket leads to success", - existingSockets: []string{"/var/run/crio/crio.sock"}, - expectedError: false, - expectedSocket: "/var/run/crio/crio.sock", - }, - { - name: "Correct Docker CRI socket is returned", - existingSockets: []string{"/var/run/docker.sock"}, - expectedError: false, - expectedSocket: constants.DefaultDockerCRISocket, - }, - { - name: "CRI and Docker sockets lead to an error", - existingSockets: []string{ - "/var/run/docker.sock", - "/var/run/crio/crio.sock", - }, - expectedError: true, - }, - { - name: "Docker and containerd lead to Docker being used", - existingSockets: []string{ - "/var/run/docker.sock", - "/run/containerd/containerd.sock", - }, - expectedError: false, - expectedSocket: constants.DefaultDockerCRISocket, - }, - { - name: "A couple of CRI sockets lead to an error", - existingSockets: []string{ - "/var/run/crio/crio.sock", - "/run/containerd/containerd.sock", - }, - expectedError: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - socket, err := detectCRISocketImpl(func(path string) bool { - for _, existing := range test.existingSockets { - if path == existing { - return true - } - } - - return false - }) - if (err != nil) != test.expectedError { - t.Fatalf("detectCRISocketImpl returned unexpected result\n\tExpected error: %t\n\tGot error: %t", test.expectedError, err != nil) - } - if !test.expectedError && socket != test.expectedSocket { - t.Fatalf("detectCRISocketImpl returned unexpected CRI socket\n\tExpected socket: %s\n\tReturned socket: %s", - test.expectedSocket, socket) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/runtime/runtime_unix.go b/cmd/kubeadm/app/util/runtime/runtime_unix.go deleted file mode 100644 index b15c303731373..0000000000000 --- a/cmd/kubeadm/app/util/runtime/runtime_unix.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build !windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "os" -) - -const ( - dockerSocket = "/var/run/docker.sock" // The Docker socket is not CRI compatible - containerdSocket = "/run/containerd/containerd.sock" -) - -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - - return fileInfo.Mode()&os.ModeSocket != 0 -} diff --git a/cmd/kubeadm/app/util/runtime/runtime_windows.go b/cmd/kubeadm/app/util/runtime/runtime_windows.go deleted file mode 100644 index 0c6a7b496dcaa..0000000000000 --- a/cmd/kubeadm/app/util/runtime/runtime_windows.go +++ /dev/null @@ -1,38 +0,0 @@ -// +build windows - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - winio "github.com/Microsoft/go-winio" -) - -const ( - dockerSocket = "//./pipe/docker_engine" // The Docker socket is not CRI compatible - containerdSocket = "//./pipe/containerd-containerd" // Proposed containerd named pipe for Windows -) - -// isExistingSocket checks if path exists and is domain socket -func isExistingSocket(path string) bool { - _, err := winio.DialPipe(path, nil) - if err != nil { - return false - } - - return true -} diff --git a/cmd/kubeadm/app/util/staticpod/BUILD b/cmd/kubeadm/app/util/staticpod/BUILD deleted file mode 100644 index 623a5aebd2764..0000000000000 --- a/cmd/kubeadm/app/util/staticpod/BUILD +++ /dev/null @@ -1,50 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["utils_test.go"], - embed = [":go_default_library"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - ], -) - -go_library( - name = "go_default_library", - srcs = ["utils.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util:go_default_library", - "//cmd/kubeadm/app/util/patches:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go deleted file mode 100644 index 8c3481d8d2740..0000000000000 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ /dev/null @@ -1,376 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package staticpod - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "math" - "net/url" - "os" - "sort" - "strings" - - "github.com/pkg/errors" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/patches" -) - -const ( - // kubeControllerManagerBindAddressArg represents the bind-address argument of the kube-controller-manager configuration. - kubeControllerManagerBindAddressArg = "bind-address" - - // kubeSchedulerBindAddressArg represents the bind-address argument of the kube-scheduler configuration. - kubeSchedulerBindAddressArg = "bind-address" -) - -// ComponentPod returns a Pod object from the container, volume and annotations specifications -func ComponentPod(container v1.Container, volumes map[string]v1.Volume, annotations map[string]string) v1.Pod { - return v1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: container.Name, - Namespace: metav1.NamespaceSystem, - // The component and tier labels are useful for quickly identifying the control plane Pods when doing a .List() - // against Pods in the kube-system namespace. Can for example be used together with the WaitForPodsWithLabel function - Labels: map[string]string{"component": container.Name, "tier": kubeadmconstants.ControlPlaneTier}, - Annotations: annotations, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{container}, - PriorityClassName: "system-node-critical", - HostNetwork: true, - Volumes: VolumeMapToSlice(volumes), - }, - } -} - -// ComponentResources returns the v1.ResourceRequirements object needed for allocating a specified amount of the CPU -func ComponentResources(cpu string) v1.ResourceRequirements { - return v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse(cpu), - }, - } -} - -// NewVolume creates a v1.Volume with a hostPath mount to the specified location -func NewVolume(name, path string, pathType *v1.HostPathType) v1.Volume { - return v1.Volume{ - Name: name, - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: path, - Type: pathType, - }, - }, - } -} - -// NewVolumeMount creates a v1.VolumeMount to the specified location -func NewVolumeMount(name, path string, readOnly bool) v1.VolumeMount { - return v1.VolumeMount{ - Name: name, - MountPath: path, - ReadOnly: readOnly, - } -} - -// VolumeMapToSlice returns a slice of volumes from a map's values -func VolumeMapToSlice(volumes map[string]v1.Volume) []v1.Volume { - v := make([]v1.Volume, 0, len(volumes)) - - for _, vol := range volumes { - v = append(v, vol) - } - - sort.Slice(v, func(i, j int) bool { - return strings.Compare(v[i].Name, v[j].Name) == -1 - }) - - return v -} - -// VolumeMountMapToSlice returns a slice of volumes from a map's values -func VolumeMountMapToSlice(volumeMounts map[string]v1.VolumeMount) []v1.VolumeMount { - v := make([]v1.VolumeMount, 0, len(volumeMounts)) - - for _, volMount := range volumeMounts { - v = append(v, volMount) - } - - sort.Slice(v, func(i, j int) bool { - return strings.Compare(v[i].Name, v[j].Name) == -1 - }) - - return v -} - -// GetExtraParameters builds a list of flag arguments two string-string maps, one with default, base commands and one with overrides -func GetExtraParameters(overrides map[string]string, defaults map[string]string) []string { - var command []string - for k, v := range overrides { - if len(v) > 0 { - command = append(command, fmt.Sprintf("--%s=%s", k, v)) - } - } - for k, v := range defaults { - if _, overrideExists := overrides[k]; !overrideExists { - command = append(command, fmt.Sprintf("--%s=%s", k, v)) - } - } - return command -} - -// PatchStaticPod applies patches stored in patchesDir to a static Pod. -func PatchStaticPod(pod *v1.Pod, patchesDir string, output io.Writer) (*v1.Pod, error) { - // Marshal the Pod manifest into YAML. - podYAML, err := kubeadmutil.MarshalToYaml(pod, v1.SchemeGroupVersion) - if err != nil { - return pod, errors.Wrapf(err, "failed to marshal Pod manifest to YAML") - } - - var knownTargets = []string{ - kubeadmconstants.Etcd, - kubeadmconstants.KubeAPIServer, - kubeadmconstants.KubeControllerManager, - kubeadmconstants.KubeScheduler, - } - - patchManager, err := patches.GetPatchManagerForPath(patchesDir, knownTargets, output) - if err != nil { - return pod, err - } - - patchTarget := &patches.PatchTarget{ - Name: pod.Name, - StrategicMergePatchObject: v1.Pod{}, - Data: podYAML, - } - if err := patchManager.ApplyPatchesToTarget(patchTarget); err != nil { - return pod, err - } - - obj, err := kubeadmutil.UnmarshalFromYaml(patchTarget.Data, v1.SchemeGroupVersion) - if err != nil { - return pod, errors.Wrap(err, "failed to unmarshal patched manifest from YAML") - } - - pod2, ok := obj.(*v1.Pod) - if !ok { - return pod, errors.Wrap(err, "patched manifest is not a valid Pod object") - } - - return pod2, nil -} - -// WriteStaticPodToDisk writes a static pod file to disk -func WriteStaticPodToDisk(componentName, manifestDir string, pod v1.Pod) error { - - // creates target folder if not already exists - if err := os.MkdirAll(manifestDir, 0700); err != nil { - return errors.Wrapf(err, "failed to create directory %q", manifestDir) - } - - // writes the pod to disk - serialized, err := kubeadmutil.MarshalToYaml(&pod, v1.SchemeGroupVersion) - if err != nil { - return errors.Wrapf(err, "failed to marshal manifest for %q to YAML", componentName) - } - - filename := kubeadmconstants.GetStaticPodFilepath(componentName, manifestDir) - - if err := ioutil.WriteFile(filename, serialized, 0600); err != nil { - return errors.Wrapf(err, "failed to write static pod manifest file for %q (%q)", componentName, filename) - } - - return nil -} - -// ReadStaticPodFromDisk reads a static pod file from disk -func ReadStaticPodFromDisk(manifestPath string) (*v1.Pod, error) { - buf, err := ioutil.ReadFile(manifestPath) - if err != nil { - return &v1.Pod{}, errors.Wrapf(err, "failed to read manifest for %q", manifestPath) - } - - obj, err := kubeadmutil.UnmarshalFromYaml(buf, v1.SchemeGroupVersion) - if err != nil { - return &v1.Pod{}, errors.Errorf("failed to unmarshal manifest for %q from YAML: %v", manifestPath, err) - } - - pod := obj.(*v1.Pod) - - return pod, nil -} - -// LivenessProbe creates a Probe object with a HTTPGet handler -func LivenessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { - // sets initialDelaySeconds same as periodSeconds to skip one period before running a check - return createHTTPProbe(host, path, port, scheme, 10, 15, 8, 10) -} - -// ReadinessProbe creates a Probe object with a HTTPGet handler -func ReadinessProbe(host, path string, port int, scheme v1.URIScheme) *v1.Probe { - // sets initialDelaySeconds as '0' because we don't want to delay user infrastructure checks - // looking for "ready" status on kubeadm static Pods - return createHTTPProbe(host, path, port, scheme, 0, 15, 3, 1) -} - -// StartupProbe creates a Probe object with a HTTPGet handler -func StartupProbe(host, path string, port int, scheme v1.URIScheme, timeoutForControlPlane *metav1.Duration) *v1.Probe { - periodSeconds, timeoutForControlPlaneSeconds := int32(10), constants.DefaultControlPlaneTimeout.Seconds() - if timeoutForControlPlane != nil { - timeoutForControlPlaneSeconds = timeoutForControlPlane.Seconds() - } - // sets failureThreshold big enough to guarantee the full timeout can cover the worst case scenario for the control-plane to come alive - // we ignore initialDelaySeconds in the calculation here for simplicity - failureThreshold := int32(math.Ceil(timeoutForControlPlaneSeconds / float64(periodSeconds))) - // sets initialDelaySeconds same as periodSeconds to skip one period before running a check - return createHTTPProbe(host, path, port, scheme, periodSeconds, 15, failureThreshold, periodSeconds) -} - -func createHTTPProbe(host, path string, port int, scheme v1.URIScheme, initialDelaySeconds, timeoutSeconds, failureThreshold, periodSeconds int32) *v1.Probe { - return &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Host: host, - Path: path, - Port: intstr.FromInt(port), - Scheme: scheme, - }, - }, - InitialDelaySeconds: initialDelaySeconds, - TimeoutSeconds: timeoutSeconds, - FailureThreshold: failureThreshold, - PeriodSeconds: periodSeconds, - } -} - -// GetAPIServerProbeAddress returns the probe address for the API server -func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string { - // In the case of a self-hosted deployment, the initial host on which kubeadm --init is run, - // will generate a DaemonSet with a nodeSelector such that all nodes with the label - // node-role.kubernetes.io/master='' will have the API server deployed to it. Since the init - // is run only once on an initial host, the API advertise address will be invalid for any - // future hosts that do not have the same address. Furthermore, since liveness and readiness - // probes do not support the Downward API we cannot dynamically set the advertise address to - // the node's IP. The only option then is to use localhost. - if endpoint != nil && endpoint.AdvertiseAddress != "" { - return getProbeAddress(endpoint.AdvertiseAddress) - } - - return "127.0.0.1" -} - -// GetControllerManagerProbeAddress returns the kubernetes controller manager probe address -func GetControllerManagerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { - if addr, exists := cfg.ControllerManager.ExtraArgs[kubeControllerManagerBindAddressArg]; exists { - return getProbeAddress(addr) - } - return "127.0.0.1" -} - -// GetSchedulerProbeAddress returns the kubernetes scheduler probe address -func GetSchedulerProbeAddress(cfg *kubeadmapi.ClusterConfiguration) string { - if addr, exists := cfg.Scheduler.ExtraArgs[kubeSchedulerBindAddressArg]; exists { - return getProbeAddress(addr) - } - return "127.0.0.1" -} - -// GetEtcdProbeEndpoint takes a kubeadm Etcd configuration object and attempts to parse -// the first URL in the listen-metrics-urls argument, returning an etcd probe hostname, -// port and scheme -func GetEtcdProbeEndpoint(cfg *kubeadmapi.Etcd, isIPv6 bool) (string, int, v1.URIScheme) { - localhost := "127.0.0.1" - if isIPv6 { - localhost = "::1" - } - if cfg.Local == nil || cfg.Local.ExtraArgs == nil { - return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP - } - if arg, exists := cfg.Local.ExtraArgs["listen-metrics-urls"]; exists { - // Use the first url in the listen-metrics-urls if multiple URL's are specified. - arg = strings.Split(arg, ",")[0] - parsedURL, err := url.Parse(arg) - if err != nil { - return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP - } - // Parse scheme - scheme := v1.URISchemeHTTP - if parsedURL.Scheme == "https" { - scheme = v1.URISchemeHTTPS - } - // Parse hostname - hostname := parsedURL.Hostname() - if len(hostname) == 0 { - hostname = localhost - } - // Parse port - port := kubeadmconstants.EtcdMetricsPort - portStr := parsedURL.Port() - if len(portStr) != 0 { - p, err := kubeadmutil.ParsePort(portStr) - if err == nil { - port = p - } - } - return hostname, port, scheme - } - return localhost, kubeadmconstants.EtcdMetricsPort, v1.URISchemeHTTP -} - -// ManifestFilesAreEqual compares 2 files. It returns true if their contents are equal, false otherwise -func ManifestFilesAreEqual(path1, path2 string) (bool, error) { - content1, err := ioutil.ReadFile(path1) - if err != nil { - return false, err - } - content2, err := ioutil.ReadFile(path2) - if err != nil { - return false, err - } - - return bytes.Equal(content1, content2), nil -} - -// getProbeAddress returns a valid probe address. -// Kubeadm uses the bind-address to configure the probe address. It's common to use the -// unspecified address "0.0.0.0" or "::" as bind-address when we want to listen in all interfaces, -// however this address can't be used as probe #86504. -// If the address is an unspecified address getProbeAddress returns empty, -// that means that kubelet will use the PodIP as probe address. -func getProbeAddress(addr string) string { - if addr == "0.0.0.0" || addr == "::" { - return "" - } - return addr -} diff --git a/cmd/kubeadm/app/util/staticpod/utils_test.go b/cmd/kubeadm/app/util/staticpod/utils_test.go deleted file mode 100644 index dfb0eafc85b0c..0000000000000 --- a/cmd/kubeadm/app/util/staticpod/utils_test.go +++ /dev/null @@ -1,836 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package staticpod - -import ( - "io/ioutil" - "os" - "path/filepath" - "reflect" - "sort" - "strconv" - "testing" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func TestComponentResources(t *testing.T) { - a := ComponentResources("250m") - if a.Requests == nil { - t.Errorf( - "failed componentResources, return value was nil", - ) - } -} - -func TestGetAPIServerProbeAddress(t *testing.T) { - tests := []struct { - desc string - endpoint *kubeadmapi.APIEndpoint - expected string - }{ - { - desc: "nil endpoint returns 127.0.0.1", - expected: "127.0.0.1", - }, - { - desc: "empty AdvertiseAddress endpoint returns 127.0.0.1", - endpoint: &kubeadmapi.APIEndpoint{}, - expected: "127.0.0.1", - }, - { - desc: "filled in AdvertiseAddress endpoint returns it", - endpoint: &kubeadmapi.APIEndpoint{ - AdvertiseAddress: "10.10.10.10", - }, - expected: "10.10.10.10", - }, - { - desc: "filled in ipv6 AdvertiseAddress endpoint returns it", - endpoint: &kubeadmapi.APIEndpoint{ - AdvertiseAddress: "2001:abcd:bcda::1", - }, - expected: "2001:abcd:bcda::1", - }, - { - desc: "filled in 0.0.0.0 AdvertiseAddress endpoint returns empty", - endpoint: &kubeadmapi.APIEndpoint{ - AdvertiseAddress: "0.0.0.0", - }, - expected: "", - }, - { - desc: "filled in :: AdvertiseAddress endpoint returns empty", - endpoint: &kubeadmapi.APIEndpoint{ - AdvertiseAddress: "::", - }, - expected: "", - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - actual := GetAPIServerProbeAddress(test.endpoint) - if actual != test.expected { - t.Errorf("Unexpected result from GetAPIServerProbeAddress:\n\texpected: %s\n\tactual: %s", test.expected, actual) - } - }) - } -} - -func TestGetControllerManagerProbeAddress(t *testing.T) { - tests := []struct { - desc string - cfg *kubeadmapi.ClusterConfiguration - expected string - }{ - { - desc: "no controller manager extra args leads to 127.0.0.1 being used", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{}, - }, - }, - expected: "127.0.0.1", - }, - { - desc: "setting controller manager extra address arg to something acknowledges it", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "10.10.10.10", - }, - }, - }, - expected: "10.10.10.10", - }, - { - desc: "setting controller manager extra ipv6 address arg to something acknowledges it", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "2001:abcd:bcda::1", - }, - }, - }, - expected: "2001:abcd:bcda::1", - }, - { - desc: "setting controller manager extra address arg to 0.0.0.0 returns empty", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "0.0.0.0", - }, - }, - }, - expected: "", - }, - { - desc: "setting controller manager extra ipv6 address arg to :: returns empty", - cfg: &kubeadmapi.ClusterConfiguration{ - ControllerManager: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeControllerManagerBindAddressArg: "::", - }, - }, - }, - expected: "", - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - actual := GetControllerManagerProbeAddress(test.cfg) - if actual != test.expected { - t.Errorf("Unexpected result from GetControllerManagerProbeAddress:\n\texpected: %s\n\tactual: %s", test.expected, actual) - } - }) - } -} - -func TestGetSchedulerProbeAddress(t *testing.T) { - tests := []struct { - desc string - cfg *kubeadmapi.ClusterConfiguration - expected string - }{ - { - desc: "no scheduler extra args leads to 127.0.0.1 being used", - cfg: &kubeadmapi.ClusterConfiguration{ - Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{}, - }, - }, - expected: "127.0.0.1", - }, - { - desc: "setting scheduler extra address arg to something acknowledges it", - cfg: &kubeadmapi.ClusterConfiguration{ - Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "10.10.10.10", - }, - }, - }, - expected: "10.10.10.10", - }, - { - desc: "setting scheduler extra ipv6 address arg to something acknowledges it", - cfg: &kubeadmapi.ClusterConfiguration{ - Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "2001:abcd:bcda::1", - }, - }, - }, - expected: "2001:abcd:bcda::1", - }, - { - desc: "setting scheduler extra ipv6 address arg to 0.0.0.0 returns empty", - cfg: &kubeadmapi.ClusterConfiguration{ - Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "0.0.0.0", - }, - }, - }, - expected: "", - }, - { - desc: "setting scheduler extra ipv6 address arg to :: returns empty", - cfg: &kubeadmapi.ClusterConfiguration{ - Scheduler: kubeadmapi.ControlPlaneComponent{ - ExtraArgs: map[string]string{ - kubeSchedulerBindAddressArg: "::", - }, - }, - }, - expected: "", - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - actual := GetSchedulerProbeAddress(test.cfg) - if actual != test.expected { - t.Errorf("Unexpected result from GetSchedulerProbeAddress:\n\texpected: %s\n\tactual: %s", test.expected, actual) - } - }) - } -} -func TestGetEtcdProbeEndpoint(t *testing.T) { - var tests = []struct { - name string - cfg *kubeadmapi.Etcd - isIPv6 bool - expectedHostname string - expectedPort int - expectedScheme v1.URIScheme - }{ - { - name: "etcd probe URL from two URLs", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://1.2.3.4:1234,https://4.3.2.1:2381"}, - }, - }, - isIPv6: false, - expectedHostname: "1.2.3.4", - expectedPort: 1234, - expectedScheme: v1.URISchemeHTTPS, - }, - { - name: "etcd probe URL with HTTP scheme", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "http://1.2.3.4:1234"}, - }, - }, - isIPv6: false, - expectedHostname: "1.2.3.4", - expectedPort: 1234, - expectedScheme: v1.URISchemeHTTP, - }, - { - name: "etcd probe URL without scheme should result in defaults", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "1.2.3.4"}, - }, - }, - isIPv6: false, - expectedHostname: "127.0.0.1", - expectedPort: kubeadmconstants.EtcdMetricsPort, - expectedScheme: v1.URISchemeHTTP, - }, - { - name: "etcd probe URL without port", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://1.2.3.4"}, - }, - }, - isIPv6: false, - expectedHostname: "1.2.3.4", - expectedPort: kubeadmconstants.EtcdMetricsPort, - expectedScheme: v1.URISchemeHTTPS, - }, - { - name: "etcd probe URL from two IPv6 URLs", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://[2001:abcd:bcda::1]:1234,https://[2001:abcd:bcda::2]:2381"}, - }, - }, - isIPv6: true, - expectedHostname: "2001:abcd:bcda::1", - expectedPort: 1234, - expectedScheme: v1.URISchemeHTTPS, - }, - { - name: "etcd probe localhost IPv6 URL with HTTP scheme", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "http://[::1]:1234"}, - }, - }, - isIPv6: true, - expectedHostname: "::1", - expectedPort: 1234, - expectedScheme: v1.URISchemeHTTP, - }, - { - name: "etcd probe IPv6 URL with HTTP scheme", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "http://[2001:abcd:bcda::1]:1234"}, - }, - }, - isIPv6: true, - expectedHostname: "2001:abcd:bcda::1", - expectedPort: 1234, - expectedScheme: v1.URISchemeHTTP, - }, - { - name: "etcd probe IPv6 URL without port", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{ - ExtraArgs: map[string]string{ - "listen-metrics-urls": "https://[2001:abcd:bcda::1]"}, - }, - }, - isIPv6: true, - expectedHostname: "2001:abcd:bcda::1", - expectedPort: kubeadmconstants.EtcdMetricsPort, - expectedScheme: v1.URISchemeHTTPS, - }, - { - name: "etcd probe URL from defaults", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{}, - }, - isIPv6: false, - expectedHostname: "127.0.0.1", - expectedPort: kubeadmconstants.EtcdMetricsPort, - expectedScheme: v1.URISchemeHTTP, - }, - { - name: "etcd probe URL from defaults if IPv6", - cfg: &kubeadmapi.Etcd{ - Local: &kubeadmapi.LocalEtcd{}, - }, - isIPv6: true, - expectedHostname: "::1", - expectedPort: kubeadmconstants.EtcdMetricsPort, - expectedScheme: v1.URISchemeHTTP, - }, - } - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - hostname, port, scheme := GetEtcdProbeEndpoint(rt.cfg, rt.isIPv6) - if hostname != rt.expectedHostname { - t.Errorf("%q test case failed:\n\texpected hostname: %s\n\tgot: %s", - rt.name, rt.expectedHostname, hostname) - } - if port != rt.expectedPort { - t.Errorf("%q test case failed:\n\texpected port: %d\n\tgot: %d", - rt.name, rt.expectedPort, port) - } - if scheme != rt.expectedScheme { - t.Errorf("%q test case failed:\n\texpected scheme: %v\n\tgot: %v", - rt.name, rt.expectedScheme, scheme) - } - }) - } -} - -func TestComponentPod(t *testing.T) { - var tests = []struct { - name string - expected v1.Pod - }{ - { - name: "foo", - expected: v1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "kube-system", - Labels: map[string]string{"component": "foo", "tier": "control-plane"}, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "foo", - }, - }, - PriorityClassName: "system-node-critical", - HostNetwork: true, - Volumes: []v1.Volume{}, - }, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - c := v1.Container{Name: rt.name} - actual := ComponentPod(c, map[string]v1.Volume{}, nil) - if !reflect.DeepEqual(rt.expected, actual) { - t.Errorf( - "failed componentPod:\n\texpected: %v\n\t actual: %v", - rt.expected, - actual, - ) - } - }) - } -} - -func TestNewVolume(t *testing.T) { - hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - var tests = []struct { - name string - path string - expected v1.Volume - pathType *v1.HostPathType - }{ - { - name: "foo", - path: "/etc/foo", - expected: v1.Volume{ - Name: "foo", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: "/etc/foo", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - pathType: &hostPathDirectoryOrCreate, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := NewVolume(rt.name, rt.path, rt.pathType) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf( - "failed newVolume:\n\texpected: %v\n\t actual: %v", - rt.expected, - actual, - ) - } - }) - } -} - -func TestNewVolumeMount(t *testing.T) { - var tests = []struct { - name string - path string - ro bool - expected v1.VolumeMount - }{ - { - name: "foo", - path: "/etc/foo", - ro: false, - expected: v1.VolumeMount{ - Name: "foo", - MountPath: "/etc/foo", - ReadOnly: false, - }, - }, - { - name: "bar", - path: "/etc/foo/bar", - ro: true, - expected: v1.VolumeMount{ - Name: "bar", - MountPath: "/etc/foo/bar", - ReadOnly: true, - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := NewVolumeMount(rt.name, rt.path, rt.ro) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf( - "failed newVolumeMount:\n\texpected: %v\n\t actual: %v", - rt.expected, - actual, - ) - } - }) - } -} -func TestVolumeMapToSlice(t *testing.T) { - testVolumes := map[string]v1.Volume{ - "foo": { - Name: "foo", - }, - "bar": { - Name: "bar", - }, - } - volumeSlice := VolumeMapToSlice(testVolumes) - if len(volumeSlice) != 2 { - t.Errorf("Expected slice length of 1, got %d", len(volumeSlice)) - } - if volumeSlice[0].Name != "bar" { - t.Errorf("Expected first volume name \"bar\", got %s", volumeSlice[0].Name) - } - if volumeSlice[1].Name != "foo" { - t.Errorf("Expected second volume name \"foo\", got %s", volumeSlice[1].Name) - } -} - -func TestVolumeMountMapToSlice(t *testing.T) { - testVolumeMounts := map[string]v1.VolumeMount{ - "foo": { - Name: "foo", - }, - "bar": { - Name: "bar", - }, - } - volumeMountSlice := VolumeMountMapToSlice(testVolumeMounts) - if len(volumeMountSlice) != 2 { - t.Errorf("Expected slice length of 1, got %d", len(volumeMountSlice)) - } - if volumeMountSlice[0].Name != "bar" { - t.Errorf("Expected first volume mount name \"bar\", got %s", volumeMountSlice[0].Name) - } - if volumeMountSlice[1].Name != "foo" { - t.Errorf("Expected second volume name \"foo\", got %s", volumeMountSlice[1].Name) - } -} - -func TestGetExtraParameters(t *testing.T) { - var tests = []struct { - name string - overrides map[string]string - defaults map[string]string - expected []string - }{ - { - name: "with admission-control default NamespaceLifecycle", - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - defaults: map[string]string{ - "admission-control": "NamespaceLifecycle", - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - }, - { - name: "without admission-control default", - overrides: map[string]string{ - "admission-control": "NamespaceLifecycle,LimitRanger", - }, - defaults: map[string]string{ - "insecure-bind-address": "127.0.0.1", - "allow-privileged": "true", - }, - expected: []string{ - "--admission-control=NamespaceLifecycle,LimitRanger", - "--insecure-bind-address=127.0.0.1", - "--allow-privileged=true", - }, - }, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - actual := GetExtraParameters(rt.overrides, rt.defaults) - sort.Strings(actual) - sort.Strings(rt.expected) - if !reflect.DeepEqual(actual, rt.expected) { - t.Errorf("failed getExtraParameters:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual) - } - }) - } -} - -const ( - validPod = ` -apiVersion: v1 -kind: Pod -metadata: - labels: - component: etcd - tier: control-plane - name: etcd - namespace: kube-system -spec: - containers: - - image: gcr.io/google_containers/etcd-amd64:3.1.11 -status: {} -` - invalidPod = `---{ broken yaml @@@` -) - -func TestReadStaticPodFromDisk(t *testing.T) { - tests := []struct { - description string - podYaml string - expectErr bool - writeManifest bool - }{ - { - description: "valid pod is marshaled", - podYaml: validPod, - writeManifest: true, - expectErr: false, - }, - { - description: "invalid pod fails to unmarshal", - podYaml: invalidPod, - writeManifest: true, - expectErr: true, - }, - { - description: "non-existent file returns error", - podYaml: ``, - writeManifest: false, - expectErr: true, - }, - } - - for _, rt := range tests { - t.Run(rt.description, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - manifestPath := filepath.Join(tmpdir, "pod.yaml") - if rt.writeManifest { - err := ioutil.WriteFile(manifestPath, []byte(rt.podYaml), 0644) - if err != nil { - t.Fatalf("Failed to write pod manifest\n%s\n\tfatal error: %v", rt.description, err) - } - } - - _, actualErr := ReadStaticPodFromDisk(manifestPath) - if (actualErr != nil) != rt.expectErr { - t.Errorf( - "ReadStaticPodFromDisk failed\n%s\n\texpected error: %t\n\tgot: %t\n\tactual error: %v", - rt.description, - rt.expectErr, - (actualErr != nil), - actualErr, - ) - } - }) - } -} - -func TestManifestFilesAreEqual(t *testing.T) { - var tests = []struct { - description string - podYamls []string - expectedResult bool - expectErr bool - }{ - { - description: "manifests are equal", - podYamls: []string{validPod, validPod}, - expectedResult: true, - expectErr: false, - }, - { - description: "manifests are not equal", - podYamls: []string{validPod, validPod + "\n"}, - expectedResult: false, - expectErr: false, - }, - { - description: "first manifest doesn't exist", - podYamls: []string{validPod, ""}, - expectedResult: false, - expectErr: true, - }, - { - description: "second manifest doesn't exist", - podYamls: []string{"", validPod}, - expectedResult: false, - expectErr: true, - }, - } - - for _, rt := range tests { - t.Run(rt.description, func(t *testing.T) { - tmpdir := testutil.SetupTempDir(t) - defer os.RemoveAll(tmpdir) - - // write 2 manifests - for i := 0; i < 2; i++ { - if rt.podYamls[i] != "" { - manifestPath := filepath.Join(tmpdir, strconv.Itoa(i)+".yaml") - err := ioutil.WriteFile(manifestPath, []byte(rt.podYamls[i]), 0644) - if err != nil { - t.Fatalf("Failed to write manifest file\n%s\n\tfatal error: %v", rt.description, err) - } - } - } - - // compare them - result, actualErr := ManifestFilesAreEqual(filepath.Join(tmpdir, "0.yaml"), filepath.Join(tmpdir, "1.yaml")) - if result != rt.expectedResult { - t.Errorf( - "ManifestFilesAreEqual failed\n%s\nexpected result: %t\nactual result: %t", - rt.description, - rt.expectedResult, - result, - ) - } - if (actualErr != nil) != rt.expectErr { - t.Errorf( - "ManifestFilesAreEqual failed\n%s\n\texpected error: %t\n\tgot: %t\n\tactual error: %v", - rt.description, - rt.expectErr, - (actualErr != nil), - actualErr, - ) - } - }) - } -} - -func TestPatchStaticPod(t *testing.T) { - type file struct { - name string - data string - } - - tests := []struct { - name string - files []*file - pod *v1.Pod - expectedPod *v1.Pod - expectedError bool - }{ - { - name: "valid: patch a kube-apiserver target using a couple of ordered patches", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-apiserver", - Namespace: "foo", - }, - }, - expectedPod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-apiserver", - Namespace: "bar2", - }, - }, - files: []*file{ - { - name: "kube-apiserver1+merge.json", - data: `{"metadata":{"namespace":"bar2"}}`, - }, - { - name: "kube-apiserver0+json.json", - data: `[{"op": "replace", "path": "/metadata/namespace", "value": "bar1"}]`, - }, - }, - }, - { - name: "invalid: unknown patch target name", - pod: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "bar", - }, - }, - expectedError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tempDir, err := ioutil.TempDir("", "patch-files") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempDir) - - for _, file := range tc.files { - filePath := filepath.Join(tempDir, file.name) - err := ioutil.WriteFile(filePath, []byte(file.data), 0644) - if err != nil { - t.Fatalf("could not write temporary file %q", filePath) - } - } - - pod, err := PatchStaticPod(tc.pod, tempDir, ioutil.Discard) - if (err != nil) != tc.expectedError { - t.Fatalf("expected error: %v, got: %v, error: %v", tc.expectedError, (err != nil), err) - } - if err != nil { - return - } - - if tc.expectedPod.String() != pod.String() { - t.Fatalf("expected object:\n%s\ngot:\n%s", tc.expectedPod.String(), pod.String()) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/template.go b/cmd/kubeadm/app/util/template.go deleted file mode 100644 index 5951f8c8297a3..0000000000000 --- a/cmd/kubeadm/app/util/template.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "text/template" - - "github.com/pkg/errors" -) - -// ParseTemplate validates and parses passed as argument template -func ParseTemplate(strtmpl string, obj interface{}) ([]byte, error) { - var buf bytes.Buffer - tmpl, err := template.New("template").Parse(strtmpl) - if err != nil { - return nil, errors.Wrap(err, "error when parsing template") - } - err = tmpl.Execute(&buf, obj) - if err != nil { - return nil, errors.Wrap(err, "error when executing template") - } - return buf.Bytes(), nil -} diff --git a/cmd/kubeadm/app/util/template_test.go b/cmd/kubeadm/app/util/template_test.go deleted file mode 100644 index c2dd0ed24548a..0000000000000 --- a/cmd/kubeadm/app/util/template_test.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "testing" -) - -const ( - validTmpl = "image: {{ .ImageRepository }}/pause:3.2" - validTmplOut = "image: k8s.gcr.io/pause:3.2" - doNothing = "image: k8s.gcr.io/pause:3.2" - invalidTmpl1 = "{{ .baz }/d}" - invalidTmpl2 = "{{ !foobar }}" -) - -func TestParseTemplate(t *testing.T) { - var tmplTests = []struct { - name string - template string - data interface{} - output string - errExpected bool - }{ - { - name: "should parse a valid template and set the right values", - template: validTmpl, - data: struct{ ImageRepository, Arch string }{ - ImageRepository: "k8s.gcr.io", - Arch: "amd64", - }, - output: validTmplOut, - errExpected: false, - }, - { - name: "should noop if there aren't any {{ .foo }} present", - template: doNothing, - data: struct{ ImageRepository, Arch string }{ - ImageRepository: "k8s.gcr.io", - Arch: "amd64", - }, - output: doNothing, - errExpected: false, - }, - { - name: "invalid syntax, passing nil", - template: invalidTmpl1, - data: nil, - output: "", - errExpected: true, - }, - { - name: "invalid syntax", - template: invalidTmpl2, - data: struct{}{}, - output: "", - errExpected: true, - }, - } - for _, tt := range tmplTests { - t.Run(tt.name, func(t *testing.T) { - outbytes, err := ParseTemplate(tt.template, tt.data) - if tt.errExpected != (err != nil) { - t.Errorf( - "failed TestParseTemplate:\n\texpected err: %t\n\t actual: %s", - tt.errExpected, - err, - ) - } - if tt.output != string(outbytes) { - t.Errorf( - "failed TestParseTemplate:\n\texpected bytes: %s\n\t actual: %s", - tt.output, - outbytes, - ) - } - }) - } -} diff --git a/cmd/kubeadm/app/util/version.go b/cmd/kubeadm/app/util/version.go deleted file mode 100644 index 9ac0cc7d00c8c..0000000000000 --- a/cmd/kubeadm/app/util/version.go +++ /dev/null @@ -1,255 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "io/ioutil" - "net/http" - "regexp" - "strings" - "time" - - "github.com/pkg/errors" - - netutil "k8s.io/apimachinery/pkg/util/net" - versionutil "k8s.io/apimachinery/pkg/util/version" - pkgversion "k8s.io/component-base/version" - "k8s.io/klog/v2" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -const ( - getReleaseVersionTimeout = time.Duration(10 * time.Second) -) - -var ( - kubeReleaseBucketURL = "https://dl.k8s.io" - kubeReleaseRegex = regexp.MustCompile(`^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)([-0-9a-zA-Z_\.+]*)?$`) - kubeReleaseLabelRegex = regexp.MustCompile(`^((latest|stable)+(-[1-9](\.[1-9]([0-9])?)?)?)\z`) - kubeBucketPrefixes = regexp.MustCompile(`^((release|ci|ci-cross)/)?([-\w_\.+]+)$`) -) - -// KubernetesReleaseVersion is helper function that can fetch -// available version information from release servers based on -// label names, like "stable" or "latest". -// -// If argument is already semantic version string, it -// will return same string. -// -// In case of labels, it tries to fetch from release -// servers and then return actual semantic version. -// -// Available names on release servers: -// stable (latest stable release) -// stable-1 (latest stable release in 1.x) -// stable-1.0 (and similarly 1.1, 1.2, 1.3, ...) -// latest (latest release, including alpha/beta) -// latest-1 (latest release in 1.x, including alpha/beta) -// latest-1.0 (and similarly 1.1, 1.2, 1.3, ...) -func KubernetesReleaseVersion(version string) (string, error) { - return kubernetesReleaseVersion(version, fetchFromURL) -} - -// kubernetesReleaseVersion is a helper function to fetch -// available version information. Used for testing to eliminate -// the need for internet calls. -func kubernetesReleaseVersion(version string, fetcher func(string, time.Duration) (string, error)) (string, error) { - ver := normalizedBuildVersion(version) - if len(ver) != 0 { - return ver, nil - } - - bucketURL, versionLabel, err := splitVersion(version) - if err != nil { - return "", err - } - - // revalidate, if exact build from e.g. CI bucket requested. - ver = normalizedBuildVersion(versionLabel) - if len(ver) != 0 { - return ver, nil - } - - // kubeReleaseLabelRegex matches labels such as: latest, latest-1, latest-1.10 - if kubeReleaseLabelRegex.MatchString(versionLabel) { - // Try to obtain a client version. - // pkgversion.Get().String() should always return a correct version added by the golang - // linker and the build system. The version can still be missing when doing unit tests - // on individual packages. - clientVersion, clientVersionErr := kubeadmVersion(pkgversion.Get().String()) - // Fetch version from the internet. - url := fmt.Sprintf("%s/%s.txt", bucketURL, versionLabel) - body, err := fetcher(url, getReleaseVersionTimeout) - if err != nil { - if clientVersionErr == nil { - // Handle air-gapped environments by falling back to the client version. - klog.Warningf("could not fetch a Kubernetes version from the internet: %v", err) - klog.Warningf("falling back to the local client version: %s", clientVersion) - return kubernetesReleaseVersion(clientVersion, fetcher) - } - } - - if clientVersionErr != nil { - if err != nil { - klog.Warningf("could not obtain neither client nor remote version; fall back to: %s", constants.CurrentKubernetesVersion) - return kubernetesReleaseVersion(constants.CurrentKubernetesVersion.String(), fetcher) - } - - klog.Warningf("could not obtain client version; using remote version: %s", body) - return kubernetesReleaseVersion(body, fetcher) - } - - // both the client and the remote version are obtained; validate them and pick a stable version - body, err = validateStableVersion(body, clientVersion) - if err != nil { - return "", err - } - // Re-validate received version and return. - return kubernetesReleaseVersion(body, fetcher) - } - return "", errors.Errorf("version %q doesn't match patterns for neither semantic version nor labels (stable, latest, ...)", version) -} - -// KubernetesVersionToImageTag is helper function that replaces all -// non-allowed symbols in tag strings with underscores. -// Image tag can only contain lowercase and uppercase letters, digits, -// underscores, periods and dashes. -// Current usage is for CI images where all of symbols except '+' are valid, -// but function is for generic usage where input can't be always pre-validated. -func KubernetesVersionToImageTag(version string) string { - allowed := regexp.MustCompile(`[^-a-zA-Z0-9_\.]`) - return allowed.ReplaceAllString(version, "_") -} - -// KubernetesIsCIVersion checks if user requested CI version -func KubernetesIsCIVersion(version string) bool { - subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1) - if len(subs) == 1 && len(subs[0]) == 4 && strings.HasPrefix(subs[0][2], "ci") { - return true - } - return false -} - -// Internal helper: returns normalized build version (with "v" prefix if needed) -// If input doesn't match known version pattern, returns empty string. -func normalizedBuildVersion(version string) string { - if kubeReleaseRegex.MatchString(version) { - if strings.HasPrefix(version, "v") { - return version - } - return "v" + version - } - return "" -} - -// Internal helper: split version parts, -// Return base URL and cleaned-up version -func splitVersion(version string) (string, string, error) { - var urlSuffix string - subs := kubeBucketPrefixes.FindAllStringSubmatch(version, 1) - if len(subs) != 1 || len(subs[0]) != 4 { - return "", "", errors.Errorf("invalid version %q", version) - } - - switch { - case strings.HasPrefix(subs[0][2], "ci"): - // Just use whichever the user specified - urlSuffix = subs[0][2] - default: - urlSuffix = "release" - } - url := fmt.Sprintf("%s/%s", kubeReleaseBucketURL, urlSuffix) - return url, subs[0][3], nil -} - -// Internal helper: return content of URL -func fetchFromURL(url string, timeout time.Duration) (string, error) { - klog.V(2).Infof("fetching Kubernetes version from URL: %s", url) - client := &http.Client{Timeout: timeout, Transport: netutil.SetOldTransportDefaults(&http.Transport{})} - resp, err := client.Get(url) - if err != nil { - return "", errors.Errorf("unable to get URL %q: %s", url, err.Error()) - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", errors.Errorf("unable to read content of URL %q: %s", url, err.Error()) - } - bodyString := strings.TrimSpace(string(body)) - - if resp.StatusCode != http.StatusOK { - msg := fmt.Sprintf("unable to fetch file. URL: %q, status: %v", url, resp.Status) - return bodyString, errors.New(msg) - } - return bodyString, nil -} - -// kubeadmVersion returns the version of the client without metadata. -func kubeadmVersion(info string) (string, error) { - v, err := versionutil.ParseSemantic(info) - if err != nil { - return "", errors.Wrap(err, "kubeadm version error") - } - // There is no utility in versionutil to get the version without the metadata, - // so this needs some manual formatting. - // Discard offsets after a release label and keep the labels down to e.g. `alpha.0` instead of - // including the offset e.g. `alpha.0.206`. This is done to comply with GCR image tags. - pre := v.PreRelease() - patch := v.Patch() - if len(pre) > 0 { - if patch > 0 { - // If the patch version is more than zero, decrement it and remove the label. - // this is done to comply with the latest stable patch release. - patch = patch - 1 - pre = "" - } else { - split := strings.Split(pre, ".") - if len(split) > 2 { - pre = split[0] + "." + split[1] // Exclude the third element - } else if len(split) < 2 { - pre = split[0] + ".0" // Append .0 to a partial label - } - pre = "-" + pre - } - } - vStr := fmt.Sprintf("v%d.%d.%d%s", v.Major(), v.Minor(), patch, pre) - return vStr, nil -} - -// Validate if the remote version is one Minor release newer than the client version. -// This is done to conform with "stable-X" and only allow remote versions from -// the same Patch level release. -func validateStableVersion(remoteVersion, clientVersion string) (string, error) { - verRemote, err := versionutil.ParseGeneric(remoteVersion) - if err != nil { - return "", errors.Wrap(err, "remote version error") - } - verClient, err := versionutil.ParseGeneric(clientVersion) - if err != nil { - return "", errors.Wrap(err, "client version error") - } - // If the remote Major version is bigger or if the Major versions are the same, - // but the remote Minor is bigger use the client version release. This handles Major bumps too. - if verClient.Major() < verRemote.Major() || - (verClient.Major() == verRemote.Major()) && verClient.Minor() < verRemote.Minor() { - estimatedRelease := fmt.Sprintf("stable-%d.%d", verClient.Major(), verClient.Minor()) - klog.Infof("remote version is much newer: %s; falling back to: %s", remoteVersion, estimatedRelease) - return estimatedRelease, nil - } - return remoteVersion, nil -} diff --git a/cmd/kubeadm/app/util/version_test.go b/cmd/kubeadm/app/util/version_test.go deleted file mode 100644 index 527ca18bce32f..0000000000000 --- a/cmd/kubeadm/app/util/version_test.go +++ /dev/null @@ -1,481 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "errors" - "fmt" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "path" - "strings" - "testing" - "time" -) - -func TestEmptyVersion(t *testing.T) { - - ver, err := KubernetesReleaseVersion("") - if err == nil { - t.Error("KubernetesReleaseVersion returned successfully, but error expected") - } - if ver != "" { - t.Error("KubernetesReleaseVersion returned value, expected only error") - } -} - -func TestValidVersion(t *testing.T) { - validVersions := []string{ - "v1.3.0", - "v1.4.0-alpha.0", - "v1.4.5", - "v1.4.0-beta.0", - "v2.0.0", - "v1.6.0-alpha.0.536+d60d9f3269288f", - "v1.5.0-alpha.0.1078+1044b6822497da-pull", - "v1.5.0-alpha.1.822+49b9e32fad9f32-pull-gke-gci", - "v1.6.1+coreos.0", - } - for _, s := range validVersions { - t.Run(s, func(t *testing.T) { - ver, err := kubernetesReleaseVersion(s, errorFetcher) - t.Log("Valid: ", s, ver, err) - if err != nil { - t.Errorf("KubernetesReleaseVersion unexpected error for version %q: %v", s, err) - } - if ver != s { - t.Errorf("KubernetesReleaseVersion should return same valid version string. %q != %q", s, ver) - } - }) - } -} - -func TestInvalidVersion(t *testing.T) { - invalidVersions := []string{ - "v1.3", - "1.4", - "b1.4.0", - "c1.4.5+git", - "something1.2", - } - for _, s := range invalidVersions { - t.Run(s, func(t *testing.T) { - ver, err := kubernetesReleaseVersion(s, errorFetcher) - t.Log("Invalid: ", s, ver, err) - if err == nil { - t.Errorf("KubernetesReleaseVersion error expected for version %q, but returned successfully", s) - } - if ver != "" { - t.Errorf("KubernetesReleaseVersion should return empty string in case of error. Returned %q for version %q", ver, s) - } - }) - } -} - -func TestValidConvenientForUserVersion(t *testing.T) { - validVersions := []string{ - "1.4.0", - "1.4.5+git", - "1.6.1_coreos.0", - } - for _, s := range validVersions { - t.Run(s, func(t *testing.T) { - ver, err := kubernetesReleaseVersion(s, errorFetcher) - t.Log("Valid: ", s, ver, err) - if err != nil { - t.Errorf("KubernetesReleaseVersion unexpected error for version %q: %v", s, err) - } - if ver != "v"+s { - t.Errorf("KubernetesReleaseVersion should return semantic version string. %q vs. %q", s, ver) - } - }) - } -} - -func TestVersionFromNetwork(t *testing.T) { - type T struct { - Content string - Expected string - FetcherErrorExpected bool - ErrorExpected bool - } - - currentVersion := normalizedBuildVersion(constants.CurrentKubernetesVersion.String()) - - cases := map[string]T{ - "stable": {"stable-1", "v1.4.6", false, false}, // recursive pointer to stable-1 - "stable-1": {"v1.4.6", "v1.4.6", false, false}, - "stable-1.3": {"v1.3.10", "v1.3.10", false, false}, - "latest": {"v1.6.0-alpha.0", "v1.6.0-alpha.0", false, false}, - "latest-1.3": {"v1.3.11-beta.0", "v1.3.11-beta.0", false, false}, - "latest-1.5": {"", currentVersion, true, false}, // fallback to currentVersion on fetcher error - "invalid-version": {"", "", false, true}, // invalid version cannot be parsed - } - - for k, v := range cases { - t.Run(k, func(t *testing.T) { - - fileFetcher := func(url string, timeout time.Duration) (string, error) { - key := strings.TrimSuffix(path.Base(url), ".txt") - res, found := cases[key] - if found { - if v.FetcherErrorExpected { - return "error", errors.New("expected error") - } - return res.Content, nil - } - return "Unknown test case key!", errors.New("unknown test case key") - } - - ver, err := kubernetesReleaseVersion(k, fileFetcher) - t.Logf("Key: %q. Result: %q, Error: %v", k, ver, err) - switch { - case err != nil && !v.ErrorExpected: - t.Errorf("KubernetesReleaseVersion: unexpected error for %q. Error: %+v", k, err) - case err == nil && v.ErrorExpected: - t.Errorf("KubernetesReleaseVersion: error expected for key %q, but result is %q", k, ver) - case ver != v.Expected: - t.Errorf("KubernetesReleaseVersion: unexpected result for key %q. Expected: %q Actual: %q", k, v.Expected, ver) - } - }) - } -} - -func TestVersionToTag(t *testing.T) { - type T struct { - input string - expected string - } - cases := []T{ - // NOP - {"", ""}, - // Official releases - {"v1.0.0", "v1.0.0"}, - // CI or custom builds - {"v10.1.2-alpha.1.100+0123456789abcdef+SOMETHING", "v10.1.2-alpha.1.100_0123456789abcdef_SOMETHING"}, - // random and invalid input: should return safe value - {"v1,0!0+üñµ", "v1_0_0____"}, - } - - for _, tc := range cases { - t.Run(fmt.Sprintf("input:%s/expected:%s", tc.input, tc.expected), func(t *testing.T) { - tag := KubernetesVersionToImageTag(tc.input) - t.Logf("KubernetesVersionToImageTag: Input: %q. Result: %q. Expected: %q", tc.input, tag, tc.expected) - if tag != tc.expected { - t.Errorf("failed KubernetesVersionToImageTag: Input: %q. Result: %q. Expected: %q", tc.input, tag, tc.expected) - } - }) - } -} - -func TestSplitVersion(t *testing.T) { - type T struct { - input string - bucket string - label string - valid bool - } - cases := []T{ - // Release area - {"v1.7.0", "https://dl.k8s.io/release", "v1.7.0", true}, - {"v1.8.0-alpha.2.1231+afabd012389d53a", "https://dl.k8s.io/release", "v1.8.0-alpha.2.1231+afabd012389d53a", true}, - {"release/v1.7.0", "https://dl.k8s.io/release", "v1.7.0", true}, - {"release/latest-1.7", "https://dl.k8s.io/release", "latest-1.7", true}, - // CI builds area - {"ci/latest", "https://dl.k8s.io/ci", "latest", true}, - {"ci/latest-1.7", "https://dl.k8s.io/ci", "latest-1.7", true}, - // unknown label in default (release) area: splitVersion validate only areas. - {"unknown-1", "https://dl.k8s.io/release", "unknown-1", true}, - // unknown area, not valid input. - {"unknown/latest-1", "", "", false}, - } - // kubeReleaseBucketURL can be overridden during network tests, thus ensure - // it will contain value corresponding to expected outcome for this unit test - kubeReleaseBucketURL = "https://dl.k8s.io" - - for _, tc := range cases { - t.Run(fmt.Sprintf("input:%s/label:%s", tc.input, tc.label), func(t *testing.T) { - bucket, label, err := splitVersion(tc.input) - switch { - case err != nil && tc.valid: - t.Errorf("splitVersion: unexpected error for %q. Error: %v", tc.input, err) - case err == nil && !tc.valid: - t.Errorf("splitVersion: error expected for key %q, but result is %q, %q", tc.input, bucket, label) - case bucket != tc.bucket: - t.Errorf("splitVersion: unexpected bucket result for key %q. Expected: %q Actual: %q", tc.input, tc.bucket, bucket) - case label != tc.label: - t.Errorf("splitVersion: unexpected label result for key %q. Expected: %q Actual: %q", tc.input, tc.label, label) - } - }) - } -} - -func TestKubernetesIsCIVersion(t *testing.T) { - type T struct { - input string - expected bool - } - cases := []T{ - {"", false}, - // Official releases - {"v1.0.0", false}, - {"release/v1.0.0", false}, - // CI builds - {"ci/latest-1", true}, - {"ci/v1.9.0-alpha.1.123+acbcbfd53bfa0a", true}, - } - - for _, tc := range cases { - t.Run(fmt.Sprintf("input:%s/expected:%t", tc.input, tc.expected), func(t *testing.T) { - result := KubernetesIsCIVersion(tc.input) - t.Logf("KubernetesIsCIVersion: Input: %q. Result: %v. Expected: %v", tc.input, result, tc.expected) - if result != tc.expected { - t.Errorf("failed KubernetesIsCIVersion: Input: %q. Result: %v. Expected: %v", tc.input, result, tc.expected) - } - }) - } -} - -// Validate KubernetesReleaseVersion but with bucket prefixes -func TestCIBuildVersion(t *testing.T) { - type T struct { - input string - expected string - valid bool - } - cases := []T{ - // Official releases - {"v1.7.0", "v1.7.0", true}, - {"release/v1.8.0", "v1.8.0", true}, - {"1.4.0-beta.0", "v1.4.0-beta.0", true}, - {"release/0invalid", "", false}, - // CI or custom builds - {"ci/v1.9.0-alpha.1.123+acbcbfd53bfa0a", "v1.9.0-alpha.1.123+acbcbfd53bfa0a", true}, - {"ci/1.9.0-alpha.1.123+acbcbfd53bfa0a", "v1.9.0-alpha.1.123+acbcbfd53bfa0a", true}, - {"ci/0invalid", "", false}, - } - - for _, tc := range cases { - t.Run(fmt.Sprintf("input:%s/expected:%s", tc.input, tc.expected), func(t *testing.T) { - - fileFetcher := func(url string, timeout time.Duration) (string, error) { - if tc.valid { - return tc.expected, nil - } - return "Unknown test case key!", errors.New("unknown test case key") - } - - ver, err := kubernetesReleaseVersion(tc.input, fileFetcher) - t.Logf("Input: %q. Result: %q, Error: %v", tc.input, ver, err) - switch { - case err != nil && tc.valid: - t.Errorf("KubernetesReleaseVersion: unexpected error for input %q. Error: %v", tc.input, err) - case err == nil && !tc.valid: - t.Errorf("KubernetesReleaseVersion: error expected for input %q, but result is %q", tc.input, ver) - case ver != tc.expected: - t.Errorf("KubernetesReleaseVersion: unexpected result for input %q. Expected: %q Actual: %q", tc.input, tc.expected, ver) - } - }) - } -} - -func TestNormalizedBuildVersionVersion(t *testing.T) { - type T struct { - input string - expected string - } - cases := []T{ - {"v1.7.0", "v1.7.0"}, - {"v1.8.0-alpha.2.1231+afabd012389d53a", "v1.8.0-alpha.2.1231+afabd012389d53a"}, - {"1.7.0", "v1.7.0"}, - {"unknown-1", ""}, - } - - for _, tc := range cases { - t.Run(fmt.Sprintf("input:%s/expected:%s", tc.input, tc.expected), func(t *testing.T) { - output := normalizedBuildVersion(tc.input) - if output != tc.expected { - t.Errorf("normalizedBuildVersion: unexpected output %q for input %q. Expected: %q", output, tc.input, tc.expected) - } - }) - } -} - -func TestKubeadmVersion(t *testing.T) { - type T struct { - name string - input string - output string - outputError bool - parsingError bool - } - cases := []T{ - { - name: "valid version with label and metadata", - input: "v1.8.0-alpha.2.1231+afabd012389d53a", - output: "v1.8.0-alpha.2", - }, - { - name: "valid version with label and extra metadata", - input: "v1.8.0-alpha.2.1231+afabd012389d53a.extra", - output: "v1.8.0-alpha.2", - }, - { - name: "valid patch version with label and extra metadata", - input: "v1.11.3-beta.0.38+135cc4c1f47994", - output: "v1.11.2", - }, - { - name: "valid version with label extra", - input: "v1.8.0-alpha.2.1231", - output: "v1.8.0-alpha.2", - }, - { - name: "valid patch version with label", - input: "v1.9.11-beta.0", - output: "v1.9.10", - }, - { - name: "handle version with partial label", - input: "v1.8.0-alpha", - output: "v1.8.0-alpha.0", - }, - { - name: "handle version missing 'v'", - input: "1.11.0", - output: "v1.11.0", - }, - { - name: "valid version without label and metadata", - input: "v1.8.0", - output: "v1.8.0", - }, - { - name: "valid patch version without label and metadata", - input: "v1.8.2", - output: "v1.8.2", - }, - { - name: "invalid version", - input: "foo", - parsingError: true, - }, - { - name: "invalid version with stray dash", - input: "v1.9.11-", - parsingError: true, - }, - { - name: "invalid version without patch release", - input: "v1.9", - parsingError: true, - }, - { - name: "invalid version with label and metadata", - input: "v1.8.0-alpha.2.1231+afabd012389d53a", - output: "v1.8.0-alpha.3", - outputError: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - output, err := kubeadmVersion(tc.input) - if (err != nil) != tc.parsingError { - t.Fatalf("expected error: %v, got: %v", tc.parsingError, err != nil) - } - if (output != tc.output) != tc.outputError { - t.Fatalf("expected output: %s, got: %s, for input: %s", tc.output, output, tc.input) - } - }) - } -} - -func TestValidateStableVersion(t *testing.T) { - type T struct { - name string - remoteVersion string - clientVersion string - output string - expectedError bool - } - cases := []T{ - { - name: "valid: remote version is newer; return stable label [1]", - remoteVersion: "v1.12.0", - clientVersion: "v1.11.0", - output: "stable-1.11", - }, - { - name: "valid: remote version is newer; return stable label [2]", - remoteVersion: "v2.0.0", - clientVersion: "v1.11.0", - output: "stable-1.11", - }, - { - name: "valid: remote version is newer; return stable label [3]", - remoteVersion: "v2.1.5", - clientVersion: "v1.11.5", - output: "stable-1.11", - }, - { - name: "valid: return the remote version as it is part of the same release", - remoteVersion: "v1.11.5", - clientVersion: "v1.11.0", - output: "v1.11.5", - }, - { - name: "valid: return the same version", - remoteVersion: "v1.11.0", - clientVersion: "v1.11.0", - output: "v1.11.0", - }, - { - name: "invalid: client version is empty", - remoteVersion: "v1.12.1", - clientVersion: "", - expectedError: true, - }, - { - name: "invalid: error parsing the remote version", - remoteVersion: "invalid-version", - clientVersion: "v1.12.0", - expectedError: true, - }, - { - name: "invalid: error parsing the client version", - remoteVersion: "v1.12.0", - clientVersion: "invalid-version", - expectedError: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - output, err := validateStableVersion(tc.remoteVersion, tc.clientVersion) - if (err != nil) != tc.expectedError { - t.Fatalf("expected error: %v, got: %v", tc.expectedError, err != nil) - } - if output != tc.output { - t.Fatalf("expected output: %s, got: %s", tc.output, output) - } - }) - } -} - -func errorFetcher(url string, timeout time.Duration) (string, error) { - return "should not make internet calls", fmt.Errorf("should not make internet calls, tried to request url: %s", url) -} diff --git a/cmd/kubeadm/kubeadm.go b/cmd/kubeadm/kubeadm.go deleted file mode 100644 index 626ec1fef0c46..0000000000000 --- a/cmd/kubeadm/kubeadm.go +++ /dev/null @@ -1,26 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "k8s.io/kubernetes/cmd/kubeadm/app" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" -) - -func main() { - kubeadmutil.CheckErr(app.Run()) -} diff --git a/cmd/kubeadm/test/BUILD b/cmd/kubeadm/test/BUILD deleted file mode 100644 index 584937d82834e..0000000000000 --- a/cmd/kubeadm/test/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["util.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/test", - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/apis/kubeadm/v1beta2:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/certs:go_default_library", - "//cmd/kubeadm/app/util/config:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubeadm/test/cmd:all-srcs", - "//cmd/kubeadm/test/kubeconfig:all-srcs", - "//cmd/kubeadm/test/resources:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/test/cmd/BUILD b/cmd/kubeadm/test/cmd/BUILD deleted file mode 100644 index 4d3ba384b8276..0000000000000 --- a/cmd/kubeadm/test/cmd/BUILD +++ /dev/null @@ -1,56 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = ["util.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/test/cmd", - deps = [ - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "completion_test.go", - "init_test.go", - "join_test.go", - "token_test.go", - "version_test.go", - ], - data = ["//cmd/kubeadm"] + glob(["testdata/**"]), - embed = [":go_default_library"], - tags = [ - "integration", - "skip", - ], - deps = [ - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/phases/certs:go_default_library", - "//cmd/kubeadm/app/util/pkiutil:go_default_library", - "//cmd/kubeadm/test:go_default_library", - "//vendor/github.com/lithammer/dedent:go_default_library", - "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/sigs.k8s.io/yaml:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/test/cmd/completion_test.go b/cmd/kubeadm/test/cmd/completion_test.go deleted file mode 100644 index f61372fc1120e..0000000000000 --- a/cmd/kubeadm/test/cmd/completion_test.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import "testing" - -func TestCmdCompletion(t *testing.T) { - kubeadmPath := getKubeadmPath() - var tests = []struct { - name string - args string - expected bool - }{ - {"shell not expected", "", false}, - {"unsupported shell type", "foo", false}, - } - - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "completion", rt.args) - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdCompletion running 'kubeadm completion %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - }) - } -} diff --git a/cmd/kubeadm/test/cmd/init_test.go b/cmd/kubeadm/test/cmd/init_test.go deleted file mode 100644 index d120f87dec286..0000000000000 --- a/cmd/kubeadm/test/cmd/init_test.go +++ /dev/null @@ -1,358 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "fmt" - "os" - "os/exec" - "strings" - "testing" - - "github.com/lithammer/dedent" - "github.com/pkg/errors" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" - testutil "k8s.io/kubernetes/cmd/kubeadm/test" -) - -func runKubeadmInit(args ...string) (string, string, int, error) { - const dryRunDir = "KUBEADM_INIT_DRYRUN_DIR" - if err := os.Setenv(dryRunDir, os.TempDir()); err != nil { - panic(fmt.Sprintf("could not set the %s environment variable", dryRunDir)) - } - kubeadmPath := getKubeadmPath() - kubeadmArgs := []string{"init", "--dry-run", "--ignore-preflight-errors=all"} - kubeadmArgs = append(kubeadmArgs, args...) - return RunCmd(kubeadmPath, kubeadmArgs...) -} - -func TestCmdInitToken(t *testing.T) { - initTest := []struct { - name string - args string - expected bool - }{ - { - name: "invalid token size", - args: "--token=abcd:1234567890abcd", - expected: false, - }, - { - name: "invalid token non-lowercase", - args: "--token=Abcdef:1234567890abcdef", - expected: false, - }, - { - name: "valid token is accepted", - args: "--token=abcdef.0123456789abcdef", - expected: true, - }, - } - - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, err := runKubeadmInit(rt.args) - if (err == nil) != rt.expected { - t.Fatalf(dedent.Dedent(` - CmdInitToken test case %q failed with an error: %v - command 'kubeadm init %s' - expected: %t - err: %t - `), - rt.name, - err, - rt.args, - rt.expected, - (err == nil), - ) - } - }) - } -} - -func TestCmdInitKubernetesVersion(t *testing.T) { - initTest := []struct { - name string - args string - expected bool - }{ - { - name: "invalid semantic version string is detected", - args: "--kubernetes-version=v1.1", - expected: false, - }, - { - name: "valid version is accepted", - args: "--kubernetes-version=" + constants.CurrentKubernetesVersion.String(), - expected: true, - }, - } - - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, err := runKubeadmInit(rt.args) - if (err == nil) != rt.expected { - t.Fatalf(dedent.Dedent(` - CmdInitKubernetesVersion test case %q failed with an error: %v - command 'kubeadm init %s' - expected: %t - err: %t - `), - rt.name, - err, - rt.args, - rt.expected, - (err == nil), - ) - } - }) - } -} - -func TestCmdInitConfig(t *testing.T) { - initTest := []struct { - name string - args string - expected bool - }{ - { - name: "fail on non existing path", - args: "--config=/does/not/exist/foo/bar", - expected: false, - }, - { - name: "can't load old v1alpha1 config", - args: "--config=testdata/init/v1alpha1.yaml", - expected: false, - }, - { - name: "can't load old v1alpha2 config", - args: "--config=testdata/init/v1alpha2.yaml", - expected: false, - }, - { - name: "can't load old v1alpha3 config", - args: "--config=testdata/init/v1alpha3.yaml", - expected: false, - }, - { - name: "can load v1beta1 config", - args: "--config=testdata/init/v1beta1.yaml", - expected: true, - }, - { - name: "don't allow mixed arguments v1beta1", - args: "--kubernetes-version=1.11.0 --config=testdata/init/v1beta1.yaml", - expected: false, - }, - { - name: "can load v1beta2 config", - args: "--config=testdata/init/v1beta2.yaml", - expected: true, - }, - { - name: "don't allow mixed arguments v1beta2", - args: "--kubernetes-version=1.11.0 --config=testdata/init/v1beta2.yaml", - expected: false, - }, - { - name: "can load current component config", - args: "--config=testdata/init/current-component-config.yaml", - expected: true, - }, - { - name: "can't load old component config", - args: "--config=testdata/init/old-component-config.yaml", - expected: false, - }, - } - - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, err := runKubeadmInit(rt.args) - if (err == nil) != rt.expected { - t.Fatalf(dedent.Dedent(` - CmdInitConfig test case %q failed with an error: %v - command 'kubeadm init %s' - expected: %t - err: %t - `), - rt.name, - err, - rt.args, - rt.expected, - (err == nil), - ) - } - }) - } -} - -func TestCmdInitCertPhaseCSR(t *testing.T) { - tests := []struct { - name string - baseName string - expectedError string - }{ - { - name: "generate CSR", - baseName: certs.KubeadmCertKubeletClient().BaseName, - }, - { - name: "fails on CSR", - baseName: certs.KubeadmCertRootCA().BaseName, - expectedError: "unknown flag: --csr-only", - }, - { - name: "fails on all", - baseName: "all", - expectedError: "unknown flag: --csr-only", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - csrDir := testutil.SetupTempDir(t) - cert := certs.KubeadmCertKubeletClient() - kubeadmPath := getKubeadmPath() - _, stderr, _, err := RunCmd(kubeadmPath, - "init", - "phase", - "certs", - test.baseName, - "--csr-only", - "--csr-dir="+csrDir, - ) - - if test.expectedError != "" { - cause := errors.Cause(err) - _, ok := cause.(*exec.ExitError) - if !ok { - t.Fatalf("expected exitErr: got %T (%v)", cause, err) - } - - if !strings.Contains(stderr, test.expectedError) { - t.Errorf("expected %q to contain %q", stderr, test.expectedError) - } - return - } - - if err != nil { - t.Fatalf("couldn't run kubeadm: %v", err) - } - - if _, _, err := pkiutil.TryLoadCSRAndKeyFromDisk(csrDir, cert.BaseName); err != nil { - t.Fatalf("couldn't load certificate %q: %v", cert.BaseName, err) - } - }) - } -} - -func TestCmdInitAPIPort(t *testing.T) { - initTest := []struct { - name string - args string - expected bool - }{ - { - name: "fail on non-string port", - args: "--apiserver-bind-port=foobar", - expected: false, - }, - { - name: "fail on too large port number", - args: "--apiserver-bind-port=100000", - expected: false, - }, - { - name: "fail on negative port number", - args: "--apiserver-bind-port=-6000", - expected: false, - }, - { - name: "accept a valid port number", - args: "--apiserver-bind-port=6000", - expected: true, - }, - } - - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, err := runKubeadmInit(rt.args) - if (err == nil) != rt.expected { - t.Fatalf(dedent.Dedent(` - CmdInitAPIPort test case %q failed with an error: %v - command 'kubeadm init %s' - expected: %t - err: %t - `), - rt.name, - err, - rt.args, - rt.expected, - (err == nil), - ) - } - }) - } -} - -// TestCmdInitFeatureGates test that feature gates won't make kubeadm panic. -// When go panics it will exit with a 2 code. While we don't expect the init -// calls to succeed in these tests, we ensure that the exit code of calling -// kubeadm with different feature gates is not 2. -func TestCmdInitFeatureGates(t *testing.T) { - const PanicExitcode = 2 - - initTest := []struct { - name string - args string - }{ - { - name: "no feature gates passed", - args: "", - }, - { - name: "feature gate IPv6DualStack=true", - args: "--feature-gates=IPv6DualStack=true", - }, - { - name: "feature gate PublicKeysECDSA=true", - args: "--feature-gates=PublicKeysECDSA=true", - }, - } - - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, exitcode, err := runKubeadmInit(rt.args) - if exitcode == PanicExitcode { - t.Fatalf(dedent.Dedent(` - CmdInitFeatureGates test case %q failed with an error: %v - command 'kubeadm init %s' - got exit code: %t (panic); unexpected - `), - rt.name, - err, - rt.args, - PanicExitcode, - ) - } - }) - } -} diff --git a/cmd/kubeadm/test/cmd/join_test.go b/cmd/kubeadm/test/cmd/join_test.go deleted file mode 100644 index 3d5c70605f512..0000000000000 --- a/cmd/kubeadm/test/cmd/join_test.go +++ /dev/null @@ -1,248 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import "testing" - -// kubeadmReset executes "kubeadm reset" and restarts kubelet. -func kubeadmReset() error { - kubeadmPath := getKubeadmPath() - _, _, _, err := RunCmd(kubeadmPath, "reset") - return err -} - -func TestCmdJoinConfig(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"config", "--config=foobar", false}, - {"config path", "--config=/does/not/exist/foo/bar", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinConfig running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinDiscoveryFile(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"valid discovery file", "--discovery-file=foobar", false}, - {"invalid discovery file", "--discovery-file=file:wrong", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinDiscoveryFile running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinDiscoveryToken(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"valid discovery token", "--discovery-token=foobar", false}, - {"valid discovery token url", "--discovery-token=token://asdf:asdf", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinDiscoveryToken running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinNodeName(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"valid node name", "--node-name=foobar", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinNodeName running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinTLSBootstrapToken(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"valid bootstrap token", "--tls-bootstrap-token=foobar", false}, - {"valid bootstrap token url", "--tls-bootstrap-token=token://asdf:asdf", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinTLSBootstrapToken running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinToken(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"valid token", "--token=foobar", false}, - {"valid token url", "--token=token://asdf:asdf", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinToken running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinBadArgs(t *testing.T) { - kubeadmPath := getKubeadmPath() - var initTest = []struct { - name string - args string - expected bool - }{ - {"discovery-token and discovery-file can't both be set", "--discovery-token=abcdef.1234567890123456 --discovery-file=file:///tmp/foo.bar", false}, // DiscoveryToken, DiscoveryFile can't both be set - {"discovery-token or discovery-file must be set", "", false}, // DiscoveryToken or DiscoveryFile must be set - } - - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinBadArgs 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} - -func TestCmdJoinArgsMixed(t *testing.T) { - var initTest = []struct { - name string - args string - expected bool - }{ - {"discovery-token and config", "--discovery-token=abcdef.1234567890abcdef --config=/etc/kubernetes/kubeadm.config", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range initTest { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "join", rt.args, "--ignore-preflight-errors=all") - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdJoinArgsMixed running 'kubeadm join %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} diff --git a/cmd/kubeadm/test/cmd/testdata/init/current-component-config.yaml b/cmd/kubeadm/test/cmd/testdata/init/current-component-config.yaml deleted file mode 100644 index a3e15b92b0def..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/current-component-config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration ---- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration diff --git a/cmd/kubeadm/test/cmd/testdata/init/old-component-config.yaml b/cmd/kubeadm/test/cmd/testdata/init/old-component-config.yaml deleted file mode 100644 index 5334ce9f2b010..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/old-component-config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: ClusterConfiguration ---- -apiVersion: kubelet.config.k8s.io/v1alpha1 -kind: KubeletConfiguration diff --git a/cmd/kubeadm/test/cmd/testdata/init/v1alpha1.yaml b/cmd/kubeadm/test/cmd/testdata/init/v1alpha1.yaml deleted file mode 100644 index ae4dc96cb8a5b..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/v1alpha1.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1alpha1 -kind: MasterConfiguration diff --git a/cmd/kubeadm/test/cmd/testdata/init/v1alpha2.yaml b/cmd/kubeadm/test/cmd/testdata/init/v1alpha2.yaml deleted file mode 100644 index ddbc1f9f5bd1c..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/v1alpha2.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1alpha2 -kind: MasterConfiguration diff --git a/cmd/kubeadm/test/cmd/testdata/init/v1alpha3.yaml b/cmd/kubeadm/test/cmd/testdata/init/v1alpha3.yaml deleted file mode 100644 index 75bd6364291f9..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/v1alpha3.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1alpha3 -kind: InitConfiguration diff --git a/cmd/kubeadm/test/cmd/testdata/init/v1beta1.yaml b/cmd/kubeadm/test/cmd/testdata/init/v1beta1.yaml deleted file mode 100644 index 79a70e48bb505..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/v1beta1.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta1 -kind: InitConfiguration diff --git a/cmd/kubeadm/test/cmd/testdata/init/v1beta2.yaml b/cmd/kubeadm/test/cmd/testdata/init/v1beta2.yaml deleted file mode 100644 index 078cd6075faad..0000000000000 --- a/cmd/kubeadm/test/cmd/testdata/init/v1beta2.yaml +++ /dev/null @@ -1,2 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta2 -kind: InitConfiguration diff --git a/cmd/kubeadm/test/cmd/token_test.go b/cmd/kubeadm/test/cmd/token_test.go deleted file mode 100644 index 9a1b8964940cb..0000000000000 --- a/cmd/kubeadm/test/cmd/token_test.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "regexp" - "testing" -) - -const ( - TokenExpectedRegex = "^\\S{6}\\.\\S{16}\n$" -) - -func TestCmdTokenGenerate(t *testing.T) { - kubeadmPath := getKubeadmPath() - stdout, _, _, err := RunCmd(kubeadmPath, "token", "generate") - if err != nil { - t.Fatalf("'kubeadm token generate' exited uncleanly: %v", err) - } - - matched, err := regexp.MatchString(TokenExpectedRegex, stdout) - if err != nil { - t.Fatalf("encountered an error while trying to match 'kubeadm token generate' stdout: %v", err) - } - if !matched { - t.Errorf("'kubeadm token generate' stdout did not match expected regex; wanted: [%q], got: [%s]", TokenExpectedRegex, stdout) - } -} - -func TestCmdTokenGenerateTypoError(t *testing.T) { - /* - Since we expect users to do things like this: - - $ TOKEN=$(kubeadm token generate) - - we want to make sure that if they have a typo in their command, we exit - with a non-zero status code after showing the command's usage, so that - the usage itself isn't captured as a token without the user noticing. - */ - kubeadmPath := getKubeadmPath() - _, _, _, err := RunCmd(kubeadmPath, "token", "genorate") // subtle typo - if err == nil { - t.Error("'kubeadm token genorate' (a deliberate typo) exited without an error when we expected non-zero exit status") - } -} -func TestCmdTokenDelete(t *testing.T) { - var tests = []struct { - name string - args string - expected bool - }{ - {"no token provided", "", false}, - {"invalid token", "foobar", false}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range tests { - t.Run(rt.name, func(t *testing.T) { - _, _, _, actual := RunCmd(kubeadmPath, "token", "delete", rt.args) - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdTokenDelete running 'kubeadm token %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - kubeadmReset() - }) - } -} diff --git a/cmd/kubeadm/test/cmd/util.go b/cmd/kubeadm/test/cmd/util.go deleted file mode 100644 index 2c33f99cbc58a..0000000000000 --- a/cmd/kubeadm/test/cmd/util.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "bytes" - "os" - "os/exec" - "testing" - - "github.com/pkg/errors" - - "github.com/spf13/cobra" -) - -// Forked from test/e2e/framework because the e2e framework is quite bloated -// for our purposes here, and modified to remove undesired logging. - -func runCmdNoWrap(command string, args ...string) (string, string, int, error) { - var bout, berr bytes.Buffer - cmd := exec.Command(command, args...) - cmd.Stdout = &bout - cmd.Stderr = &berr - err := cmd.Run() - stdout, stderr := bout.String(), berr.String() - return stdout, stderr, cmd.ProcessState.ExitCode(), err -} - -// RunCmd is a utility function for kubeadm testing that executes a specified command -func RunCmd(command string, args ...string) (string, string, int, error) { - stdout, stderr, retcode, err := runCmdNoWrap(command, args...) - if err != nil { - return stdout, stderr, retcode, errors.Wrapf(err, "error running %s %v; \nretcode %d, \nstdout %q, \nstderr %q, \ngot error", - command, args, retcode, stdout, stderr) - } - return stdout, stderr, retcode, nil -} - -// RunSubCommand is a utility function for kubeadm testing that executes a Cobra sub command -func RunSubCommand(t *testing.T, subCmds []*cobra.Command, command string, args ...string) { - subCmd := getSubCommand(t, subCmds, command) - subCmd.SetArgs(args) - if err := subCmd.Execute(); err != nil { - t.Fatalf("Could not execute subcommand: %s", command) - } -} - -// AssertSubCommandHasFlags is a utility function for kubeadm testing that assert if a Cobra sub command has expected flags -func AssertSubCommandHasFlags(t *testing.T, subCmds []*cobra.Command, command string, flags ...string) { - subCmd := getSubCommand(t, subCmds, command) - - for _, flag := range flags { - if subCmd.Flags().Lookup(flag) == nil { - t.Errorf("Could not find expecte flag %s for command %s", flag, command) - } - } -} - -func getSubCommand(t *testing.T, subCmds []*cobra.Command, name string) *cobra.Command { - for _, subCmd := range subCmds { - if subCmd.Name() == name { - return subCmd - } - } - t.Fatalf("Unable to find sub command %s", name) - - return nil -} - -// getKubeadmPath returns the contents of the environment variable KUBEADM_PATH -// or panics if it's empty -func getKubeadmPath() string { - kubeadmPath := os.Getenv("KUBEADM_PATH") - if len(kubeadmPath) == 0 { - panic("the environment variable KUBEADM_PATH must point to the kubeadm binary path") - } - return kubeadmPath -} diff --git a/cmd/kubeadm/test/cmd/version_test.go b/cmd/kubeadm/test/cmd/version_test.go deleted file mode 100644 index 30820035a656f..0000000000000 --- a/cmd/kubeadm/test/cmd/version_test.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeadm - -import ( - "encoding/json" - "regexp" - "testing" - - "sigs.k8s.io/yaml" -) - -const ( - ShortExpectedRegex = "^v.+\n$" - NormalExpectedRegex = "^kubeadm version: &version\\.Info{Major:\".+\", Minor:\".+\", GitVersion:\".+\", GitCommit:\".+\", GitTreeState:\".+\", BuildDate:\".+\", GoVersion:\".+\", Compiler:\".+\", Platform:\".+\"}\n$" -) - -var ( - VersionInfo = []string{"major", "minor", "gitVersion", "gitCommit", "gitTreeState", "buildDate", "goVersion", "compiler", "platform"} -) - -func TestCmdVersion(t *testing.T) { - var versionTest = []struct { - name string - args string - regex string - expected bool - }{ - {"invalid output option", "--output=valid", "", false}, - {"short output", "--output=short", ShortExpectedRegex, true}, - {"default output", "", NormalExpectedRegex, true}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range versionTest { - t.Run(rt.name, func(t *testing.T) { - args := []string{"version"} - if len(rt.args) > 0 { - args = append(args, rt.args) - } - stdout, _, _, actual := RunCmd(kubeadmPath, args...) - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdVersion running 'kubeadm version %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - - if rt.expected { - matched, err := regexp.MatchString(rt.regex, stdout) - if err != nil { - t.Fatalf("encountered an error while trying to match 'kubeadm version %s' stdout: %v", rt.args, err) - } - if !matched { - t.Errorf("'kubeadm version %s' stdout did not match expected regex; wanted: [%q], got: [%s]", rt.args, rt.regex, stdout) - } - } - }) - } -} - -func TestCmdVersionOutputJsonOrYaml(t *testing.T) { - var versionTest = []struct { - name string - args string - format string - expected bool - }{ - {"json output", "--output=json", "json", true}, - {"yaml output", "--output=yaml", "yaml", true}, - } - - kubeadmPath := getKubeadmPath() - for _, rt := range versionTest { - t.Run(rt.name, func(t *testing.T) { - stdout, _, _, actual := RunCmd(kubeadmPath, "version", rt.args) - if (actual == nil) != rt.expected { - t.Errorf( - "failed CmdVersion running 'kubeadm version %s' with an error: %v\n\texpected: %t\n\t actual: %t", - rt.args, - actual, - rt.expected, - (actual == nil), - ) - } - - if rt.expected { - var obj interface{} - switch rt.format { - case "json": - err := json.Unmarshal([]byte(stdout), &obj) - if err != nil { - t.Errorf("failed to parse json from 'kubeadm version %s': %s", rt.args, err) - } - case "yaml": - err := yaml.Unmarshal([]byte(stdout), &obj) - if err != nil { - t.Errorf("failed to parse yaml from 'kubeadm version %s': %s", rt.args, err) - } - } - - m := obj.(map[string]interface{}) - if m["clientVersion"] == nil { - t.Errorf("failed to get the information of clientVersion from 'kubeadm version %s'", rt.args) - } - info := m["clientVersion"].(map[string]interface{}) - for _, key := range VersionInfo { - if len(info[key].(string)) == 0 { - t.Errorf("failed to get the information of %s from 'kubeadm version %s'", key, rt.args) - } - } - } - }) - } -} diff --git a/cmd/kubeadm/test/kubeconfig/BUILD b/cmd/kubeadm/test/kubeconfig/BUILD deleted file mode 100644 index 792e593e1361c..0000000000000 --- a/cmd/kubeadm/test/kubeconfig/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["util.go"], - importpath = "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig", - deps = [ - "//cmd/kubeadm/app/util/certs:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubeadm/test/kubeconfig/util.go b/cmd/kubeadm/test/kubeconfig/util.go deleted file mode 100644 index 46774cd02ab05..0000000000000 --- a/cmd/kubeadm/test/kubeconfig/util.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeconfig - -import ( - "crypto/x509" - "encoding/pem" - "testing" - - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - certstestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" -) - -// AssertKubeConfigCurrentCluster is a utility function for kubeadm testing that asserts if the CurrentCluster in -// the given KubeConfig object contains refers to a specific cluster -func AssertKubeConfigCurrentCluster(t *testing.T, config *clientcmdapi.Config, expectedAPIServerAddress string, expectedAPIServerCaCert *x509.Certificate) { - currentContext := config.Contexts[config.CurrentContext] - currentCluster := config.Clusters[currentContext.Cluster] - - // Assert expectedAPIServerAddress - if currentCluster.Server != expectedAPIServerAddress { - t.Errorf("kubeconfig.currentCluster.Server is [%s], expected [%s]", currentCluster.Server, expectedAPIServerAddress) - } - - // Assert the APIServerCaCert - if len(currentCluster.CertificateAuthorityData) == 0 { - t.Error("kubeconfig.currentCluster.CertificateAuthorityData is empty, expected not empty") - return - } - - block, _ := pem.Decode(currentCluster.CertificateAuthorityData) - currentAPIServerCaCert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - t.Errorf("kubeconfig.currentCluster.CertificateAuthorityData is not a valid CA: %v", err) - return - } - - if !currentAPIServerCaCert.Equal(expectedAPIServerCaCert) { - t.Errorf("kubeconfig.currentCluster.CertificateAuthorityData not correspond to the expected CA cert") - } -} - -// AssertKubeConfigCurrentAuthInfoWithClientCert is a utility function for kubeadm testing that asserts if the CurrentAuthInfo in -// the given KubeConfig object contains a clientCert that refers to a specific client name, is signed by the expected CA, includes the expected organizations -func AssertKubeConfigCurrentAuthInfoWithClientCert(t *testing.T, config *clientcmdapi.Config, signinCa *x509.Certificate, expectedClientName string, expectedOrganizations ...string) { - currentContext := config.Contexts[config.CurrentContext] - currentAuthInfo := config.AuthInfos[currentContext.AuthInfo] - - // assert clientCert - if len(currentAuthInfo.ClientCertificateData) == 0 { - t.Error("kubeconfig.currentAuthInfo.ClientCertificateData is empty, expected not empty") - return - } - - block, _ := pem.Decode(config.AuthInfos[currentContext.AuthInfo].ClientCertificateData) - currentClientCert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - t.Errorf("kubeconfig.currentAuthInfo.ClientCertificateData is not a valid CA: %v", err) - return - } - - // Asserts the clientCert is signed by the signinCa - certstestutil.AssertCertificateIsSignedByCa(t, currentClientCert, signinCa) - - // Asserts the clientCert has ClientAuth ExtKeyUsage - certstestutil.AssertCertificateHasClientAuthUsage(t, currentClientCert) - - // Asserts the clientCert has expected expectedUserName as CommonName - certstestutil.AssertCertificateHasCommonName(t, currentClientCert, expectedClientName) - - // Asserts the clientCert has expected Organizations - certstestutil.AssertCertificateHasOrganizations(t, currentClientCert, expectedOrganizations...) -} - -// AssertKubeConfigCurrentAuthInfoWithToken is a utility function for kubeadm testing that asserts if the CurrentAuthInfo in -// the given KubeConfig object refers to expected token -func AssertKubeConfigCurrentAuthInfoWithToken(t *testing.T, config *clientcmdapi.Config, expectedClientName, expectedToken string) { - currentContext := config.Contexts[config.CurrentContext] - currentAuthInfo := config.AuthInfos[currentContext.AuthInfo] - - // assert token - if currentAuthInfo.Token != expectedToken { - t.Errorf("kubeconfig.currentAuthInfo.Token [%s], expected [%s]", currentAuthInfo.Token, expectedToken) - return - } -} - -// AssertKubeConfigCurrentContextWithClusterName is a utility function for kubeadm testing that asserts if the Current Cluster config in -// the given KubeConfig object refers to expected cluster name -func AssertKubeConfigCurrentContextWithClusterName(t *testing.T, config *clientcmdapi.Config, expectedClusterName string) { - currentContext := config.Contexts[config.CurrentContext] - - // assert cluster name - if currentContext.Cluster != expectedClusterName { - t.Errorf("kubeconfig.currentContext.clusterName [%s], expected [%s]", currentContext.Cluster, expectedClusterName) - return - } -} diff --git a/cmd/kubeadm/test/resources/BUILD b/cmd/kubeadm/test/resources/BUILD deleted file mode 100644 index 20a7ec198a533..0000000000000 --- a/cmd/kubeadm/test/resources/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = [ - "cluster_status.go", - "configmap.go", - "pods.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubeadm/test/resources", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubeadm/app/apis/kubeadm:go_default_library", - "//cmd/kubeadm/app/constants:go_default_library", - "//cmd/kubeadm/app/util/apiclient:go_default_library", - "//cmd/kubeadm/app/util/staticpod:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubeadm/test/resources/cluster_status.go b/cmd/kubeadm/test/resources/cluster_status.go deleted file mode 100644 index 7534358c12ac3..0000000000000 --- a/cmd/kubeadm/test/resources/cluster_status.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "encoding/json" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" -) - -// ClusterStatusWithAPIEndpoint returns a FakeConfigMap containing a -// cluster status with the provided endpoint for nodeName as a single -// entry -func ClusterStatusWithAPIEndpoint(nodeName string, endpoint kubeadmapi.APIEndpoint) *FakeConfigMap { - marshaledClusterStatus, _ := json.Marshal(kubeadmapi.ClusterStatus{ - APIEndpoints: map[string]kubeadmapi.APIEndpoint{ - nodeName: endpoint, - }, - }) - return &FakeConfigMap{ - Name: constants.KubeadmConfigConfigMap, - Data: map[string]string{ - constants.ClusterStatusConfigMapKey: string(marshaledClusterStatus), - }, - } -} diff --git a/cmd/kubeadm/test/resources/configmap.go b/cmd/kubeadm/test/resources/configmap.go deleted file mode 100644 index 15a7a8f9ed85c..0000000000000 --- a/cmd/kubeadm/test/resources/configmap.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" -) - -// FakeConfigMap represents a fake config map -type FakeConfigMap struct { - Name string - Data map[string]string -} - -// Create creates a fake configmap using the provided client -func (c *FakeConfigMap) Create(client clientset.Interface) error { - return apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: c.Name, - Namespace: metav1.NamespaceSystem, - }, - Data: c.Data, - }) -} diff --git a/cmd/kubeadm/test/resources/pods.go b/cmd/kubeadm/test/resources/pods.go deleted file mode 100644 index d154bfb5711ec..0000000000000 --- a/cmd/kubeadm/test/resources/pods.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "context" - "fmt" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" -) - -// FakeStaticPod represents a fake static pod -type FakeStaticPod struct { - NodeName string - Component string - Annotations map[string]string -} - -// Pod returns a pod structure representing the fake static pod with a -// given suffix -func (p *FakeStaticPod) Pod(suffix string) *v1.Pod { - pod := staticpodutil.ComponentPod( - v1.Container{ - Name: p.Component, - Image: fmt.Sprintf("%s-image:tag", p.Component), - }, - map[string]v1.Volume{}, - p.Annotations, - ) - if len(suffix) > 0 { - pod.ObjectMeta.Name = fmt.Sprintf("%s-%s-%s", p.Component, p.NodeName, suffix) - } else { - pod.ObjectMeta.Name = fmt.Sprintf("%s-%s", p.Component, p.NodeName) - } - pod.Spec.NodeName = p.NodeName - return &pod -} - -// Create creates a fake static pod using the provided client -func (p *FakeStaticPod) Create(client clientset.Interface) error { - return p.CreateWithPodSuffix(client, "") -} - -// CreateWithPodSuffix creates a fake static pod using the provided -// client and suffix -func (p *FakeStaticPod) CreateWithPodSuffix(client clientset.Interface, suffix string) error { - _, err := client.CoreV1().Pods(metav1.NamespaceSystem).Create(context.TODO(), p.Pod(suffix), metav1.CreateOptions{}) - return err -} diff --git a/cmd/kubeadm/test/util.go b/cmd/kubeadm/test/util.go deleted file mode 100644 index 104ba220e2b42..0000000000000 --- a/cmd/kubeadm/test/util.go +++ /dev/null @@ -1,166 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "fmt" - "html/template" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/lithammer/dedent" - - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" - kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - certtestutil "k8s.io/kubernetes/cmd/kubeadm/app/util/certs" - configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" -) - -// SetupTempDir is a utility function for kubeadm testing, that creates a temporary directory -// NB. it is up to the caller to cleanup the folder at the end of the test with defer os.RemoveAll(tmpdir) -func SetupTempDir(t *testing.T) string { - tmpdir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("Couldn't create tmpdir") - } - - return tmpdir -} - -// SetupInitConfigurationFile is a utility function for kubeadm testing that writes a InitConfiguration file -// into /config subfolder of a given temporary directory. -// The function returns the path of the created InitConfiguration file. -func SetupInitConfigurationFile(t *testing.T, tmpdir string, cfg *kubeadmapi.InitConfiguration) string { - - cfgPath := filepath.Join(tmpdir, "config/masterconfig.yaml") - if err := os.MkdirAll(filepath.Dir(cfgPath), os.FileMode(0755)); err != nil { - t.Fatalf("Couldn't create cfgDir") - } - - cfgTemplate := template.Must(template.New("init").Parse(dedent.Dedent(fmt.Sprintf(` - apiVersion: kubeadm.k8s.io/v1beta2 - kind: InitConfiguration - apiEndpoint: - advertiseAddress: {{.LocalAPIEndpoint.AdvertiseAddress}} - bindPort: {{.LocalAPIEndpoint.BindPort}} - nodeRegistration: - name: {{.NodeRegistration.Name}} - --- - apiVersion: kubeadm.k8s.io/v1beta2 - kind: ClusterConfiguration - certificatesDir: {{.CertificatesDir}} - kubernetesVersion: %s - `, kubeadmconstants.MinimumControlPlaneVersion)))) - - f, err := os.Create(cfgPath) - if err != nil { - t.Fatalf("error creating masterconfig file %s: %v", cfgPath, err) - } - - err = cfgTemplate.Execute(f, cfg) - if err != nil { - t.Fatalf("error generating masterconfig file %s: %v", cfgPath, err) - } - f.Close() - - return cfgPath -} - -// SetupEmptyFiles is a utility function for kubeadm testing that creates one or more empty files (touch) -func SetupEmptyFiles(t *testing.T, tmpdir string, fileNames ...string) { - for _, fileName := range fileNames { - newFile, err := os.Create(filepath.Join(tmpdir, fileName)) - if err != nil { - t.Fatalf("Error creating file %s in %s: %v", fileName, tmpdir, err) - } - newFile.Close() - } -} - -// SetupPkiDirWithCertificateAuthority is a utility function for kubeadm testing that creates a -// CertificateAuthority cert/key pair into /pki subfolder of a given temporary directory. -// The function returns the path of the created pki. -func SetupPkiDirWithCertificateAuthority(t *testing.T, tmpdir string) string { - caCert, caKey := certtestutil.SetupCertificateAuthority(t) - - certDir := filepath.Join(tmpdir, "pki") - if err := pkiutil.WriteCertAndKey(certDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil { - t.Fatalf("failure while saving CA certificate and key: %v", err) - } - - return certDir -} - -// AssertFilesCount is a utility function for kubeadm testing that asserts if the given folder contains -// count files. -func AssertFilesCount(t *testing.T, dirName string, count int) { - files, err := ioutil.ReadDir(dirName) - if err != nil { - t.Fatalf("Couldn't read files from tmpdir: %s", err) - } - - countFiles := 0 - for _, f := range files { - if !f.IsDir() { - countFiles++ - } - } - - if countFiles != count { - t.Errorf("dir does contains %d, %d expected", len(files), count) - for _, f := range files { - t.Error(f.Name()) - } - } -} - -// AssertFileExists is a utility function for kubeadm testing that asserts if the given folder contains -// the given files. -func AssertFileExists(t *testing.T, dirName string, fileNames ...string) { - for _, fileName := range fileNames { - path := filepath.Join(dirName, fileName) - - if _, err := os.Stat(path); os.IsNotExist(err) { - t.Errorf("file %s does not exist", fileName) - } - } -} - -// AssertError checks that the provided error matches the expected output -func AssertError(t *testing.T, err error, expected string) { - if err == nil { - t.Errorf("no error was found, but '%s' was expected", expected) - return - } - if err.Error() != expected { - t.Errorf("error '%s' does not match expected error: '%s'", err.Error(), expected) - } -} - -// GetDefaultInternalConfig returns a defaulted kubeadmapi.InitConfiguration -func GetDefaultInternalConfig(t *testing.T) *kubeadmapi.InitConfiguration { - internalcfg, err := configutil.DefaultedInitConfiguration(&kubeadmapiv1beta2.InitConfiguration{}, &kubeadmapiv1beta2.ClusterConfiguration{}) - if err != nil { - t.Fatalf("unexpected error getting default config: %v", err) - } - - return internalcfg -} diff --git a/cmd/kubectl-convert/BUILD b/cmd/kubectl-convert/BUILD deleted file mode 100644 index 81d97a7215440..0000000000000 --- a/cmd/kubectl-convert/BUILD +++ /dev/null @@ -1,44 +0,0 @@ -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kubectl", - embed = [":go_default_library"], - pure = "on", - visibility = ["//visibility:public"], - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["kubectl-convert.go"], - importpath = "k8s.io/kubernetes/cmd/kubectl-convert", - visibility = ["//visibility:private"], - deps = [ - "//pkg/kubectl/cmd/convert:go_default_library", - "//staging/src/k8s.io/cli-runtime/pkg/genericclioptions:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/cmd/util:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/util/logs:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = [ - "//build/visible_to:cmd_kubectl_CONSUMERS", - ], -) diff --git a/cmd/kubectl-convert/OWNERS b/cmd/kubectl-convert/OWNERS deleted file mode 100644 index a1f6b7eee3089..0000000000000 --- a/cmd/kubectl-convert/OWNERS +++ /dev/null @@ -1,9 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- sig-cli-maintainers -reviewers: -- sig-cli -labels: -- area/kubectl -- sig/cli diff --git a/cmd/kubectl-convert/kubectl-convert.go b/cmd/kubectl-convert/kubectl-convert.go deleted file mode 100644 index d030ae8bf3ad6..0000000000000 --- a/cmd/kubectl-convert/kubectl-convert.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - goflag "flag" - "os" - - "github.com/spf13/pflag" - - "k8s.io/cli-runtime/pkg/genericclioptions" - cliflag "k8s.io/component-base/cli/flag" - cmdutil "k8s.io/kubectl/pkg/cmd/util" - "k8s.io/kubectl/pkg/util/logs" - "k8s.io/kubernetes/pkg/kubectl/cmd/convert" -) - -func main() { - flags := pflag.NewFlagSet("kubectl-convert", pflag.ExitOnError) - pflag.CommandLine = flags - - kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() - kubeConfigFlags.AddFlags(flags) - matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags) - - f := cmdutil.NewFactory(matchVersionKubeConfigFlags) - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - // cliflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - cmd := convert.NewCmdConvert(f, genericclioptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr}) - matchVersionKubeConfigFlags.AddFlags(cmd.PersistentFlags()) - if err := cmd.Execute(); err != nil { - os.Exit(1) - } - -} diff --git a/cmd/kubectl/BUILD b/cmd/kubectl/BUILD deleted file mode 100644 index e386c96032719..0000000000000 --- a/cmd/kubectl/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kubectl", - embed = [":go_default_library"], - pure = "on", - visibility = ["//visibility:public"], - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["kubectl.go"], - importpath = "k8s.io/kubernetes/cmd/kubectl", - visibility = ["//visibility:private"], - deps = [ - "//staging/src/k8s.io/client-go/plugin/pkg/client/auth:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/cmd:go_default_library", - "//staging/src/k8s.io/kubectl/pkg/util/logs:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = [ - "//build/visible_to:cmd_kubectl_CONSUMERS", - ], -) diff --git a/cmd/kubectl/OWNERS b/cmd/kubectl/OWNERS deleted file mode 100644 index a1f6b7eee3089..0000000000000 --- a/cmd/kubectl/OWNERS +++ /dev/null @@ -1,9 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- sig-cli-maintainers -reviewers: -- sig-cli -labels: -- area/kubectl -- sig/cli diff --git a/cmd/kubectl/kubectl.go b/cmd/kubectl/kubectl.go deleted file mode 100644 index a5e24493a4100..0000000000000 --- a/cmd/kubectl/kubectl.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - goflag "flag" - "math/rand" - "os" - "time" - - "github.com/spf13/pflag" - - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/kubectl/pkg/cmd" - "k8s.io/kubectl/pkg/util/logs" - - // Import to initialize client auth plugins. - _ "k8s.io/client-go/plugin/pkg/client/auth" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := cmd.NewDefaultKubectlCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - // cliflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/kubelet/BUILD b/cmd/kubelet/BUILD deleted file mode 100644 index b98345e9a60f0..0000000000000 --- a/cmd/kubelet/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") -load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") - -go_binary( - name = "kubelet", - embed = [":go_default_library"], - x_defs = version_x_defs(), -) - -go_library( - name = "go_default_library", - srcs = ["kubelet.go"], - importpath = "k8s.io/kubernetes/cmd/kubelet", - deps = [ - "//cmd/kubelet/app:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/restclient:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubelet/app:all-srcs", - ], - tags = ["automanaged"], -) diff --git a/cmd/kubelet/OWNERS b/cmd/kubelet/OWNERS deleted file mode 100644 index 0dd725e2937f8..0000000000000 --- a/cmd/kubelet/OWNERS +++ /dev/null @@ -1,11 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- sig-node-approvers # see https://github.com/kubernetes/kubernetes/blob/master/OWNERS_ALIASES#LC220:~:text=sig%2Dnode%2Dapprovers - -reviewers: -- sig-node-reviewers # see https://github.com/kubernetes/kubernetes/blob/master/OWNERS_ALIASES#LC220:~:text=sig%2Dnode%2Dreviewers - -labels: -- area/kubelet -- sig/node diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD deleted file mode 100644 index bfd511422e250..0000000000000 --- a/cmd/kubelet/app/BUILD +++ /dev/null @@ -1,179 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_library( - name = "go_default_library", - srcs = [ - "auth.go", - "init_others.go", - "init_windows.go", - "plugins.go", - "plugins_providers.go", - "server.go", - "server_linux.go", - "server_unsupported.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubelet/app", - visibility = ["//visibility:public"], - deps = [ - "//cmd/kubelet/app/options:go_default_library", - "//pkg/api/legacyscheme:go_default_library", - "//pkg/apis/core:go_default_library", - "//pkg/capabilities:go_default_library", - "//pkg/cloudprovider/providers:go_default_library", - "//pkg/credentialprovider:go_default_library", - "//pkg/credentialprovider/aws:go_default_library", - "//pkg/credentialprovider/azure:go_default_library", - "//pkg/credentialprovider/gcp:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubelet:go_default_library", - "//pkg/kubelet/apis/config:go_default_library", - "//pkg/kubelet/apis/config/scheme:go_default_library", - "//pkg/kubelet/apis/config/validation:go_default_library", - "//pkg/kubelet/cadvisor:go_default_library", - "//pkg/kubelet/certificate:go_default_library", - "//pkg/kubelet/certificate/bootstrap:go_default_library", - "//pkg/kubelet/cm:go_default_library", - "//pkg/kubelet/cm/cpuset:go_default_library", - "//pkg/kubelet/config:go_default_library", - "//pkg/kubelet/container:go_default_library", - "//pkg/kubelet/eviction:go_default_library", - "//pkg/kubelet/eviction/api:go_default_library", - "//pkg/kubelet/kubeletconfig:go_default_library", - "//pkg/kubelet/kubeletconfig/configfiles:go_default_library", - "//pkg/kubelet/metrics:go_default_library", - "//pkg/kubelet/server:go_default_library", - "//pkg/kubelet/stats/pidlimit:go_default_library", - "//pkg/kubelet/types:go_default_library", - "//pkg/util/filesystem:go_default_library", - "//pkg/util/flock:go_default_library", - "//pkg/util/node:go_default_library", - "//pkg/util/oom:go_default_library", - "//pkg/util/rlimit:go_default_library", - "//pkg/volume:go_default_library", - "//pkg/volume/awsebs:go_default_library", - "//pkg/volume/azure_file:go_default_library", - "//pkg/volume/azuredd:go_default_library", - "//pkg/volume/cephfs:go_default_library", - "//pkg/volume/cinder:go_default_library", - "//pkg/volume/configmap:go_default_library", - "//pkg/volume/csi:go_default_library", - "//pkg/volume/csimigration:go_default_library", - "//pkg/volume/downwardapi:go_default_library", - "//pkg/volume/emptydir:go_default_library", - "//pkg/volume/fc:go_default_library", - "//pkg/volume/flexvolume:go_default_library", - "//pkg/volume/flocker:go_default_library", - "//pkg/volume/gcepd:go_default_library", - "//pkg/volume/git_repo:go_default_library", - "//pkg/volume/glusterfs:go_default_library", - "//pkg/volume/hostpath:go_default_library", - "//pkg/volume/iscsi:go_default_library", - "//pkg/volume/local:go_default_library", - "//pkg/volume/nfs:go_default_library", - "//pkg/volume/portworx:go_default_library", - "//pkg/volume/projected:go_default_library", - "//pkg/volume/quobyte:go_default_library", - "//pkg/volume/rbd:go_default_library", - "//pkg/volume/scaleio:go_default_library", - "//pkg/volume/secret:go_default_library", - "//pkg/volume/storageos:go_default_library", - "//pkg/volume/util/hostutil:go_default_library", - "//pkg/volume/util/subpath:go_default_library", - "//pkg/volume/vsphere_volume:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authentication/authenticatorfactory:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/authorization/authorizerfactory:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/dynamiccertificates:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/healthz:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - "//staging/src/k8s.io/client-go/util/certificate:go_default_library", - "//staging/src/k8s.io/client-go/util/connrotation:go_default_library", - "//staging/src/k8s.io/client-go/util/keyutil:go_default_library", - "//staging/src/k8s.io/cloud-provider:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/configz:go_default_library", - "//staging/src/k8s.io/component-base/featuregate:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics:go_default_library", - "//staging/src/k8s.io/component-base/metrics/legacyregistry:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library", - "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - "//staging/src/k8s.io/mount-utils:go_default_library", - "//vendor/github.com/coreos/go-systemd/daemon:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:android": [ - "//vendor/k8s.io/utils/inotify:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/k8s.io/utils/inotify:go_default_library", - ], - "@io_bazel_rules_go//go/platform:windows": [ - "//pkg/windows/service:go_default_library", - "//vendor/golang.org/x/sys/windows:go_default_library", - ], - "//conditions:default": [], - }), -) - -go_test( - name = "go_default_test", - srcs = [ - "init_windows_test.go", - "server_bootstrap_test.go", - "server_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//pkg/apis/certificates/v1:go_default_library", - "//pkg/controller/certificates/authority:go_default_library", - "//staging/src/k8s.io/api/certificates/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/util/cert:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kubelet/app/options:all-srcs", - ], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/kubelet/app/OWNERS b/cmd/kubelet/app/OWNERS deleted file mode 100644 index d7211290337b6..0000000000000 --- a/cmd/kubelet/app/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: -- sig-node-reviewers diff --git a/cmd/kubelet/app/auth.go b/cmd/kubelet/app/auth.go deleted file mode 100644 index 76b35f5db98db..0000000000000 --- a/cmd/kubelet/app/auth.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "errors" - "fmt" - "reflect" - - "k8s.io/apimachinery/pkg/types" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authentication/authenticatorfactory" - "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/authorization/authorizerfactory" - "k8s.io/apiserver/pkg/server/dynamiccertificates" - genericoptions "k8s.io/apiserver/pkg/server/options" - clientset "k8s.io/client-go/kubernetes" - authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1" - authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1" - - kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" - "k8s.io/kubernetes/pkg/kubelet/server" -) - -// BuildAuth creates an authenticator, an authorizer, and a matching authorizer attributes getter compatible with the kubelet's needs -// It returns AuthInterface, a run method to start internal controllers (like cert reloading) and error. -func BuildAuth(nodeName types.NodeName, client clientset.Interface, config kubeletconfig.KubeletConfiguration) (server.AuthInterface, func(<-chan struct{}), error) { - // Get clients, if provided - var ( - tokenClient authenticationclient.TokenReviewInterface - sarClient authorizationclient.SubjectAccessReviewInterface - ) - if client != nil && !reflect.ValueOf(client).IsNil() { - tokenClient = client.AuthenticationV1().TokenReviews() - sarClient = client.AuthorizationV1().SubjectAccessReviews() - } - - authenticator, runAuthenticatorCAReload, err := BuildAuthn(tokenClient, config.Authentication) - if err != nil { - return nil, nil, err - } - - attributes := server.NewNodeAuthorizerAttributesGetter(nodeName) - - authorizer, err := BuildAuthz(sarClient, config.Authorization) - if err != nil { - return nil, nil, err - } - - return server.NewKubeletAuth(authenticator, attributes, authorizer), runAuthenticatorCAReload, nil -} - -// BuildAuthn creates an authenticator compatible with the kubelet's needs -func BuildAuthn(client authenticationclient.TokenReviewInterface, authn kubeletconfig.KubeletAuthentication) (authenticator.Request, func(<-chan struct{}), error) { - var dynamicCAContentFromFile *dynamiccertificates.DynamicFileCAContent - var err error - if len(authn.X509.ClientCAFile) > 0 { - dynamicCAContentFromFile, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", authn.X509.ClientCAFile) - if err != nil { - return nil, nil, err - } - } - - authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ - Anonymous: authn.Anonymous.Enabled, - CacheTTL: authn.Webhook.CacheTTL.Duration, - ClientCertificateCAContentProvider: dynamicCAContentFromFile, - } - - if authn.Webhook.Enabled { - if client == nil { - return nil, nil, errors.New("no client provided, cannot use webhook authentication") - } - authenticatorConfig.WebhookRetryBackoff = genericoptions.DefaultAuthWebhookRetryBackoff() - authenticatorConfig.TokenAccessReviewClient = client - } - - authenticator, _, err := authenticatorConfig.New() - if err != nil { - return nil, nil, err - } - - return authenticator, func(stopCh <-chan struct{}) { - if dynamicCAContentFromFile != nil { - go dynamicCAContentFromFile.Run(1, stopCh) - } - }, err -} - -// BuildAuthz creates an authorizer compatible with the kubelet's needs -func BuildAuthz(client authorizationclient.SubjectAccessReviewInterface, authz kubeletconfig.KubeletAuthorization) (authorizer.Authorizer, error) { - switch authz.Mode { - case kubeletconfig.KubeletAuthorizationModeAlwaysAllow: - return authorizerfactory.NewAlwaysAllowAuthorizer(), nil - - case kubeletconfig.KubeletAuthorizationModeWebhook: - if client == nil { - return nil, errors.New("no client provided, cannot use webhook authorization") - } - authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ - SubjectAccessReviewClient: client, - AllowCacheTTL: authz.Webhook.CacheAuthorizedTTL.Duration, - DenyCacheTTL: authz.Webhook.CacheUnauthorizedTTL.Duration, - WebhookRetryBackoff: genericoptions.DefaultAuthWebhookRetryBackoff(), - } - return authorizerConfig.New() - - case "": - return nil, fmt.Errorf("no authorization mode specified") - - default: - return nil, fmt.Errorf("unknown authorization mode %s", authz.Mode) - - } -} diff --git a/cmd/kubelet/app/init_others.go b/cmd/kubelet/app/init_others.go deleted file mode 100644 index 1fe895f5d0ac4..0000000000000 --- a/cmd/kubelet/app/init_others.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build !windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -func initForOS(service bool, priorityClass string) error { - return nil -} diff --git a/cmd/kubelet/app/init_windows.go b/cmd/kubelet/app/init_windows.go deleted file mode 100644 index bed76806701f5..0000000000000 --- a/cmd/kubelet/app/init_windows.go +++ /dev/null @@ -1,64 +0,0 @@ -// +build windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "golang.org/x/sys/windows" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/windows/service" -) - -const ( - serviceName = "kubelet" -) - -// getPriorityValue returns the value associated with a Windows process priorityClass -// Ref: https://docs.microsoft.com/en-us/windows/win32/cimwin32prov/setpriority-method-in-class-win32-process -func getPriorityValue(priorityClassName string) uint32 { - var priorityClassMap = map[string]uint32{ - "IDLE_PRIORITY_CLASS": uint32(64), - "BELOW_NORMAL_PRIORITY_CLASS": uint32(16384), - "NORMAL_PRIORITY_CLASS": uint32(32), - "ABOVE_NORMAL_PRIORITY_CLASS": uint32(32768), - "HIGH_PRIORITY_CLASS": uint32(128), - "REALTIME_PRIORITY_CLASS": uint32(256), - } - return priorityClassMap[priorityClassName] -} - -func initForOS(windowsService bool, windowsPriorityClass string) error { - priority := getPriorityValue(windowsPriorityClass) - if priority == 0 { - return fmt.Errorf("unknown priority class %s, valid ones are available at "+ - "https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities", windowsPriorityClass) - } - kubeletProcessHandle := windows.CurrentProcess() - // Set the priority of the kubelet process to given priority - klog.Infof("Setting the priority of kubelet process to %s", windowsPriorityClass) - if err := windows.SetPriorityClass(kubeletProcessHandle, priority); err != nil { - return err - } - - if windowsService { - return service.InitService(serviceName) - } - return nil -} diff --git a/cmd/kubelet/app/init_windows_test.go b/cmd/kubelet/app/init_windows_test.go deleted file mode 100644 index 2920e8c69c814..0000000000000 --- a/cmd/kubelet/app/init_windows_test.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import "testing" - -func TestIsValidPriorityClass(t *testing.T) { - testCases := []struct { - description string - priorityClassName string - expectedPriorityValue uint32 - }{ - { - description: "Invalid Priority Class", - priorityClassName: "myPriorityClass", - expectedPriorityValue: 0, - }, - { - description: "Valid Priority Class", - priorityClassName: "IDLE_PRIORITY_CLASS", - expectedPriorityValue: uint32(64), - }, - } - for _, test := range testCases { - actualPriorityValue := getPriorityValue(test.priorityClassName) - if test.expectedPriorityValue != actualPriorityValue { - t.Fatalf("unexpected error for %s", test.description) - } - } -} diff --git a/cmd/kubelet/app/options/BUILD b/cmd/kubelet/app/options/BUILD deleted file mode 100644 index 53932c63591d9..0000000000000 --- a/cmd/kubelet/app/options/BUILD +++ /dev/null @@ -1,90 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "container_runtime.go", - "globalflags.go", - "globalflags_linux.go", - "globalflags_other.go", - "globalflags_providers.go", - "options.go", - "osflags_others.go", - "osflags_windows.go", - ], - importpath = "k8s.io/kubernetes/cmd/kubelet/app/options", - deps = [ - "//pkg/apis/core:go_default_library", - "//pkg/cluster/ports:go_default_library", - "//pkg/credentialprovider/azure:go_default_library", - "//pkg/credentialprovider/gcp:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubelet/apis:go_default_library", - "//pkg/kubelet/apis/config:go_default_library", - "//pkg/kubelet/apis/config/scheme:go_default_library", - "//pkg/kubelet/apis/config/validation:go_default_library", - "//pkg/kubelet/config:go_default_library", - "//pkg/kubelet/types:go_default_library", - "//pkg/util/flag:go_default_library", - "//pkg/util/taints:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - ] + select({ - "@io_bazel_rules_go//go/platform:android": [ - "//vendor/github.com/google/cadvisor/container/common:go_default_library", - "//vendor/github.com/google/cadvisor/container/containerd:go_default_library", - "//vendor/github.com/google/cadvisor/container/docker:go_default_library", - "//vendor/github.com/google/cadvisor/container/raw:go_default_library", - "//vendor/github.com/google/cadvisor/machine:go_default_library", - "//vendor/github.com/google/cadvisor/manager:go_default_library", - "//vendor/github.com/google/cadvisor/storage:go_default_library", - ], - "@io_bazel_rules_go//go/platform:linux": [ - "//vendor/github.com/google/cadvisor/container/common:go_default_library", - "//vendor/github.com/google/cadvisor/container/containerd:go_default_library", - "//vendor/github.com/google/cadvisor/container/docker:go_default_library", - "//vendor/github.com/google/cadvisor/container/raw:go_default_library", - "//vendor/github.com/google/cadvisor/machine:go_default_library", - "//vendor/github.com/google/cadvisor/manager:go_default_library", - "//vendor/github.com/google/cadvisor/storage:go_default_library", - ], - "//conditions:default": [], - }), -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) - -go_test( - name = "go_default_test", - srcs = ["options_test.go"], - embed = [":go_default_library"], - deps = [ - "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) diff --git a/cmd/kubelet/app/options/container_runtime.go b/cmd/kubelet/app/options/container_runtime.go deleted file mode 100644 index b58953b88ef50..0000000000000 --- a/cmd/kubelet/app/options/container_runtime.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "runtime" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/pkg/kubelet/config" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" -) - -const ( - // When these values are updated, also update test/utils/image/manifest.go - defaultPodSandboxImageName = "k8s.gcr.io/pause" - defaultPodSandboxImageVersion = "3.2" -) - -var ( - defaultPodSandboxImage = defaultPodSandboxImageName + - ":" + defaultPodSandboxImageVersion -) - -// NewContainerRuntimeOptions will create a new ContainerRuntimeOptions with -// default values. -func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions { - dockerEndpoint := "" - if runtime.GOOS != "windows" { - dockerEndpoint = "unix:///var/run/docker.sock" - } - - return &config.ContainerRuntimeOptions{ - ContainerRuntime: kubetypes.DockerContainerRuntime, - DockerEndpoint: dockerEndpoint, - DockershimRootDirectory: "/var/lib/dockershim", - PodSandboxImage: defaultPodSandboxImage, - ImagePullProgressDeadline: metav1.Duration{Duration: 1 * time.Minute}, - - CNIBinDir: "/opt/cni/bin", - CNIConfDir: "/etc/cni/net.d", - CNICacheDir: "/var/lib/cni/cache", - } -} diff --git a/cmd/kubelet/app/options/globalflags.go b/cmd/kubelet/app/options/globalflags.go deleted file mode 100644 index fdc20d181779f..0000000000000 --- a/cmd/kubelet/app/options/globalflags.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "flag" - "fmt" - "os" - "strings" - - "github.com/spf13/pflag" - - // libs that provide registration functions - "k8s.io/component-base/logs" - "k8s.io/component-base/version/verflag" - "k8s.io/klog/v2" - - // ensure libs have a chance to globally register their flags - _ "k8s.io/kubernetes/pkg/credentialprovider/azure" - _ "k8s.io/kubernetes/pkg/credentialprovider/gcp" -) - -// AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register -// against the global flagsets from "flag" and "github.com/spf13/pflag". -// We do this in order to prevent unwanted flags from leaking into the Kubelet's flagset. -func AddGlobalFlags(fs *pflag.FlagSet) { - addKlogFlags(fs) - addCadvisorFlags(fs) - addCredentialProviderFlags(fs) - verflag.AddFlags(fs) - logs.AddFlags(fs) -} - -// normalize replaces underscores with hyphens -// we should always use hyphens instead of underscores when registering kubelet flags -func normalize(s string) string { - return strings.Replace(s, "_", "-", -1) -} - -// register adds a flag to local that targets the Value associated with the Flag named globalName in global -func register(global *flag.FlagSet, local *pflag.FlagSet, globalName string) { - if f := global.Lookup(globalName); f != nil { - pflagFlag := pflag.PFlagFromGoFlag(f) - pflagFlag.Name = normalize(pflagFlag.Name) - local.AddFlag(pflagFlag) - } else { - panic(fmt.Sprintf("failed to find flag in global flagset (flag): %s", globalName)) - } -} - -// pflagRegister adds a flag to local that targets the Value associated with the Flag named globalName in global -func pflagRegister(global, local *pflag.FlagSet, globalName string) { - if f := global.Lookup(globalName); f != nil { - f.Name = normalize(f.Name) - local.AddFlag(f) - } else { - panic(fmt.Sprintf("failed to find flag in global flagset (pflag): %s", globalName)) - } -} - -// registerDeprecated registers the flag with register, and then marks it deprecated -func registerDeprecated(global *flag.FlagSet, local *pflag.FlagSet, globalName, deprecated string) { - register(global, local, globalName) - local.Lookup(normalize(globalName)).Deprecated = deprecated -} - -// addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider -func addCredentialProviderFlags(fs *pflag.FlagSet) { - // lookup flags in global flag set and re-register the values with our flagset - global := pflag.CommandLine - local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) - - addLegacyCloudProviderCredentialProviderFlags(global, local) - - fs.AddFlagSet(local) -} - -// addKlogFlags adds flags from k8s.io/klog -func addKlogFlags(fs *pflag.FlagSet) { - local := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - klog.InitFlags(local) - fs.AddGoFlagSet(local) -} diff --git a/cmd/kubelet/app/options/globalflags_linux.go b/cmd/kubelet/app/options/globalflags_linux.go deleted file mode 100644 index 61c190f86f019..0000000000000 --- a/cmd/kubelet/app/options/globalflags_linux.go +++ /dev/null @@ -1,80 +0,0 @@ -// +build linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "flag" - "os" - - "github.com/spf13/pflag" - - // ensure libs have a chance to globally register their flags - _ "github.com/google/cadvisor/container/common" - _ "github.com/google/cadvisor/container/containerd" - _ "github.com/google/cadvisor/container/docker" - _ "github.com/google/cadvisor/container/raw" - _ "github.com/google/cadvisor/machine" - _ "github.com/google/cadvisor/manager" - _ "github.com/google/cadvisor/storage" -) - -// addCadvisorFlags adds flags from cadvisor -func addCadvisorFlags(fs *pflag.FlagSet) { - // lookup flags in global flag set and re-register the values with our flagset - global := flag.CommandLine - local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) - - // These flags were also implicit from cadvisor, but are actually used by something in the core repo: - // TODO(mtaufen): This one is stil used by our salt, but for heaven's sake it's even deprecated in cadvisor - register(global, local, "docker_root") - // e2e node tests rely on this - register(global, local, "housekeeping_interval") - - // These flags were implicit from cadvisor, and are mistakes that should be registered deprecated: - const deprecated = "This is a cadvisor flag that was mistakenly registered with the Kubelet. Due to legacy concerns, it will follow the standard CLI deprecation timeline before being removed." - - registerDeprecated(global, local, "application_metrics_count_limit", deprecated) - registerDeprecated(global, local, "boot_id_file", deprecated) - registerDeprecated(global, local, "container_hints", deprecated) - registerDeprecated(global, local, "containerd", deprecated) - registerDeprecated(global, local, "docker", deprecated) - registerDeprecated(global, local, "docker_env_metadata_whitelist", deprecated) - registerDeprecated(global, local, "docker_only", deprecated) - registerDeprecated(global, local, "docker-tls", deprecated) - registerDeprecated(global, local, "docker-tls-ca", deprecated) - registerDeprecated(global, local, "docker-tls-cert", deprecated) - registerDeprecated(global, local, "docker-tls-key", deprecated) - registerDeprecated(global, local, "enable_load_reader", deprecated) - registerDeprecated(global, local, "event_storage_age_limit", deprecated) - registerDeprecated(global, local, "event_storage_event_limit", deprecated) - registerDeprecated(global, local, "global_housekeeping_interval", deprecated) - registerDeprecated(global, local, "log_cadvisor_usage", deprecated) - registerDeprecated(global, local, "machine_id_file", deprecated) - registerDeprecated(global, local, "storage_driver_user", deprecated) - registerDeprecated(global, local, "storage_driver_password", deprecated) - registerDeprecated(global, local, "storage_driver_host", deprecated) - registerDeprecated(global, local, "storage_driver_db", deprecated) - registerDeprecated(global, local, "storage_driver_table", deprecated) - registerDeprecated(global, local, "storage_driver_secure", deprecated) - registerDeprecated(global, local, "storage_driver_buffer_duration", deprecated) - registerDeprecated(global, local, "containerd-namespace", deprecated) - - // finally, add cadvisor flags to the provided flagset - fs.AddFlagSet(local) -} diff --git a/cmd/kubelet/app/options/globalflags_other.go b/cmd/kubelet/app/options/globalflags_other.go deleted file mode 100644 index b4a04f9f40f86..0000000000000 --- a/cmd/kubelet/app/options/globalflags_other.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build !linux - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" -) - -func addCadvisorFlags(fs *pflag.FlagSet) { -} diff --git a/cmd/kubelet/app/options/globalflags_providerless.go b/cmd/kubelet/app/options/globalflags_providerless.go deleted file mode 100644 index 7fba07df799f4..0000000000000 --- a/cmd/kubelet/app/options/globalflags_providerless.go +++ /dev/null @@ -1,27 +0,0 @@ -// +build providerless - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" -) - -func addLegacyCloudProviderCredentialProviderFlags(global, local *pflag.FlagSet) { - // no-op when no legacy providers are compiled in -} diff --git a/cmd/kubelet/app/options/globalflags_providers.go b/cmd/kubelet/app/options/globalflags_providers.go deleted file mode 100644 index 5d2dd44d34542..0000000000000 --- a/cmd/kubelet/app/options/globalflags_providers.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build !providerless - -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" -) - -func addLegacyCloudProviderCredentialProviderFlags(global, local *pflag.FlagSet) { - // TODO(#58034): This is not a static file, so it's not quite as straightforward as --google-json-key. - // We need to figure out how ACR users can dynamically provide pull credentials before we can deprecate this. - pflagRegister(global, local, "azure-container-registry-config") -} diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go deleted file mode 100644 index 50dff501feca4..0000000000000 --- a/cmd/kubelet/app/options/options.go +++ /dev/null @@ -1,553 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package options contains all of the primary arguments for a kubelet. -package options - -import ( - "fmt" - _ "net/http/pprof" // Enable pprof HTTP handlers. - "path/filepath" - "runtime" - "strings" - - "github.com/spf13/pflag" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - utilfeature "k8s.io/apiserver/pkg/util/feature" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/kubelet/config/v1beta1" - "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/cluster/ports" - "k8s.io/kubernetes/pkg/features" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" - kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" - kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" - kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation" - "k8s.io/kubernetes/pkg/kubelet/config" - utilflag "k8s.io/kubernetes/pkg/util/flag" - utiltaints "k8s.io/kubernetes/pkg/util/taints" -) - -const defaultRootDir = "/var/lib/kubelet" - -// KubeletFlags contains configuration flags for the Kubelet. -// A configuration field should go in KubeletFlags instead of KubeletConfiguration if any of these are true: -// - its value will never, or cannot safely be changed during the lifetime of a node, or -// - its value cannot be safely shared between nodes at the same time (e.g. a hostname); -// KubeletConfiguration is intended to be shared between nodes. -// In general, please try to avoid adding flags or configuration fields, -// we already have a confusingly large amount of them. -type KubeletFlags struct { - KubeConfig string - BootstrapKubeconfig string - - // Insert a probability of random errors during calls to the master. - ChaosChance float64 - // Crash immediately, rather than eating panics. - ReallyCrashForTesting bool - - // HostnameOverride is the hostname used to identify the kubelet instead - // of the actual hostname. - HostnameOverride string - // NodeIP is IP address of the node. - // If set, kubelet will use this IP address for the node. - NodeIP string - - // Container-runtime-specific options. - config.ContainerRuntimeOptions - - // certDirectory is the directory where the TLS certs are located. - // If tlsCertFile and tlsPrivateKeyFile are provided, this flag will be ignored. - CertDirectory string - - // cloudProvider is the provider for cloud services. - // +optional - CloudProvider string - - // cloudConfigFile is the path to the cloud provider configuration file. - // +optional - CloudConfigFile string - - // rootDirectory is the directory path to place kubelet files (volume - // mounts,etc). - RootDirectory string - - // The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. - // The Kubelet will create this directory if it does not already exist. - // The path may be absolute or relative; relative paths are under the Kubelet's current working directory. - // Providing this flag enables dynamic kubelet configuration. - // To use this flag, the DynamicKubeletConfig feature gate must be enabled. - DynamicConfigDir cliflag.StringFlag - - // The Kubelet will load its initial configuration from this file. - // The path may be absolute or relative; relative paths are under the Kubelet's current working directory. - // Omit this flag to use the combination of built-in default configuration values and flags. - KubeletConfigFile string - - // registerNode enables automatic registration with the apiserver. - RegisterNode bool - - // registerWithTaints are an array of taints to add to a node object when - // the kubelet registers itself. This only takes effect when registerNode - // is true and upon the initial registration of the node. - RegisterWithTaints []core.Taint - - // WindowsService should be set to true if kubelet is running as a service on Windows. - // Its corresponding flag only gets registered in Windows builds. - WindowsService bool - - // WindowsPriorityClass sets the priority class associated with the Kubelet process - // Its corresponding flag only gets registered in Windows builds - // The default priority class associated with any process in Windows is NORMAL_PRIORITY_CLASS. Keeping it as is - // to maintain backwards compatibility. - // Source: https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities - WindowsPriorityClass string - - // remoteRuntimeEndpoint is the endpoint of remote runtime service - RemoteRuntimeEndpoint string - // remoteImageEndpoint is the endpoint of remote image service - RemoteImageEndpoint string - // experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path - ExperimentalMounterPath string - // This flag, if set, enables a check prior to mount operations to verify that the required components - // (binaries, etc.) to mount the volume are available on the underlying node. If the check is enabled - // and fails the mount operation fails. - ExperimentalCheckNodeCapabilitiesBeforeMount bool - // This flag, if set, will avoid including `EvictionHard` limits while computing Node Allocatable. - // Refer to [Node Allocatable](https://git.k8s.io/community/contributors/design-proposals/node/node-allocatable.md) doc for more information. - ExperimentalNodeAllocatableIgnoreEvictionThreshold bool - // Node Labels are the node labels to add when registering the node in the cluster - NodeLabels map[string]string - // lockFilePath is the path that kubelet will use to as a lock file. - // It uses this file as a lock to synchronize with other kubelet processes - // that may be running. - LockFilePath string - // ExitOnLockContention is a flag that signifies to the kubelet that it is running - // in "bootstrap" mode. This requires that 'LockFilePath' has been set. - // This will cause the kubelet to listen to inotify events on the lock file, - // releasing it and exiting when another process tries to open that file. - ExitOnLockContention bool - // seccompProfileRoot is the directory path for seccomp profiles. - SeccompProfileRoot string - - // DEPRECATED FLAGS - // minimumGCAge is the minimum age for a finished container before it is - // garbage collected. - MinimumGCAge metav1.Duration - // maxPerPodContainerCount is the maximum number of old instances to - // retain per container. Each container takes up some disk space. - MaxPerPodContainerCount int32 - // maxContainerCount is the maximum number of old instances of containers - // to retain globally. Each container takes up some disk space. - MaxContainerCount int32 - // masterServiceNamespace is The namespace from which the kubernetes - // master services should be injected into pods. - MasterServiceNamespace string - // registerSchedulable tells the kubelet to register the node as - // schedulable. Won't have any effect if register-node is false. - // DEPRECATED: use registerWithTaints instead - RegisterSchedulable bool - // nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade. - NonMasqueradeCIDR string - // This flag, if set, instructs the kubelet to keep volumes from terminated pods mounted to the node. - // This can be useful for debugging volume related issues. - KeepTerminatedPodVolumes bool - // EnableCAdvisorJSONEndpoints enables some cAdvisor endpoints that will be removed in future versions - EnableCAdvisorJSONEndpoints bool -} - -// NewKubeletFlags will create a new KubeletFlags with default values -func NewKubeletFlags() *KubeletFlags { - remoteRuntimeEndpoint := "" - if runtime.GOOS == "linux" { - remoteRuntimeEndpoint = "unix:///var/run/dockershim.sock" - } else if runtime.GOOS == "windows" { - remoteRuntimeEndpoint = "npipe:////./pipe/dockershim" - } - - return &KubeletFlags{ - ContainerRuntimeOptions: *NewContainerRuntimeOptions(), - CertDirectory: "/var/lib/kubelet/pki", - RootDirectory: defaultRootDir, - MasterServiceNamespace: metav1.NamespaceDefault, - MaxContainerCount: -1, - MaxPerPodContainerCount: 1, - MinimumGCAge: metav1.Duration{Duration: 0}, - NonMasqueradeCIDR: "10.0.0.0/8", - RegisterSchedulable: true, - RemoteRuntimeEndpoint: remoteRuntimeEndpoint, - NodeLabels: make(map[string]string), - RegisterNode: true, - SeccompProfileRoot: filepath.Join(defaultRootDir, "seccomp"), - // prior to the introduction of this flag, there was a hardcoded cap of 50 images - EnableCAdvisorJSONEndpoints: false, - } -} - -// ValidateKubeletFlags validates Kubelet's configuration flags and returns an error if they are invalid. -func ValidateKubeletFlags(f *KubeletFlags) error { - // ensure that nobody sets DynamicConfigDir if the dynamic config feature gate is turned off - if f.DynamicConfigDir.Provided() && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { - return fmt.Errorf("the DynamicKubeletConfig feature gate must be enabled in order to use the --dynamic-config-dir flag") - } - - unknownLabels := sets.NewString() - for k := range f.NodeLabels { - if isKubernetesLabel(k) && !kubeletapis.IsKubeletLabel(k) { - unknownLabels.Insert(k) - } - } - if len(unknownLabels) > 0 { - return fmt.Errorf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v\n--node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", unknownLabels.List(), strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")) - } - - return nil -} - -func isKubernetesLabel(key string) bool { - namespace := getLabelNamespace(key) - if namespace == "kubernetes.io" || strings.HasSuffix(namespace, ".kubernetes.io") { - return true - } - if namespace == "k8s.io" || strings.HasSuffix(namespace, ".k8s.io") { - return true - } - return false -} - -func getLabelNamespace(key string) string { - if parts := strings.SplitN(key, "/", 2); len(parts) == 2 { - return parts[0] - } - return "" -} - -// NewKubeletConfiguration will create a new KubeletConfiguration with default values -func NewKubeletConfiguration() (*kubeletconfig.KubeletConfiguration, error) { - scheme, _, err := kubeletscheme.NewSchemeAndCodecs() - if err != nil { - return nil, err - } - versioned := &v1beta1.KubeletConfiguration{} - scheme.Default(versioned) - config := &kubeletconfig.KubeletConfiguration{} - if err := scheme.Convert(versioned, config, nil); err != nil { - return nil, err - } - applyLegacyDefaults(config) - return config, nil -} - -// applyLegacyDefaults applies legacy default values to the KubeletConfiguration in order to -// preserve the command line API. This is used to construct the baseline default KubeletConfiguration -// before the first round of flag parsing. -func applyLegacyDefaults(kc *kubeletconfig.KubeletConfiguration) { - // --anonymous-auth - kc.Authentication.Anonymous.Enabled = true - // --authentication-token-webhook - kc.Authentication.Webhook.Enabled = false - // --authorization-mode - kc.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow - // --read-only-port - kc.ReadOnlyPort = ports.KubeletReadOnlyPort -} - -// KubeletServer encapsulates all of the parameters necessary for starting up -// a kubelet. These can either be set via command line or directly. -type KubeletServer struct { - KubeletFlags - kubeletconfig.KubeletConfiguration -} - -// NewKubeletServer will create a new KubeletServer with default values. -func NewKubeletServer() (*KubeletServer, error) { - config, err := NewKubeletConfiguration() - if err != nil { - return nil, err - } - return &KubeletServer{ - KubeletFlags: *NewKubeletFlags(), - KubeletConfiguration: *config, - }, nil -} - -// ValidateKubeletServer validates configuration of KubeletServer and returns an error if the input configuration is invalid. -func ValidateKubeletServer(s *KubeletServer) error { - // please add any KubeletConfiguration validation to the kubeletconfigvalidation.ValidateKubeletConfiguration function - if err := kubeletconfigvalidation.ValidateKubeletConfiguration(&s.KubeletConfiguration); err != nil { - return err - } - if err := ValidateKubeletFlags(&s.KubeletFlags); err != nil { - return err - } - return nil -} - -// AddFlags adds flags for a specific KubeletServer to the specified FlagSet -func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { - s.KubeletFlags.AddFlags(fs) - AddKubeletConfigFlags(fs, &s.KubeletConfiguration) -} - -// AddFlags adds flags for a specific KubeletFlags to the specified FlagSet -func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) { - fs := pflag.NewFlagSet("", pflag.ExitOnError) - defer func() { - // Unhide deprecated flags. We want deprecated flags to show in Kubelet help. - // We have some hidden flags, but we might as well unhide these when they are deprecated, - // as silently deprecating and removing (even hidden) things is unkind to people who use them. - fs.VisitAll(func(f *pflag.Flag) { - if len(f.Deprecated) > 0 { - f.Hidden = false - } - }) - mainfs.AddFlagSet(fs) - }() - - f.ContainerRuntimeOptions.AddFlags(fs) - f.addOSFlags(fs) - - fs.StringVar(&f.KubeletConfigFile, "config", f.KubeletConfigFile, "The Kubelet will load its initial configuration from this file. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Omit this flag to use the built-in default configuration values. Command-line flags override configuration from this file.") - fs.StringVar(&f.KubeConfig, "kubeconfig", f.KubeConfig, "Path to a kubeconfig file, specifying how to connect to the API server. Providing --kubeconfig enables API server mode, omitting --kubeconfig enables standalone mode.") - - fs.StringVar(&f.BootstrapKubeconfig, "bootstrap-kubeconfig", f.BootstrapKubeconfig, "Path to a kubeconfig file that will be used to get client certificate for kubelet. "+ - "If the file specified by --kubeconfig does not exist, the bootstrap kubeconfig is used to request a client certificate from the API server. "+ - "On success, a kubeconfig file referencing the generated client certificate and key is written to the path specified by --kubeconfig. "+ - "The client certificate and key file will be stored in the directory pointed by --cert-dir.") - - fs.StringVar(&f.HostnameOverride, "hostname-override", f.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname. If --cloud-provider is set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).") - - fs.StringVar(&f.NodeIP, "node-ip", f.NodeIP, "IP address (or comma-separated dual-stack IP addresses) of the node. If unset, kubelet will use the node's default IPv4 address, if any, or its default IPv6 address if it has no IPv4 addresses. You can pass '::' to make it prefer the default IPv6 address rather than the default IPv4 address.") - - fs.StringVar(&f.CertDirectory, "cert-dir", f.CertDirectory, "The directory where the TLS certs are located. "+ - "If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.") - - fs.StringVar(&f.RootDirectory, "root-dir", f.RootDirectory, "Directory path for managing kubelet files (volume mounts,etc).") - - fs.Var(&f.DynamicConfigDir, "dynamic-config-dir", "The Kubelet will use this directory for checkpointing downloaded configurations and tracking configuration health. The Kubelet will create this directory if it does not already exist. The path may be absolute or relative; relative paths start at the Kubelet's current working directory. Providing this flag enables dynamic Kubelet configuration. The DynamicKubeletConfig feature gate must be enabled to pass this flag; this gate currently defaults to true because the feature is beta.") - - fs.BoolVar(&f.RegisterNode, "register-node", f.RegisterNode, "Register the node with the apiserver. If --kubeconfig is not provided, this flag is irrelevant, as the Kubelet won't have an apiserver to register with.") - fs.Var(utiltaints.NewTaintsVar(&f.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") - - // EXPERIMENTAL FLAGS - fs.StringVar(&f.RemoteRuntimeEndpoint, "container-runtime-endpoint", f.RemoteRuntimeEndpoint, "[Experimental] The endpoint of remote runtime service. Currently unix socket endpoint is supported on Linux, while npipe and tcp endpoints are supported on windows. Examples:'unix:///var/run/dockershim.sock', 'npipe:////./pipe/dockershim'") - fs.StringVar(&f.RemoteImageEndpoint, "image-service-endpoint", f.RemoteImageEndpoint, "[Experimental] The endpoint of remote image service. If not specified, it will be the same with container-runtime-endpoint by default. Currently unix socket endpoint is supported on Linux, while npipe and tcp endpoints are supported on windows. Examples:'unix:///var/run/dockershim.sock', 'npipe:////./pipe/dockershim'") - bindableNodeLabels := cliflag.ConfigurationMap(f.NodeLabels) - fs.Var(&bindableNodeLabels, "node-labels", fmt.Sprintf(" Labels to add when registering the node in the cluster. Labels must be key=value pairs separated by ','. Labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", "))) - fs.StringVar(&f.LockFilePath, "lock-file", f.LockFilePath, " The path to file for kubelet to use as a lock file.") - fs.BoolVar(&f.ExitOnLockContention, "exit-on-lock-contention", f.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.") - - // DEPRECATED FLAGS - fs.StringVar(&f.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", f.BootstrapKubeconfig, "") - fs.MarkDeprecated("experimental-bootstrap-kubeconfig", "Use --bootstrap-kubeconfig") - fs.DurationVar(&f.MinimumGCAge.Duration, "minimum-container-ttl-duration", f.MinimumGCAge.Duration, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'") - fs.MarkDeprecated("minimum-container-ttl-duration", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.") - fs.Int32Var(&f.MaxPerPodContainerCount, "maximum-dead-containers-per-container", f.MaxPerPodContainerCount, "Maximum number of old instances to retain per container. Each container takes up some disk space.") - fs.MarkDeprecated("maximum-dead-containers-per-container", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.") - fs.Int32Var(&f.MaxContainerCount, "maximum-dead-containers", f.MaxContainerCount, "Maximum number of old instances of containers to retain globally. Each container takes up some disk space. To disable, set to a negative number.") - fs.MarkDeprecated("maximum-dead-containers", "Use --eviction-hard or --eviction-soft instead. Will be removed in a future version.") - fs.StringVar(&f.MasterServiceNamespace, "master-service-namespace", f.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods") - fs.MarkDeprecated("master-service-namespace", "This flag will be removed in a future version.") - fs.BoolVar(&f.RegisterSchedulable, "register-schedulable", f.RegisterSchedulable, "Register the node as schedulable. Won't have any effect if register-node is false.") - fs.MarkDeprecated("register-schedulable", "will be removed in a future version") - fs.StringVar(&f.NonMasqueradeCIDR, "non-masquerade-cidr", f.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade. Set to '0.0.0.0/0' to never masquerade.") - fs.MarkDeprecated("non-masquerade-cidr", "will be removed in a future version") - fs.BoolVar(&f.KeepTerminatedPodVolumes, "keep-terminated-pod-volumes", f.KeepTerminatedPodVolumes, "Keep terminated pod volumes mounted to the node after the pod terminates. Can be useful for debugging volume related issues.") - fs.MarkDeprecated("keep-terminated-pod-volumes", "will be removed in a future version") - fs.BoolVar(&f.EnableCAdvisorJSONEndpoints, "enable-cadvisor-json-endpoints", f.EnableCAdvisorJSONEndpoints, "Enable cAdvisor json /spec and /stats/* endpoints. This flag has no effect on the /stats/summary endpoint. [default=false]") - // TODO: Remove this flag in 1.20+. https://github.com/kubernetes/kubernetes/issues/68522 - fs.MarkDeprecated("enable-cadvisor-json-endpoints", "will be removed in a future version") - fs.BoolVar(&f.ReallyCrashForTesting, "really-crash-for-testing", f.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") - fs.MarkDeprecated("really-crash-for-testing", "will be removed in a future version.") - fs.Float64Var(&f.ChaosChance, "chaos-chance", f.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing.") - fs.MarkDeprecated("chaos-chance", "will be removed in a future version.") - fs.StringVar(&f.SeccompProfileRoot, "seccomp-profile-root", f.SeccompProfileRoot, " Directory path for seccomp profiles.") - fs.MarkDeprecated("seccomp-profile-root", "will be removed in 1.23, in favor of using the `/seccomp` directory") - fs.StringVar(&f.ExperimentalMounterPath, "experimental-mounter-path", f.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.") - fs.MarkDeprecated("experimental-mounter-path", "will be removed in 1.23. in favor of using CSI.") - fs.BoolVar(&f.ExperimentalCheckNodeCapabilitiesBeforeMount, "experimental-check-node-capabilities-before-mount", f.ExperimentalCheckNodeCapabilitiesBeforeMount, "[Experimental] if set true, the kubelet will check the underlying node for required components (binaries, etc.) before performing the mount") - fs.MarkDeprecated("experimental-check-node-capabilities-before-mount", "will be removed in 1.23. in favor of using CSI.") - fs.StringVar(&f.CloudProvider, "cloud-provider", f.CloudProvider, "The provider for cloud services. Set to empty string for running with no cloud provider. If set, the cloud provider determines the name of the node (consult cloud provider documentation to determine if and how the hostname is used).") - fs.MarkDeprecated("cloud-provider", "will be removed in 1.23, in favor of removing cloud provider code from Kubelet.") - fs.StringVar(&f.CloudConfigFile, "cloud-config", f.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") - fs.MarkDeprecated("cloud-config", "will be removed in 1.23, in favor of removing cloud provider code from Kubelet.") - fs.BoolVar(&f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "experimental-allocatable-ignore-eviction", f.ExperimentalNodeAllocatableIgnoreEvictionThreshold, "When set to 'true', Hard Eviction Thresholds will be ignored while calculating Node Allocatable. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details. [default=false]") - fs.MarkDeprecated("experimental-allocatable-ignore-eviction", "will be removed in 1.23.") -} - -// AddKubeletConfigFlags adds flags for a specific kubeletconfig.KubeletConfiguration to the specified FlagSet -func AddKubeletConfigFlags(mainfs *pflag.FlagSet, c *kubeletconfig.KubeletConfiguration) { - fs := pflag.NewFlagSet("", pflag.ExitOnError) - defer func() { - // All KubeletConfiguration flags are now deprecated, and any new flags that point to - // KubeletConfiguration fields are deprecated-on-creation. When removing flags at the end - // of their deprecation period, be careful to check that they have *actually* been deprecated - // members of the KubeletConfiguration for the entire deprecation period: - // e.g. if a flag was added after this deprecation function, it may not be at the end - // of its lifetime yet, even if the rest are. - deprecated := "This parameter should be set via the config file specified by the Kubelet's --config flag. See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/ for more information." - fs.VisitAll(func(f *pflag.Flag) { - f.Deprecated = deprecated - }) - mainfs.AddFlagSet(fs) - }() - - fs.BoolVar(&c.EnableServer, "enable-server", c.EnableServer, "Enable the Kubelet's server") - - fs.BoolVar(&c.FailSwapOn, "fail-swap-on", c.FailSwapOn, "Makes the Kubelet fail to start if swap is enabled on the node. ") - fs.StringVar(&c.StaticPodPath, "pod-manifest-path", c.StaticPodPath, "Path to the directory containing static pod files to run, or the path to a single static pod file. Files starting with dots will be ignored.") - fs.DurationVar(&c.SyncFrequency.Duration, "sync-frequency", c.SyncFrequency.Duration, "Max period between synchronizing running containers and config") - fs.DurationVar(&c.FileCheckFrequency.Duration, "file-check-frequency", c.FileCheckFrequency.Duration, "Duration between checking config files for new data") - fs.DurationVar(&c.HTTPCheckFrequency.Duration, "http-check-frequency", c.HTTPCheckFrequency.Duration, "Duration between checking http for new data") - fs.StringVar(&c.StaticPodURL, "manifest-url", c.StaticPodURL, "URL for accessing additional Pod specifications to run") - fs.Var(cliflag.NewColonSeparatedMultimapStringString(&c.StaticPodURLHeader), "manifest-url-header", "Comma-separated list of HTTP headers to use when accessing the url provided to --manifest-url. Multiple headers with the same name will be added in the same order provided. This flag can be repeatedly invoked. For example: --manifest-url-header 'a:hello,b:again,c:world' --manifest-url-header 'b:beautiful'") - fs.Var(utilflag.IPVar{Val: &c.Address}, "address", "The IP address for the Kubelet to serve on (set to '0.0.0.0' or '::' for listening in all interfaces and IP families)") - fs.Int32Var(&c.Port, "port", c.Port, "The port for the Kubelet to serve on.") - fs.Int32Var(&c.ReadOnlyPort, "read-only-port", c.ReadOnlyPort, "The read-only port for the Kubelet to serve on with no authentication/authorization (set to 0 to disable)") - - // Authentication - fs.BoolVar(&c.Authentication.Anonymous.Enabled, "anonymous-auth", c.Authentication.Anonymous.Enabled, ""+ - "Enables anonymous requests to the Kubelet server. Requests that are not rejected by another "+ - "authentication method are treated as anonymous requests. Anonymous requests have a username "+ - "of system:anonymous, and a group name of system:unauthenticated.") - fs.BoolVar(&c.Authentication.Webhook.Enabled, "authentication-token-webhook", c.Authentication.Webhook.Enabled, ""+ - "Use the TokenReview API to determine authentication for bearer tokens.") - fs.DurationVar(&c.Authentication.Webhook.CacheTTL.Duration, "authentication-token-webhook-cache-ttl", c.Authentication.Webhook.CacheTTL.Duration, ""+ - "The duration to cache responses from the webhook token authenticator.") - fs.StringVar(&c.Authentication.X509.ClientCAFile, "client-ca-file", c.Authentication.X509.ClientCAFile, ""+ - "If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file "+ - "is authenticated with an identity corresponding to the CommonName of the client certificate.") - - // Authorization - fs.StringVar((*string)(&c.Authorization.Mode), "authorization-mode", string(c.Authorization.Mode), ""+ - "Authorization mode for Kubelet server. Valid options are AlwaysAllow or Webhook. "+ - "Webhook mode uses the SubjectAccessReview API to determine authorization.") - fs.DurationVar(&c.Authorization.Webhook.CacheAuthorizedTTL.Duration, "authorization-webhook-cache-authorized-ttl", c.Authorization.Webhook.CacheAuthorizedTTL.Duration, ""+ - "The duration to cache 'authorized' responses from the webhook authorizer.") - fs.DurationVar(&c.Authorization.Webhook.CacheUnauthorizedTTL.Duration, "authorization-webhook-cache-unauthorized-ttl", c.Authorization.Webhook.CacheUnauthorizedTTL.Duration, ""+ - "The duration to cache 'unauthorized' responses from the webhook authorizer.") - - fs.StringVar(&c.TLSCertFile, "tls-cert-file", c.TLSCertFile, ""+ - "File containing x509 Certificate used for serving HTTPS (with intermediate certs, if any, concatenated after server cert). "+ - "If --tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate and key "+ - "are generated for the public address and saved to the directory passed to --cert-dir.") - fs.StringVar(&c.TLSPrivateKeyFile, "tls-private-key-file", c.TLSPrivateKeyFile, "File containing x509 private key matching --tls-cert-file.") - fs.BoolVar(&c.ServerTLSBootstrap, "rotate-server-certificates", c.ServerTLSBootstrap, "Auto-request and rotate the kubelet serving certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches. Requires the RotateKubeletServerCertificate feature gate to be enabled, and approval of the submitted CertificateSigningRequest objects.") - - tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames() - tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames() - fs.StringSliceVar(&c.TLSCipherSuites, "tls-cipher-suites", c.TLSCipherSuites, - "Comma-separated list of cipher suites for the server. "+ - "If omitted, the default Go cipher suites will be used. \n"+ - "Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+ - "Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".") - tlsPossibleVersions := cliflag.TLSPossibleVersions() - fs.StringVar(&c.TLSMinVersion, "tls-min-version", c.TLSMinVersion, - "Minimum TLS version supported. "+ - "Possible values: "+strings.Join(tlsPossibleVersions, ", ")) - fs.BoolVar(&c.RotateCertificates, "rotate-certificates", c.RotateCertificates, " Auto rotate the kubelet client certificates by requesting new certificates from the kube-apiserver when the certificate expiration approaches.") - - fs.Int32Var(&c.RegistryPullQPS, "registry-qps", c.RegistryPullQPS, "If > 0, limit registry pull QPS to this value. If 0, unlimited.") - fs.Int32Var(&c.RegistryBurst, "registry-burst", c.RegistryBurst, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps. Only used if --registry-qps > 0") - fs.Int32Var(&c.EventRecordQPS, "event-qps", c.EventRecordQPS, "If > 0, limit event creations per second to this value. If 0, unlimited.") - fs.Int32Var(&c.EventBurst, "event-burst", c.EventBurst, "Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. Only used if --event-qps > 0") - - fs.BoolVar(&c.EnableDebuggingHandlers, "enable-debugging-handlers", c.EnableDebuggingHandlers, "Enables server endpoints for log collection and local running of containers and commands") - fs.BoolVar(&c.EnableContentionProfiling, "contention-profiling", c.EnableContentionProfiling, "Enable lock contention profiling, if profiling is enabled") - fs.Int32Var(&c.HealthzPort, "healthz-port", c.HealthzPort, "The port of the localhost healthz endpoint (set to 0 to disable)") - fs.Var(utilflag.IPVar{Val: &c.HealthzBindAddress}, "healthz-bind-address", "The IP address for the healthz server to serve on (set to '0.0.0.0' or '::' for listening in all interfaces and IP families)") - fs.Int32Var(&c.OOMScoreAdj, "oom-score-adj", c.OOMScoreAdj, "The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]") - fs.StringVar(&c.ClusterDomain, "cluster-domain", c.ClusterDomain, "Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains") - - fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "The full path of the directory in which to search for additional third party volume plugins") - fs.StringSliceVar(&c.ClusterDNS, "cluster-dns", c.ClusterDNS, "Comma-separated list of DNS server IP address. This value is used for containers DNS server in case of Pods with \"dnsPolicy=ClusterFirst\". Note: all DNS servers appearing in the list MUST serve the same set of records otherwise name resolution within the cluster may not work correctly. There is no guarantee as to which DNS server may be contacted for name resolution.") - fs.DurationVar(&c.StreamingConnectionIdleTimeout.Duration, "streaming-connection-idle-timeout", c.StreamingConnectionIdleTimeout.Duration, "Maximum time a streaming connection can be idle before the connection is automatically closed. 0 indicates no timeout. Example: '5m'") - fs.DurationVar(&c.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", c.NodeStatusUpdateFrequency.Duration, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller.") - fs.DurationVar(&c.ImageMinimumGCAge.Duration, "minimum-image-ttl-duration", c.ImageMinimumGCAge.Duration, "Minimum age for an unused image before it is garbage collected. Examples: '300ms', '10s' or '2h45m'.") - fs.Int32Var(&c.ImageGCHighThresholdPercent, "image-gc-high-threshold", c.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Values must be within the range [0, 100], To disable image garbage collection, set to 100. ") - fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. Values must be within the range [0, 100] and should not be larger than that of --image-gc-high-threshold.") - fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.") - fs.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ - "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) - fs.StringVar(&c.KubeletCgroups, "kubelet-cgroups", c.KubeletCgroups, "Optional absolute name of cgroups to create and run the Kubelet in.") - fs.StringVar(&c.SystemCgroups, "system-cgroups", c.SystemCgroups, "Optional absolute name of cgroups in which to place all non-kernel processes that are not already inside a cgroup under '/'. Empty for no container. Rolling back the flag requires a reboot.") - - fs.StringVar(&c.ProviderID, "provider-id", c.ProviderID, "Unique identifier for identifying the node in a machine database, i.e cloudprovider") - - fs.BoolVar(&c.CgroupsPerQOS, "cgroups-per-qos", c.CgroupsPerQOS, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.") - fs.StringVar(&c.CgroupDriver, "cgroup-driver", c.CgroupDriver, "Driver that the kubelet uses to manipulate cgroups on the host. Possible values: 'cgroupfs', 'systemd'") - fs.StringVar(&c.CgroupRoot, "cgroup-root", c.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.") - fs.StringVar(&c.CPUManagerPolicy, "cpu-manager-policy", c.CPUManagerPolicy, "CPU Manager policy to use. Possible values: 'none', 'static'. Default: 'none'") - fs.DurationVar(&c.CPUManagerReconcilePeriod.Duration, "cpu-manager-reconcile-period", c.CPUManagerReconcilePeriod.Duration, " CPU Manager reconciliation period. Examples: '10s', or '1m'. If not supplied, defaults to 'NodeStatusUpdateFrequency'") - fs.Var(cliflag.NewMapStringString(&c.QOSReserved), "qos-reserved", " A set of ResourceName=Percentage (e.g. memory=50%) pairs that describe how pod resource requests are reserved at the QoS level. Currently only memory is supported. Requires the QOSReserved feature gate to be enabled.") - fs.StringVar(&c.TopologyManagerPolicy, "topology-manager-policy", c.TopologyManagerPolicy, "Topology Manager policy to use. Possible values: 'none', 'best-effort', 'restricted', 'single-numa-node'.") - fs.DurationVar(&c.RuntimeRequestTimeout.Duration, "runtime-request-timeout", c.RuntimeRequestTimeout.Duration, "Timeout of all runtime requests except long running request - pull, logs, exec and attach. When timeout exceeded, kubelet will cancel the request, throw out an error and retry later.") - fs.StringVar(&c.HairpinMode, "hairpin-mode", c.HairpinMode, "How should the kubelet setup hairpin NAT. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service. Valid values are \"promiscuous-bridge\", \"hairpin-veth\" and \"none\".") - fs.Int32Var(&c.MaxPods, "max-pods", c.MaxPods, "Number of Pods that can run on this Kubelet.") - - fs.StringVar(&c.PodCIDR, "pod-cidr", c.PodCIDR, "The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master. For IPv6, the maximum number of IP's allocated is 65536") - fs.Int64Var(&c.PodPidsLimit, "pod-max-pids", c.PodPidsLimit, "Set the maximum number of processes per pod. If -1, the kubelet defaults to the node allocatable pid capacity.") - - fs.StringVar(&c.ResolverConfig, "resolv-conf", c.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.") - - fs.BoolVar(&c.RunOnce, "runonce", c.RunOnce, "If true, exit after spawning pods from static pod files or remote urls. Exclusive with --enable-server") - - fs.BoolVar(&c.CPUCFSQuota, "cpu-cfs-quota", c.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits") - fs.DurationVar(&c.CPUCFSQuotaPeriod.Duration, "cpu-cfs-quota-period", c.CPUCFSQuotaPeriod.Duration, "Sets CPU CFS quota period value, cpu.cfs_period_us, defaults to Linux Kernel default") - fs.BoolVar(&c.EnableControllerAttachDetach, "enable-controller-attach-detach", c.EnableControllerAttachDetach, "Enables the Attach/Detach controller to manage attachment/detachment of volumes scheduled to this node, and disables kubelet from executing any attach/detach operations") - fs.BoolVar(&c.MakeIPTablesUtilChains, "make-iptables-util-chains", c.MakeIPTablesUtilChains, "If true, kubelet will ensure iptables utility rules are present on host.") - fs.Int32Var(&c.IPTablesMasqueradeBit, "iptables-masquerade-bit", c.IPTablesMasqueradeBit, "The bit of the fwmark space to mark packets for SNAT. Must be within the range [0, 31]. Please match this parameter with corresponding parameter in kube-proxy.") - fs.Int32Var(&c.IPTablesDropBit, "iptables-drop-bit", c.IPTablesDropBit, "The bit of the fwmark space to mark packets for dropping. Must be within the range [0, 31].") - fs.StringVar(&c.ContainerLogMaxSize, "container-log-max-size", c.ContainerLogMaxSize, " Set the maximum size (e.g. 10Mi) of container log file before it is rotated. This flag can only be used with --container-runtime=remote.") - fs.Int32Var(&c.ContainerLogMaxFiles, "container-log-max-files", c.ContainerLogMaxFiles, " Set the maximum number of container log files that can be present for a container. The number must be >= 2. This flag can only be used with --container-runtime=remote.") - fs.StringSliceVar(&c.AllowedUnsafeSysctls, "allowed-unsafe-sysctls", c.AllowedUnsafeSysctls, "Comma-separated whitelist of unsafe sysctls or unsafe sysctl patterns (ending in *). Use these at your own risk.") - - fs.Int32Var(&c.NodeStatusMaxImages, "node-status-max-images", c.NodeStatusMaxImages, "The maximum number of images to report in Node.Status.Images. If -1 is specified, no cap will be applied.") - fs.BoolVar(&c.KernelMemcgNotification, "kernel-memcg-notification", c.KernelMemcgNotification, "If enabled, the kubelet will integrate with the kernel memcg notification to determine if memory eviction thresholds are crossed rather than polling.") - - // Flags intended for testing, not recommended used in production environments. - fs.Int64Var(&c.MaxOpenFiles, "max-open-files", c.MaxOpenFiles, "Number of files that can be opened by Kubelet process.") - - fs.StringVar(&c.ContentType, "kube-api-content-type", c.ContentType, "Content type of requests sent to apiserver.") - fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags") - fs.Int32Var(&c.KubeAPIBurst, "kube-api-burst", c.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags") - fs.BoolVar(&c.SerializeImagePulls, "serialize-image-pulls", c.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details.") - - fs.Var(cliflag.NewLangleSeparatedMapStringString(&c.EvictionHard), "eviction-hard", "A set of eviction thresholds (e.g. memory.available<1Gi) that if met would trigger a pod eviction.") - fs.Var(cliflag.NewLangleSeparatedMapStringString(&c.EvictionSoft), "eviction-soft", "A set of eviction thresholds (e.g. memory.available<1.5Gi) that if met over a corresponding grace period would trigger a pod eviction.") - fs.Var(cliflag.NewMapStringString(&c.EvictionSoftGracePeriod), "eviction-soft-grace-period", "A set of eviction grace periods (e.g. memory.available=1m30s) that correspond to how long a soft eviction threshold must hold before triggering a pod eviction.") - fs.DurationVar(&c.EvictionPressureTransitionPeriod.Duration, "eviction-pressure-transition-period", c.EvictionPressureTransitionPeriod.Duration, "Duration for which the kubelet has to wait before transitioning out of an eviction pressure condition.") - fs.Int32Var(&c.EvictionMaxPodGracePeriod, "eviction-max-pod-grace-period", c.EvictionMaxPodGracePeriod, "Maximum allowed grace period (in seconds) to use when terminating pods in response to a soft eviction threshold being met. If negative, defer to pod specified value.") - fs.Var(cliflag.NewMapStringString(&c.EvictionMinimumReclaim), "eviction-minimum-reclaim", "A set of minimum reclaims (e.g. imagefs.available=2Gi) that describes the minimum amount of resource the kubelet will reclaim when performing a pod eviction if that resource is under pressure.") - fs.Int32Var(&c.PodsPerCore, "pods-per-core", c.PodsPerCore, "Number of Pods per core that can run on this Kubelet. The total number of Pods on this Kubelet cannot exceed max-pods, so max-pods will be used if this calculation results in a larger number of Pods allowed on the Kubelet. A value of 0 disables this limit.") - fs.BoolVar(&c.ProtectKernelDefaults, "protect-kernel-defaults", c.ProtectKernelDefaults, "Default kubelet behaviour for kernel tuning. If set, kubelet errors if any of kernel tunables is different than kubelet defaults.") - fs.StringVar(&c.ReservedSystemCPUs, "reserved-cpus", c.ReservedSystemCPUs, "A comma-separated list of CPUs or CPU ranges that are reserved for system and kubernetes usage. This specific list will supersede cpu counts in --system-reserved and --kube-reserved.") - fs.StringVar(&c.TopologyManagerScope, "topology-manager-scope", c.TopologyManagerScope, "Scope to which topology hints applied. Topology Manager collects hints from Hint Providers and applies them to defined scope to ensure the pod admission. Possible values: 'container' (default), 'pod'.") - // Node Allocatable Flags - fs.Var(cliflag.NewMapStringString(&c.SystemReserved), "system-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi,ephemeral-storage=1Gi) pairs that describe resources reserved for non-kubernetes components. Currently only cpu and memory are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]") - fs.Var(cliflag.NewMapStringString(&c.KubeReserved), "kube-reserved", "A set of ResourceName=ResourceQuantity (e.g. cpu=200m,memory=500Mi,ephemeral-storage=1Gi) pairs that describe resources reserved for kubernetes system components. Currently cpu, memory and local ephemeral storage for root file system are supported. See http://kubernetes.io/docs/user-guide/compute-resources for more detail. [default=none]") - fs.StringSliceVar(&c.EnforceNodeAllocatable, "enforce-node-allocatable", c.EnforceNodeAllocatable, "A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. Acceptable options are 'none', 'pods', 'system-reserved', and 'kube-reserved'. If the latter two options are specified, '--system-reserved-cgroup' and '--kube-reserved-cgroup' must also be set, respectively. If 'none' is specified, no additional options should be set. See https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/ for more details.") - fs.StringVar(&c.SystemReservedCgroup, "system-reserved-cgroup", c.SystemReservedCgroup, "Absolute name of the top level cgroup that is used to manage non-kubernetes components for which compute resources were reserved via '--system-reserved' flag. Ex. '/system-reserved'. [default='']") - fs.StringVar(&c.KubeReservedCgroup, "kube-reserved-cgroup", c.KubeReservedCgroup, "Absolute name of the top level cgroup that is used to manage kubernetes components for which compute resources were reserved via '--kube-reserved' flag. Ex. '/kube-reserved'. [default='']") - fs.StringVar(&c.Logging.Format, "logging-format", c.Logging.Format, `Sets the log format. Permitted formats: "text", "json".\nNon-default formats don't honor these flags: -add_dir_header, --alsologtostderr, --log_backtrace_at, --log_dir, --log_file, --log_file_max_size, --logtostderr, --skip_headers, --skip_log_headers, --stderrthreshold, --log-flush-frequency.\nNon-default choices are currently alpha and subject to change without warning.`) - fs.BoolVar(&c.Logging.Sanitization, "experimental-logging-sanitization", c.Logging.Sanitization, `[Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens). -Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`) - - // Graduated experimental flags, kept for backward compatibility - fs.BoolVar(&c.KernelMemcgNotification, "experimental-kernel-memcg-notification", c.KernelMemcgNotification, "Use kernelMemcgNotification configuration, this flag will be removed in 1.23.") -} diff --git a/cmd/kubelet/app/options/options_test.go b/cmd/kubelet/app/options/options_test.go deleted file mode 100644 index dd78175e5959b..0000000000000 --- a/cmd/kubelet/app/options/options_test.go +++ /dev/null @@ -1,192 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "reflect" - "testing" - - "github.com/spf13/pflag" - - "k8s.io/apimachinery/pkg/util/diff" - cliflag "k8s.io/component-base/cli/flag" -) - -func newKubeletServerOrDie() *KubeletServer { - s, err := NewKubeletServer() - if err != nil { - panic(err) - } - return s -} - -func cleanFlags(s *KubeletServer) { - s.DynamicConfigDir = cliflag.NewStringFlag(s.DynamicConfigDir.Value()) -} - -// TestRoundTrip ensures that flag values from the Kubelet can be serialized -// to arguments and then read back and have the same value. Also catches cases -// where the default value reported by the flag is not actually allowed to be -// specified. -func TestRoundTrip(t *testing.T) { - testCases := []struct { - name string - inputFlags func() *KubeletServer - outputFlags func() *KubeletServer - flagDefaulter func(*pflag.FlagSet) - skipDefault bool - err bool - expectArgs bool - }{ - { - name: "default flags are eliminated", - inputFlags: newKubeletServerOrDie, - outputFlags: newKubeletServerOrDie, - flagDefaulter: newKubeletServerOrDie().AddFlags, - err: false, - expectArgs: false, - }, - { - name: "default flag values round trip", - inputFlags: newKubeletServerOrDie, - outputFlags: newKubeletServerOrDie, - flagDefaulter: func(*pflag.FlagSet) {}, - err: false, - expectArgs: true, - }, - { - name: "nil address does not fail for optional argument", - inputFlags: func() *KubeletServer { - s := newKubeletServerOrDie() - s.HealthzBindAddress = "" - return s - }, - outputFlags: func() *KubeletServer { - s := newKubeletServerOrDie() - s.HealthzBindAddress = "" - return s - }, - flagDefaulter: func(*pflag.FlagSet) {}, - err: false, - expectArgs: true, - }, - } - for _, testCase := range testCases { - modifiedFlags := testCase.inputFlags() - args := asArgs(modifiedFlags.AddFlags, testCase.flagDefaulter) - if testCase.expectArgs != (len(args) > 0) { - t.Errorf("%s: unexpected args: %v", testCase.name, args) - continue - } - t.Logf("%s: args: %v", testCase.name, args) - flagSet := pflag.NewFlagSet("output", pflag.ContinueOnError) - outputFlags := testCase.outputFlags() - outputFlags.AddFlags(flagSet) - if err := flagSet.Parse(args); err != nil { - if !testCase.err { - t.Errorf("%s: unexpected flag error: %v", testCase.name, err) - } - continue - } - cleanFlags(outputFlags) - if !reflect.DeepEqual(modifiedFlags, outputFlags) { - t.Errorf("%s: flags did not round trip: %s", testCase.name, diff.ObjectReflectDiff(modifiedFlags, outputFlags)) - continue - } - } -} - -func asArgs(fn, defaultFn func(*pflag.FlagSet)) []string { - fs := pflag.NewFlagSet("extended", pflag.ContinueOnError) - fn(fs) - defaults := pflag.NewFlagSet("defaults", pflag.ContinueOnError) - defaultFn(defaults) - var args []string - fs.VisitAll(func(flag *pflag.Flag) { - // if the flag implements cliflag.OmitEmpty and the value is Empty, then just omit it from the command line - if omit, ok := flag.Value.(cliflag.OmitEmpty); ok && omit.Empty() { - return - } - s := flag.Value.String() - // if the flag has the same value as the default, we can omit it without changing the meaning of the command line - var defaultValue string - if defaultFlag := defaults.Lookup(flag.Name); defaultFlag != nil { - defaultValue = defaultFlag.Value.String() - if s == defaultValue { - return - } - } - // if the flag is a string slice, each element is specified with an independent flag invocation - if values, err := fs.GetStringSlice(flag.Name); err == nil { - for _, s := range values { - args = append(args, fmt.Sprintf("--%s=%s", flag.Name, s)) - } - } else { - if len(s) == 0 { - s = defaultValue - } - args = append(args, fmt.Sprintf("--%s=%s", flag.Name, s)) - } - }) - return args -} - -func TestValidateKubeletFlags(t *testing.T) { - tests := []struct { - name string - error bool - labels map[string]string - }{ - { - name: "Invalid kubernetes.io label", - error: true, - labels: map[string]string{ - "beta.kubernetes.io/metadata-proxy-ready": "true", - }, - }, - { - name: "Valid label outside of kubernetes.io and k8s.io", - error: false, - labels: map[string]string{ - "cloud.google.com/metadata-proxy-ready": "true", - }, - }, - { - name: "Empty label list", - error: false, - labels: map[string]string{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ValidateKubeletFlags(&KubeletFlags{ - NodeLabels: tt.labels, - }) - - if tt.error && err == nil { - t.Errorf("ValidateKubeletFlags should have failed with labels: %+v", tt.labels) - } - - if !tt.error && err != nil { - t.Errorf("ValidateKubeletFlags should not have failed with labels: %+v", tt.labels) - } - }) - } - -} diff --git a/cmd/kubelet/app/options/osflags_others.go b/cmd/kubelet/app/options/osflags_others.go deleted file mode 100644 index ab4c0ac1c61cf..0000000000000 --- a/cmd/kubelet/app/options/osflags_others.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build !windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" -) - -func (f *KubeletFlags) addOSFlags(fs *pflag.FlagSet) { -} diff --git a/cmd/kubelet/app/options/osflags_windows.go b/cmd/kubelet/app/options/osflags_windows.go deleted file mode 100644 index 95ee23483e65e..0000000000000 --- a/cmd/kubelet/app/options/osflags_windows.go +++ /dev/null @@ -1,33 +0,0 @@ -// +build windows - -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "github.com/spf13/pflag" -) - -func (f *KubeletFlags) addOSFlags(fs *pflag.FlagSet) { - fs.BoolVar(&f.WindowsService, "windows-service", f.WindowsService, "Enable Windows Service Control Manager API integration") - // The default priority class associated with any process in Windows is NORMAL_PRIORITY_CLASS. Keeping it as is - // to maintain backwards compatibility. - // Source: https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities - fs.StringVar(&f.WindowsPriorityClass, "windows-priorityclass", "NORMAL_PRIORITY_CLASS", - "Set the PriorityClass associated with kubelet process, the default ones are available at "+ - "https://docs.microsoft.com/en-us/windows/win32/procthread/scheduling-priorities") -} diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go deleted file mode 100644 index af9f708dc5258..0000000000000 --- a/cmd/kubelet/app/plugins.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -// This file exists to force the desired plugin implementations to be linked. -import ( - // Credential providers - _ "k8s.io/kubernetes/pkg/credentialprovider/aws" - _ "k8s.io/kubernetes/pkg/credentialprovider/azure" - _ "k8s.io/kubernetes/pkg/credentialprovider/gcp" - - "k8s.io/component-base/featuregate" - "k8s.io/utils/exec" - - // Volume plugins - "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/cephfs" - "k8s.io/kubernetes/pkg/volume/configmap" - "k8s.io/kubernetes/pkg/volume/csi" - "k8s.io/kubernetes/pkg/volume/downwardapi" - "k8s.io/kubernetes/pkg/volume/emptydir" - "k8s.io/kubernetes/pkg/volume/fc" - "k8s.io/kubernetes/pkg/volume/flexvolume" - "k8s.io/kubernetes/pkg/volume/flocker" - "k8s.io/kubernetes/pkg/volume/git_repo" - "k8s.io/kubernetes/pkg/volume/glusterfs" - "k8s.io/kubernetes/pkg/volume/hostpath" - "k8s.io/kubernetes/pkg/volume/iscsi" - "k8s.io/kubernetes/pkg/volume/local" - "k8s.io/kubernetes/pkg/volume/nfs" - "k8s.io/kubernetes/pkg/volume/portworx" - "k8s.io/kubernetes/pkg/volume/projected" - "k8s.io/kubernetes/pkg/volume/quobyte" - "k8s.io/kubernetes/pkg/volume/rbd" - "k8s.io/kubernetes/pkg/volume/scaleio" - "k8s.io/kubernetes/pkg/volume/secret" - "k8s.io/kubernetes/pkg/volume/storageos" - - // Cloud providers - _ "k8s.io/kubernetes/pkg/cloudprovider/providers" -) - -// ProbeVolumePlugins collects all volume plugins into an easy to use list. -func ProbeVolumePlugins(featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - allPlugins := []volume.VolumePlugin{} - - // The list of plugins to probe is decided by the kubelet binary, not - // by dynamic linking or other "magic". Plugins will be analyzed and - // initialized later. - // - // Kubelet does not currently need to configure volume plugins. - // If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig - var err error - allPlugins, err = appendLegacyProviderVolumes(allPlugins, featureGate) - if err != nil { - return allPlugins, err - } - allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(volume.VolumeConfig{})...) - allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...) - allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, local.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - return allPlugins, nil -} - -// GetDynamicPluginProber gets the probers of dynamically discoverable plugins -// for kubelet. -// Currently only Flexvolume plugins are dynamically discoverable. -func GetDynamicPluginProber(pluginDir string, runner exec.Interface) volume.DynamicPluginProber { - return flexvolume.GetDynamicPluginProber(pluginDir, runner) -} diff --git a/cmd/kubelet/app/plugins_providerless.go b/cmd/kubelet/app/plugins_providerless.go deleted file mode 100644 index dc1bd3c07a722..0000000000000 --- a/cmd/kubelet/app/plugins_providerless.go +++ /dev/null @@ -1,30 +0,0 @@ -// +build providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "k8s.io/component-base/featuregate" - - "k8s.io/kubernetes/pkg/volume" -) - -func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - // no-op when we didn't compile in support for these - return allPlugins, nil -} diff --git a/cmd/kubelet/app/plugins_providers.go b/cmd/kubelet/app/plugins_providers.go deleted file mode 100644 index 9f8ce86ff0b48..0000000000000 --- a/cmd/kubelet/app/plugins_providers.go +++ /dev/null @@ -1,78 +0,0 @@ -// +build !providerless - -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "k8s.io/component-base/featuregate" - "k8s.io/csi-translation-lib/plugins" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/awsebs" - "k8s.io/kubernetes/pkg/volume/azure_file" - "k8s.io/kubernetes/pkg/volume/azuredd" - "k8s.io/kubernetes/pkg/volume/cinder" - "k8s.io/kubernetes/pkg/volume/csimigration" - "k8s.io/kubernetes/pkg/volume/gcepd" - "k8s.io/kubernetes/pkg/volume/vsphere_volume" -) - -type probeFn func() []volume.VolumePlugin - -func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) { - // Skip appending the in-tree plugin to the list of plugins to be probed/initialized - // if the CSIMigration feature flag and plugin specific feature flag indicating - // CSI migration is complete - err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete) - if err != nil { - klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err) - // TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly - } - if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) { - klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete) - return plugins, nil - } - plugins = append(plugins, fn()...) - return plugins, nil -} - -type pluginInfo struct { - pluginMigrationFeature featuregate.Feature - pluginMigrationCompleteFeature featuregate.Feature - pluginProbeFunction probeFn -} - -func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) { - pluginMigrationStatus := make(map[string]pluginInfo) - pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins} - pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azuredd.ProbeVolumePlugins} - pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins} - pluginMigrationStatus[plugins.VSphereInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationvSphere, pluginMigrationCompleteFeature: features.CSIMigrationvSphereComplete, pluginProbeFunction: vsphere_volume.ProbeVolumePlugins} - - var err error - for pluginName, pluginInfo := range pluginMigrationStatus { - allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction) - if err != nil { - return allPlugins, err - } - } - return allPlugins, nil -} diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go deleted file mode 100644 index ff7e05feec125..0000000000000 --- a/cmd/kubelet/app/server.go +++ /dev/null @@ -1,1320 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app makes it easy to create a kubelet server for various contexts. -package app - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "math" - "net" - "net/http" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/coreos/go-systemd/daemon" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "k8s.io/klog/v2" - "k8s.io/mount-utils" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/healthz" - utilfeature "k8s.io/apiserver/pkg/util/feature" - clientset "k8s.io/client-go/kubernetes" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/record" - certutil "k8s.io/client-go/util/cert" - "k8s.io/client-go/util/certificate" - "k8s.io/client-go/util/connrotation" - "k8s.io/client-go/util/keyutil" - cloudprovider "k8s.io/cloud-provider" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/configz" - "k8s.io/component-base/featuregate" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - "k8s.io/component-base/metrics/legacyregistry" - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/pkg/api/legacyscheme" - api "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/capabilities" - "k8s.io/kubernetes/pkg/credentialprovider" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/kubelet" - kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" - kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" - kubeletconfigvalidation "k8s.io/kubernetes/pkg/kubelet/apis/config/validation" - "k8s.io/kubernetes/pkg/kubelet/cadvisor" - kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate" - "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap" - "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" - "k8s.io/kubernetes/pkg/kubelet/config" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/kubelet/eviction" - evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" - dynamickubeletconfig "k8s.io/kubernetes/pkg/kubelet/kubeletconfig" - "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles" - kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" - "k8s.io/kubernetes/pkg/kubelet/server" - "k8s.io/kubernetes/pkg/kubelet/stats/pidlimit" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - utilfs "k8s.io/kubernetes/pkg/util/filesystem" - "k8s.io/kubernetes/pkg/util/flock" - nodeutil "k8s.io/kubernetes/pkg/util/node" - "k8s.io/kubernetes/pkg/util/oom" - "k8s.io/kubernetes/pkg/util/rlimit" - "k8s.io/kubernetes/pkg/volume/util/hostutil" - "k8s.io/kubernetes/pkg/volume/util/subpath" - "k8s.io/utils/exec" - utilnet "k8s.io/utils/net" -) - -const ( - // Kubelet component name - componentKubelet = "kubelet" -) - -// NewKubeletCommand creates a *cobra.Command object with default parameters -func NewKubeletCommand() *cobra.Command { - cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) - cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - kubeletFlags := options.NewKubeletFlags() - kubeletConfig, err := options.NewKubeletConfiguration() - // programmer error - if err != nil { - klog.Fatal(err) - } - - cmd := &cobra.Command{ - Use: componentKubelet, - Long: `The kubelet is the primary "node agent" that runs on each -node. It can register the node with the apiserver using one of: the hostname; a flag to -override the hostname; or specific logic for a cloud provider. - -The kubelet works in terms of a PodSpec. A PodSpec is a YAML or JSON object -that describes a pod. The kubelet takes a set of PodSpecs that are provided through -various mechanisms (primarily through the apiserver) and ensures that the containers -described in those PodSpecs are running and healthy. The kubelet doesn't manage -containers which were not created by Kubernetes. - -Other than from an PodSpec from the apiserver, there are three ways that a container -manifest can be provided to the Kubelet. - -File: Path passed as a flag on the command line. Files under this path will be monitored -periodically for updates. The monitoring period is 20s by default and is configurable -via a flag. - -HTTP endpoint: HTTP endpoint passed as a parameter on the command line. This endpoint -is checked every 20 seconds (also configurable with a flag). - -HTTP server: The kubelet can also listen for HTTP and respond to a simple API -(underspec'd currently) to submit a new manifest.`, - // The Kubelet has special flag parsing requirements to enforce flag precedence rules, - // so we do all our parsing manually in Run, below. - // DisableFlagParsing=true provides the full set of flags passed to the kubelet in the - // `args` arg to Run, without Cobra's interference. - DisableFlagParsing: true, - Run: func(cmd *cobra.Command, args []string) { - // initial flag parse, since we disable cobra's flag parsing - if err := cleanFlagSet.Parse(args); err != nil { - cmd.Usage() - klog.Fatal(err) - } - - // check if there are non-flag arguments in the command line - cmds := cleanFlagSet.Args() - if len(cmds) > 0 { - cmd.Usage() - klog.Fatalf("unknown command: %s", cmds[0]) - } - - // short-circuit on help - help, err := cleanFlagSet.GetBool("help") - if err != nil { - klog.Fatal(`"help" flag is non-bool, programmer error, please correct`) - } - if help { - cmd.Help() - return - } - - // short-circuit on verflag - verflag.PrintAndExitIfRequested() - cliflag.PrintFlags(cleanFlagSet) - - // set feature gates from initial flags-based config - if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { - klog.Fatal(err) - } - - // validate the initial KubeletFlags - if err := options.ValidateKubeletFlags(kubeletFlags); err != nil { - klog.Fatal(err) - } - - if kubeletFlags.ContainerRuntime == "remote" && cleanFlagSet.Changed("pod-infra-container-image") { - klog.Warning("Warning: For remote container runtime, --pod-infra-container-image is ignored in kubelet, which should be set in that remote runtime instead") - } - - // load kubelet config file, if provided - if configFile := kubeletFlags.KubeletConfigFile; len(configFile) > 0 { - kubeletConfig, err = loadConfigFile(configFile) - if err != nil { - klog.Fatal(err) - } - // We must enforce flag precedence by re-parsing the command line into the new object. - // This is necessary to preserve backwards-compatibility across binary upgrades. - // See issue #56171 for more details. - if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil { - klog.Fatal(err) - } - // update feature gates based on new config - if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { - klog.Fatal(err) - } - } - - // We always validate the local configuration (command line + config file). - // This is the default "last-known-good" config for dynamic config, and must always remain valid. - if err := kubeletconfigvalidation.ValidateKubeletConfiguration(kubeletConfig); err != nil { - klog.Fatal(err) - } - - if (kubeletConfig.KubeletCgroups != "" && kubeletConfig.KubeReservedCgroup != "") && (0 != strings.Index(kubeletConfig.KubeletCgroups, kubeletConfig.KubeReservedCgroup)) { - klog.Warning("unsupported configuration:KubeletCgroups is not within KubeReservedCgroup") - } - - // use dynamic kubelet config, if enabled - var kubeletConfigController *dynamickubeletconfig.Controller - if dynamicConfigDir := kubeletFlags.DynamicConfigDir.Value(); len(dynamicConfigDir) > 0 { - var dynamicKubeletConfig *kubeletconfiginternal.KubeletConfiguration - dynamicKubeletConfig, kubeletConfigController, err = BootstrapKubeletConfigController(dynamicConfigDir, - func(kc *kubeletconfiginternal.KubeletConfiguration) error { - // Here, we enforce flag precedence inside the controller, prior to the controller's validation sequence, - // so that we get a complete validation at the same point where we can decide to reject dynamic config. - // This fixes the flag-precedence component of issue #63305. - // See issue #56171 for general details on flag precedence. - return kubeletConfigFlagPrecedence(kc, args) - }) - if err != nil { - klog.Fatal(err) - } - // If we should just use our existing, local config, the controller will return a nil config - if dynamicKubeletConfig != nil { - kubeletConfig = dynamicKubeletConfig - // Note: flag precedence was already enforced in the controller, prior to validation, - // by our above transform function. Now we simply update feature gates from the new config. - if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil { - klog.Fatal(err) - } - } - } - - // construct a KubeletServer from kubeletFlags and kubeletConfig - kubeletServer := &options.KubeletServer{ - KubeletFlags: *kubeletFlags, - KubeletConfiguration: *kubeletConfig, - } - - // use kubeletServer to construct the default KubeletDeps - kubeletDeps, err := UnsecuredDependencies(kubeletServer, utilfeature.DefaultFeatureGate) - if err != nil { - klog.Fatal(err) - } - - // add the kubelet config controller to kubeletDeps - kubeletDeps.KubeletConfigController = kubeletConfigController - - // set up signal context here in order to be reused by kubelet and docker shim - ctx := genericapiserver.SetupSignalContext() - - // run the kubelet - klog.V(5).Infof("KubeletConfiguration: %#v", kubeletServer.KubeletConfiguration) - if err := Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate); err != nil { - klog.Fatal(err) - } - }, - } - - // keep cleanFlagSet separate, so Cobra doesn't pollute it with the global flags - kubeletFlags.AddFlags(cleanFlagSet) - options.AddKubeletConfigFlags(cleanFlagSet, kubeletConfig) - options.AddGlobalFlags(cleanFlagSet) - cleanFlagSet.BoolP("help", "h", false, fmt.Sprintf("help for %s", cmd.Name())) - - // ugly, but necessary, because Cobra's default UsageFunc and HelpFunc pollute the flagset with global flags - const usageFmt = "Usage:\n %s\n\nFlags:\n%s" - cmd.SetUsageFunc(func(cmd *cobra.Command) error { - fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2)) - return nil - }) - cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { - fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine(), cleanFlagSet.FlagUsagesWrapped(2)) - }) - - return cmd -} - -// newFlagSetWithGlobals constructs a new pflag.FlagSet with global flags registered -// on it. -func newFlagSetWithGlobals() *pflag.FlagSet { - fs := pflag.NewFlagSet("", pflag.ExitOnError) - // set the normalize func, similar to k8s.io/component-base/cli//flags.go:InitFlags - fs.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - // explicitly add flags from libs that register global flags - options.AddGlobalFlags(fs) - return fs -} - -// newFakeFlagSet constructs a pflag.FlagSet with the same flags as fs, but where -// all values have noop Set implementations -func newFakeFlagSet(fs *pflag.FlagSet) *pflag.FlagSet { - ret := pflag.NewFlagSet("", pflag.ExitOnError) - ret.SetNormalizeFunc(fs.GetNormalizeFunc()) - fs.VisitAll(func(f *pflag.Flag) { - ret.VarP(cliflag.NoOp{}, f.Name, f.Shorthand, f.Usage) - }) - return ret -} - -// kubeletConfigFlagPrecedence re-parses flags over the KubeletConfiguration object. -// We must enforce flag precedence by re-parsing the command line into the new object. -// This is necessary to preserve backwards-compatibility across binary upgrades. -// See issue #56171 for more details. -func kubeletConfigFlagPrecedence(kc *kubeletconfiginternal.KubeletConfiguration, args []string) error { - // We use a throwaway kubeletFlags and a fake global flagset to avoid double-parses, - // as some Set implementations accumulate values from multiple flag invocations. - fs := newFakeFlagSet(newFlagSetWithGlobals()) - // register throwaway KubeletFlags - options.NewKubeletFlags().AddFlags(fs) - // register new KubeletConfiguration - options.AddKubeletConfigFlags(fs, kc) - // Remember original feature gates, so we can merge with flag gates later - original := kc.FeatureGates - // re-parse flags - if err := fs.Parse(args); err != nil { - return err - } - // Add back feature gates that were set in the original kc, but not in flags - for k, v := range original { - if _, ok := kc.FeatureGates[k]; !ok { - kc.FeatureGates[k] = v - } - } - return nil -} - -func loadConfigFile(name string) (*kubeletconfiginternal.KubeletConfiguration, error) { - const errFmt = "failed to load Kubelet config file %s, error %v" - // compute absolute path based on current working dir - kubeletConfigFile, err := filepath.Abs(name) - if err != nil { - return nil, fmt.Errorf(errFmt, name, err) - } - loader, err := configfiles.NewFsLoader(utilfs.DefaultFs{}, kubeletConfigFile) - if err != nil { - return nil, fmt.Errorf(errFmt, name, err) - } - kc, err := loader.Load() - if err != nil { - return nil, fmt.Errorf(errFmt, name, err) - } - return kc, err -} - -// UnsecuredDependencies returns a Dependencies suitable for being run, or an error if the server setup -// is not valid. It will not start any background processes, and does not include authentication/authorization -func UnsecuredDependencies(s *options.KubeletServer, featureGate featuregate.FeatureGate) (*kubelet.Dependencies, error) { - // Initialize the TLS Options - tlsOptions, err := InitializeTLS(&s.KubeletFlags, &s.KubeletConfiguration) - if err != nil { - return nil, err - } - - mounter := mount.New(s.ExperimentalMounterPath) - subpather := subpath.New(mounter) - hu := hostutil.NewHostUtil() - var pluginRunner = exec.New() - - var dockerOptions *kubelet.DockerOptions - if s.ContainerRuntime == kubetypes.DockerContainerRuntime { - dockerOptions = &kubelet.DockerOptions{ - DockerEndpoint: s.DockerEndpoint, - RuntimeRequestTimeout: s.RuntimeRequestTimeout.Duration, - ImagePullProgressDeadline: s.ImagePullProgressDeadline.Duration, - } - } - - plugins, err := ProbeVolumePlugins(featureGate) - if err != nil { - return nil, err - } - return &kubelet.Dependencies{ - Auth: nil, // default does not enforce auth[nz] - CAdvisorInterface: nil, // cadvisor.New launches background processes (bg http.ListenAndServe, and some bg cleaners), not set here - Cloud: nil, // cloud provider might start background processes - ContainerManager: nil, - DockerOptions: dockerOptions, - KubeClient: nil, - HeartbeatClient: nil, - EventClient: nil, - HostUtil: hu, - Mounter: mounter, - Subpather: subpather, - OOMAdjuster: oom.NewOOMAdjuster(), - OSInterface: kubecontainer.RealOS{}, - VolumePlugins: plugins, - DynamicPluginProber: GetDynamicPluginProber(s.VolumePluginDir, pluginRunner), - TLSOptions: tlsOptions}, nil -} - -// Run runs the specified KubeletServer with the given Dependencies. This should never exit. -// The kubeDeps argument may be nil - if so, it is initialized from the settings on KubeletServer. -// Otherwise, the caller is assumed to have set up the Dependencies object and a default one will -// not be generated. -func Run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) error { - logOption := logs.NewOptions() - logOption.LogFormat = s.Logging.Format - logOption.LogSanitization = s.Logging.Sanitization - logOption.Apply() - // To help debugging, immediately log version - klog.Infof("Version: %+v", version.Get()) - if err := initForOS(s.KubeletFlags.WindowsService, s.KubeletFlags.WindowsPriorityClass); err != nil { - return fmt.Errorf("failed OS init: %v", err) - } - if err := run(ctx, s, kubeDeps, featureGate); err != nil { - return fmt.Errorf("failed to run Kubelet: %v", err) - } - return nil -} - -func checkPermissions() error { - if uid := os.Getuid(); uid != 0 { - return fmt.Errorf("kubelet needs to run as uid `0`. It is being run as %d", uid) - } - // TODO: Check if kubelet is running in the `initial` user namespace. - // http://man7.org/linux/man-pages/man7/user_namespaces.7.html - return nil -} - -func setConfigz(cz *configz.Config, kc *kubeletconfiginternal.KubeletConfiguration) error { - scheme, _, err := kubeletscheme.NewSchemeAndCodecs() - if err != nil { - return err - } - versioned := kubeletconfigv1beta1.KubeletConfiguration{} - if err := scheme.Convert(kc, &versioned, nil); err != nil { - return err - } - cz.Set(versioned) - return nil -} - -func initConfigz(kc *kubeletconfiginternal.KubeletConfiguration) error { - cz, err := configz.New("kubeletconfig") - if err != nil { - klog.Errorf("unable to register configz: %s", err) - return err - } - if err := setConfigz(cz, kc); err != nil { - klog.Errorf("unable to register config: %s", err) - return err - } - return nil -} - -// makeEventRecorder sets up kubeDeps.Recorder if it's nil. It's a no-op otherwise. -func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName) { - if kubeDeps.Recorder != nil { - return - } - eventBroadcaster := record.NewBroadcaster() - kubeDeps.Recorder = eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: componentKubelet, Host: string(nodeName)}) - eventBroadcaster.StartStructuredLogging(3) - if kubeDeps.EventClient != nil { - klog.V(4).Infof("Sending events to api server.") - eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeDeps.EventClient.Events("")}) - } else { - klog.Warning("No api server defined - no events will be sent to API server.") - } -} - -func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Dependencies, featureGate featuregate.FeatureGate) (err error) { - // Set global feature gates based on the value on the initial KubeletServer - err = utilfeature.DefaultMutableFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates) - if err != nil { - return err - } - // validate the initial KubeletServer (we set feature gates first, because this validation depends on feature gates) - if err := options.ValidateKubeletServer(s); err != nil { - return err - } - - // Obtain Kubelet Lock File - if s.ExitOnLockContention && s.LockFilePath == "" { - return errors.New("cannot exit on lock file contention: no lock file specified") - } - done := make(chan struct{}) - if s.LockFilePath != "" { - klog.Infof("acquiring file lock on %q", s.LockFilePath) - if err := flock.Acquire(s.LockFilePath); err != nil { - return fmt.Errorf("unable to acquire file lock on %q: %v", s.LockFilePath, err) - } - if s.ExitOnLockContention { - klog.Infof("watching for inotify events for: %v", s.LockFilePath) - if err := watchForLockfileContention(s.LockFilePath, done); err != nil { - return err - } - } - } - - // Register current configuration with /configz endpoint - err = initConfigz(&s.KubeletConfiguration) - if err != nil { - klog.Errorf("unable to register KubeletConfiguration with configz, error: %v", err) - } - - if len(s.ShowHiddenMetricsForVersion) > 0 { - metrics.SetShowHidden() - } - - // About to get clients and such, detect standaloneMode - standaloneMode := true - if len(s.KubeConfig) > 0 { - standaloneMode = false - } - - if kubeDeps == nil { - kubeDeps, err = UnsecuredDependencies(s, featureGate) - if err != nil { - return err - } - } - - if kubeDeps.Cloud == nil { - if !cloudprovider.IsExternal(s.CloudProvider) { - cloudprovider.DeprecationWarningForProvider(s.CloudProvider) - cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) - if err != nil { - return err - } - if cloud != nil { - klog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) - } - kubeDeps.Cloud = cloud - } - } - - hostName, err := nodeutil.GetHostname(s.HostnameOverride) - if err != nil { - return err - } - nodeName, err := getNodeName(kubeDeps.Cloud, hostName) - if err != nil { - return err - } - - // if in standalone mode, indicate as much by setting all clients to nil - switch { - case standaloneMode: - kubeDeps.KubeClient = nil - kubeDeps.EventClient = nil - kubeDeps.HeartbeatClient = nil - klog.Warningf("standalone mode, no API client") - - case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil: - clientConfig, closeAllConns, err := buildKubeletClientConfig(ctx, s, nodeName) - if err != nil { - return err - } - if closeAllConns == nil { - return errors.New("closeAllConns must be a valid function other than nil") - } - kubeDeps.OnHeartbeatFailure = closeAllConns - - kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig) - if err != nil { - return fmt.Errorf("failed to initialize kubelet client: %v", err) - } - - // make a separate client for events - eventClientConfig := *clientConfig - eventClientConfig.QPS = float32(s.EventRecordQPS) - eventClientConfig.Burst = int(s.EventBurst) - kubeDeps.EventClient, err = v1core.NewForConfig(&eventClientConfig) - if err != nil { - return fmt.Errorf("failed to initialize kubelet event client: %v", err) - } - - // make a separate client for heartbeat with throttling disabled and a timeout attached - heartbeatClientConfig := *clientConfig - heartbeatClientConfig.Timeout = s.KubeletConfiguration.NodeStatusUpdateFrequency.Duration - // The timeout is the minimum of the lease duration and status update frequency - leaseTimeout := time.Duration(s.KubeletConfiguration.NodeLeaseDurationSeconds) * time.Second - if heartbeatClientConfig.Timeout > leaseTimeout { - heartbeatClientConfig.Timeout = leaseTimeout - } - - heartbeatClientConfig.QPS = float32(-1) - kubeDeps.HeartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig) - if err != nil { - return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err) - } - } - - if kubeDeps.Auth == nil { - auth, runAuthenticatorCAReload, err := BuildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) - if err != nil { - return err - } - kubeDeps.Auth = auth - runAuthenticatorCAReload(ctx.Done()) - } - - var cgroupRoots []string - nodeAllocatableRoot := cm.NodeAllocatableRoot(s.CgroupRoot, s.CgroupsPerQOS, s.CgroupDriver) - cgroupRoots = append(cgroupRoots, nodeAllocatableRoot) - kubeletCgroup, err := cm.GetKubeletContainer(s.KubeletCgroups) - if err != nil { - klog.Warningf("failed to get the kubelet's cgroup: %v. Kubelet system container metrics may be missing.", err) - } else if kubeletCgroup != "" { - cgroupRoots = append(cgroupRoots, kubeletCgroup) - } - - runtimeCgroup, err := cm.GetRuntimeContainer(s.ContainerRuntime, s.RuntimeCgroups) - if err != nil { - klog.Warningf("failed to get the container runtime's cgroup: %v. Runtime system container metrics may be missing.", err) - } else if runtimeCgroup != "" { - // RuntimeCgroups is optional, so ignore if it isn't specified - cgroupRoots = append(cgroupRoots, runtimeCgroup) - } - - if s.SystemCgroups != "" { - // SystemCgroups is optional, so ignore if it isn't specified - cgroupRoots = append(cgroupRoots, s.SystemCgroups) - } - - if kubeDeps.CAdvisorInterface == nil { - imageFsInfoProvider := cadvisor.NewImageFsInfoProvider(s.ContainerRuntime, s.RemoteRuntimeEndpoint) - kubeDeps.CAdvisorInterface, err = cadvisor.New(imageFsInfoProvider, s.RootDirectory, cgroupRoots, cadvisor.UsingLegacyCadvisorStats(s.ContainerRuntime, s.RemoteRuntimeEndpoint)) - if err != nil { - return err - } - } - - // Setup event recorder if required. - makeEventRecorder(kubeDeps, nodeName) - - if kubeDeps.ContainerManager == nil { - if s.CgroupsPerQOS && s.CgroupRoot == "" { - klog.Info("--cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /") - s.CgroupRoot = "/" - } - - var reservedSystemCPUs cpuset.CPUSet - if s.ReservedSystemCPUs != "" { - // is it safe do use CAdvisor here ?? - machineInfo, err := kubeDeps.CAdvisorInterface.MachineInfo() - if err != nil { - // if can't use CAdvisor here, fall back to non-explicit cpu list behavor - klog.Warning("Failed to get MachineInfo, set reservedSystemCPUs to empty") - reservedSystemCPUs = cpuset.NewCPUSet() - } else { - var errParse error - reservedSystemCPUs, errParse = cpuset.Parse(s.ReservedSystemCPUs) - if errParse != nil { - // invalid cpu list is provided, set reservedSystemCPUs to empty, so it won't overwrite kubeReserved/systemReserved - klog.Infof("Invalid ReservedSystemCPUs \"%s\"", s.ReservedSystemCPUs) - return errParse - } - reservedList := reservedSystemCPUs.ToSlice() - first := reservedList[0] - last := reservedList[len(reservedList)-1] - if first < 0 || last >= machineInfo.NumCores { - // the specified cpuset is outside of the range of what the machine has - klog.Infof("Invalid cpuset specified by --reserved-cpus") - return fmt.Errorf("Invalid cpuset %q specified by --reserved-cpus", s.ReservedSystemCPUs) - } - } - } else { - reservedSystemCPUs = cpuset.NewCPUSet() - } - - if reservedSystemCPUs.Size() > 0 { - // at cmd option valication phase it is tested either --system-reserved-cgroup or --kube-reserved-cgroup is specified, so overwrite should be ok - klog.Infof("Option --reserved-cpus is specified, it will overwrite the cpu setting in KubeReserved=\"%v\", SystemReserved=\"%v\".", s.KubeReserved, s.SystemReserved) - if s.KubeReserved != nil { - delete(s.KubeReserved, "cpu") - } - if s.SystemReserved == nil { - s.SystemReserved = make(map[string]string) - } - s.SystemReserved["cpu"] = strconv.Itoa(reservedSystemCPUs.Size()) - klog.Infof("After cpu setting is overwritten, KubeReserved=\"%v\", SystemReserved=\"%v\"", s.KubeReserved, s.SystemReserved) - } - kubeReserved, err := parseResourceList(s.KubeReserved) - if err != nil { - return err - } - systemReserved, err := parseResourceList(s.SystemReserved) - if err != nil { - return err - } - var hardEvictionThresholds []evictionapi.Threshold - // If the user requested to ignore eviction thresholds, then do not set valid values for hardEvictionThresholds here. - if !s.ExperimentalNodeAllocatableIgnoreEvictionThreshold { - hardEvictionThresholds, err = eviction.ParseThresholdConfig([]string{}, s.EvictionHard, nil, nil, nil) - if err != nil { - return err - } - } - experimentalQOSReserved, err := cm.ParseQOSReserved(s.QOSReserved) - if err != nil { - return err - } - - devicePluginEnabled := utilfeature.DefaultFeatureGate.Enabled(features.DevicePlugins) - - kubeDeps.ContainerManager, err = cm.NewContainerManager( - kubeDeps.Mounter, - kubeDeps.CAdvisorInterface, - cm.NodeConfig{ - RuntimeCgroupsName: s.RuntimeCgroups, - SystemCgroupsName: s.SystemCgroups, - KubeletCgroupsName: s.KubeletCgroups, - ContainerRuntime: s.ContainerRuntime, - CgroupsPerQOS: s.CgroupsPerQOS, - CgroupRoot: s.CgroupRoot, - CgroupDriver: s.CgroupDriver, - KubeletRootDir: s.RootDirectory, - ProtectKernelDefaults: s.ProtectKernelDefaults, - NodeAllocatableConfig: cm.NodeAllocatableConfig{ - KubeReservedCgroupName: s.KubeReservedCgroup, - SystemReservedCgroupName: s.SystemReservedCgroup, - EnforceNodeAllocatable: sets.NewString(s.EnforceNodeAllocatable...), - KubeReserved: kubeReserved, - SystemReserved: systemReserved, - ReservedSystemCPUs: reservedSystemCPUs, - HardEvictionThresholds: hardEvictionThresholds, - }, - QOSReserved: *experimentalQOSReserved, - ExperimentalCPUManagerPolicy: s.CPUManagerPolicy, - ExperimentalCPUManagerReconcilePeriod: s.CPUManagerReconcilePeriod.Duration, - ExperimentalPodPidsLimit: s.PodPidsLimit, - EnforceCPULimits: s.CPUCFSQuota, - CPUCFSQuotaPeriod: s.CPUCFSQuotaPeriod.Duration, - ExperimentalTopologyManagerPolicy: s.TopologyManagerPolicy, - ExperimentalTopologyManagerScope: s.TopologyManagerScope, - }, - s.FailSwapOn, - devicePluginEnabled, - kubeDeps.Recorder) - - if err != nil { - return err - } - } - - if err := checkPermissions(); err != nil { - klog.Error(err) - } - - utilruntime.ReallyCrash = s.ReallyCrashForTesting - - // TODO(vmarmol): Do this through container config. - oomAdjuster := kubeDeps.OOMAdjuster - if err := oomAdjuster.ApplyOOMScoreAdj(0, int(s.OOMScoreAdj)); err != nil { - klog.Warning(err) - } - - err = kubelet.PreInitRuntimeService(&s.KubeletConfiguration, - kubeDeps, &s.ContainerRuntimeOptions, - s.ContainerRuntime, - s.RuntimeCgroups, - s.RemoteRuntimeEndpoint, - s.RemoteImageEndpoint, - s.NonMasqueradeCIDR) - if err != nil { - return err - } - - if err := RunKubelet(s, kubeDeps, s.RunOnce); err != nil { - return err - } - - // If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops - if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) && len(s.DynamicConfigDir.Value()) > 0 && - kubeDeps.KubeletConfigController != nil && !standaloneMode && !s.RunOnce { - if err := kubeDeps.KubeletConfigController.StartSync(kubeDeps.KubeClient, kubeDeps.EventClient, string(nodeName)); err != nil { - return err - } - } - - if s.HealthzPort > 0 { - mux := http.NewServeMux() - healthz.InstallHandler(mux) - go wait.Until(func() { - err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), mux) - if err != nil { - klog.Errorf("Starting healthz server failed: %v", err) - } - }, 5*time.Second, wait.NeverStop) - } - - if s.RunOnce { - return nil - } - - // If systemd is used, notify it that we have started - go daemon.SdNotify(false, "READY=1") - - select { - case <-done: - break - case <-ctx.Done(): - break - } - - return nil -} - -// buildKubeletClientConfig constructs the appropriate client config for the kubelet depending on whether -// bootstrapping is enabled or client certificate rotation is enabled. -func buildKubeletClientConfig(ctx context.Context, s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) { - if s.RotateCertificates { - // Rules for client rotation and the handling of kube config files: - // - // 1. If the client provides only a kubeconfig file, we must use that as the initial client - // kubeadm needs the initial data in the kubeconfig to be placed into the cert store - // 2. If the client provides only an initial bootstrap kubeconfig file, we must create a - // kubeconfig file at the target location that points to the cert store, but until - // the file is present the client config will have no certs - // 3. If the client provides both and the kubeconfig is valid, we must ignore the bootstrap - // kubeconfig. - // 4. If the client provides both and the kubeconfig is expired or otherwise invalid, we must - // replace the kubeconfig with a new file that points to the cert dir - // - // The desired configuration for bootstrapping is to use a bootstrap kubeconfig and to have - // the kubeconfig file be managed by this process. For backwards compatibility with kubeadm, - // which provides a high powered kubeconfig on the master with cert/key data, we must - // bootstrap the cert manager with the contents of the initial client config. - - klog.Infof("Client rotation is on, will bootstrap in background") - certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory) - if err != nil { - return nil, nil, err - } - - // use the correct content type for cert rotation, but don't set QPS - setContentTypeForClient(certConfig, s.ContentType) - - kubeClientConfigOverrides(s, clientConfig) - - clientCertificateManager, err := buildClientCertificateManager(certConfig, clientConfig, s.CertDirectory, nodeName) - if err != nil { - return nil, nil, err - } - - legacyregistry.RawMustRegister(metrics.NewGaugeFunc( - metrics.GaugeOpts{ - Subsystem: kubeletmetrics.KubeletSubsystem, - Name: "certificate_manager_client_ttl_seconds", - Help: "Gauge of the TTL (time-to-live) of the Kubelet's client certificate. " + - "The value is in seconds until certificate expiry (negative if already expired). " + - "If client certificate is invalid or unused, the value will be +INF.", - StabilityLevel: metrics.ALPHA, - }, - func() float64 { - if c := clientCertificateManager.Current(); c != nil && c.Leaf != nil { - return math.Trunc(c.Leaf.NotAfter.Sub(time.Now()).Seconds()) - } - return math.Inf(1) - }, - )) - - // the rotating transport will use the cert from the cert manager instead of these files - transportConfig := restclient.AnonymousClientConfig(clientConfig) - - // we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable - // to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper - // or the bootstrapping credentials to potentially lay down new initial config. - closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, transportConfig, clientCertificateManager, 5*time.Minute) - if err != nil { - return nil, nil, err - } - - klog.V(2).Info("Starting client certificate rotation.") - clientCertificateManager.Start() - - return transportConfig, closeAllConns, nil - } - - if len(s.BootstrapKubeconfig) > 0 { - if err := bootstrap.LoadClientCert(ctx, s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil { - return nil, nil, err - } - } - - clientConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig}, - &clientcmd.ConfigOverrides{}, - ).ClientConfig() - if err != nil { - return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err) - } - - kubeClientConfigOverrides(s, clientConfig) - closeAllConns, err := updateDialer(clientConfig) - if err != nil { - return nil, nil, err - } - return clientConfig, closeAllConns, nil -} - -// updateDialer instruments a restconfig with a dial. the returned function allows forcefully closing all active connections. -func updateDialer(clientConfig *restclient.Config) (func(), error) { - if clientConfig.Transport != nil || clientConfig.Dial != nil { - return nil, fmt.Errorf("there is already a transport or dialer configured") - } - d := connrotation.NewDialer((&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext) - clientConfig.Dial = d.DialContext - return d.CloseAll, nil -} - -// buildClientCertificateManager creates a certificate manager that will use certConfig to request a client certificate -// if no certificate is available, or the most recent clientConfig (which is assumed to point to the cert that the manager will -// write out). -func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, certDir string, nodeName types.NodeName) (certificate.Manager, error) { - newClientsetFn := func(current *tls.Certificate) (clientset.Interface, error) { - // If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap - // credentials. In the future it would be desirable to change the behavior of bootstrap - // to always fall back to the external bootstrap credentials when such credentials are - // provided by a fundamental trust system like cloud VM identity or an HSM module. - config := certConfig - if current != nil { - config = clientConfig - } - return clientset.NewForConfig(config) - } - - return kubeletcertificate.NewKubeletClientCertificateManager( - certDir, - nodeName, - - // this preserves backwards compatibility with kubeadm which passes - // a high powered certificate to the kubelet as --kubeconfig and expects - // it to be rotated out immediately - clientConfig.CertData, - clientConfig.KeyData, - - clientConfig.CertFile, - clientConfig.KeyFile, - newClientsetFn, - ) -} - -func kubeClientConfigOverrides(s *options.KubeletServer, clientConfig *restclient.Config) { - setContentTypeForClient(clientConfig, s.ContentType) - // Override kubeconfig qps/burst settings from flags - clientConfig.QPS = float32(s.KubeAPIQPS) - clientConfig.Burst = int(s.KubeAPIBurst) -} - -// getNodeName returns the node name according to the cloud provider -// if cloud provider is specified. Otherwise, returns the hostname of the node. -func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) { - if cloud == nil { - return types.NodeName(hostname), nil - } - - instances, ok := cloud.Instances() - if !ok { - return "", fmt.Errorf("failed to get instances from cloud provider") - } - - nodeName, err := instances.CurrentNodeName(context.TODO(), hostname) - if err != nil { - return "", fmt.Errorf("error fetching current node name from cloud provider: %v", err) - } - - klog.V(2).Infof("cloud provider determined current node name to be %s", nodeName) - - return nodeName, nil -} - -// InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed -// certificate and key file are generated. Returns a configured server.TLSOptions object. -func InitializeTLS(kf *options.KubeletFlags, kc *kubeletconfiginternal.KubeletConfiguration) (*server.TLSOptions, error) { - if !kc.ServerTLSBootstrap && kc.TLSCertFile == "" && kc.TLSPrivateKeyFile == "" { - kc.TLSCertFile = path.Join(kf.CertDirectory, "kubelet.crt") - kc.TLSPrivateKeyFile = path.Join(kf.CertDirectory, "kubelet.key") - - canReadCertAndKey, err := certutil.CanReadCertAndKey(kc.TLSCertFile, kc.TLSPrivateKeyFile) - if err != nil { - return nil, err - } - if !canReadCertAndKey { - hostName, err := nodeutil.GetHostname(kf.HostnameOverride) - if err != nil { - return nil, err - } - cert, key, err := certutil.GenerateSelfSignedCertKey(hostName, nil, nil) - if err != nil { - return nil, fmt.Errorf("unable to generate self signed cert: %v", err) - } - - if err := certutil.WriteCert(kc.TLSCertFile, cert); err != nil { - return nil, err - } - - if err := keyutil.WriteKey(kc.TLSPrivateKeyFile, key); err != nil { - return nil, err - } - - klog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile) - } - } - - tlsCipherSuites, err := cliflag.TLSCipherSuites(kc.TLSCipherSuites) - if err != nil { - return nil, err - } - - if len(tlsCipherSuites) > 0 { - insecureCiphers := cliflag.InsecureTLSCiphers() - for i := 0; i < len(tlsCipherSuites); i++ { - for cipherName, cipherID := range insecureCiphers { - if tlsCipherSuites[i] == cipherID { - klog.Warningf("Use of insecure cipher '%s' detected.", cipherName) - } - } - } - } - - minTLSVersion, err := cliflag.TLSVersion(kc.TLSMinVersion) - if err != nil { - return nil, err - } - - tlsOptions := &server.TLSOptions{ - Config: &tls.Config{ - MinVersion: minTLSVersion, - CipherSuites: tlsCipherSuites, - }, - CertFile: kc.TLSCertFile, - KeyFile: kc.TLSPrivateKeyFile, - } - - if len(kc.Authentication.X509.ClientCAFile) > 0 { - clientCAs, err := certutil.NewPool(kc.Authentication.X509.ClientCAFile) - if err != nil { - return nil, fmt.Errorf("unable to load client CA file %s: %v", kc.Authentication.X509.ClientCAFile, err) - } - // Specify allowed CAs for client certificates - tlsOptions.Config.ClientCAs = clientCAs - // Populate PeerCertificates in requests, but don't reject connections without verified certificates - tlsOptions.Config.ClientAuth = tls.RequestClientCert - } - - return tlsOptions, nil -} - -// setContentTypeForClient sets the appropritae content type into the rest config -// and handles defaulting AcceptContentTypes based on that input. -func setContentTypeForClient(cfg *restclient.Config, contentType string) { - if len(contentType) == 0 { - return - } - cfg.ContentType = contentType - switch contentType { - case runtime.ContentTypeProtobuf: - cfg.AcceptContentTypes = strings.Join([]string{runtime.ContentTypeProtobuf, runtime.ContentTypeJSON}, ",") - default: - // otherwise let the rest client perform defaulting - } -} - -// RunKubelet is responsible for setting up and running a kubelet. It is used in three different applications: -// 1 Integration tests -// 2 Kubelet binary -// 3 Standalone 'kubernetes' binary -// Eventually, #2 will be replaced with instances of #3 -func RunKubelet(kubeServer *options.KubeletServer, kubeDeps *kubelet.Dependencies, runOnce bool) error { - hostname, err := nodeutil.GetHostname(kubeServer.HostnameOverride) - if err != nil { - return err - } - // Query the cloud provider for our node name, default to hostname if kubeDeps.Cloud == nil - nodeName, err := getNodeName(kubeDeps.Cloud, hostname) - if err != nil { - return err - } - hostnameOverridden := len(kubeServer.HostnameOverride) > 0 - // Setup event recorder if required. - makeEventRecorder(kubeDeps, nodeName) - - var nodeIPs []net.IP - if kubeServer.NodeIP != "" { - for _, ip := range strings.Split(kubeServer.NodeIP, ",") { - parsedNodeIP := net.ParseIP(strings.TrimSpace(ip)) - if parsedNodeIP == nil { - klog.Warningf("Could not parse --node-ip value %q; ignoring", ip) - } else { - nodeIPs = append(nodeIPs, parsedNodeIP) - } - } - } - if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && len(nodeIPs) > 1 { - return fmt.Errorf("dual-stack --node-ip %q not supported in a single-stack cluster", kubeServer.NodeIP) - } else if len(nodeIPs) > 2 || (len(nodeIPs) == 2 && utilnet.IsIPv6(nodeIPs[0]) == utilnet.IsIPv6(nodeIPs[1])) { - return fmt.Errorf("bad --node-ip %q; must contain either a single IP or a dual-stack pair of IPs", kubeServer.NodeIP) - } else if len(nodeIPs) == 2 && kubeServer.CloudProvider != "" { - return fmt.Errorf("dual-stack --node-ip %q not supported when using a cloud provider", kubeServer.NodeIP) - } else if len(nodeIPs) == 2 && (nodeIPs[0].IsUnspecified() || nodeIPs[1].IsUnspecified()) { - return fmt.Errorf("dual-stack --node-ip %q cannot include '0.0.0.0' or '::'", kubeServer.NodeIP) - } - - capabilities.Initialize(capabilities.Capabilities{ - AllowPrivileged: true, - }) - - credentialprovider.SetPreferredDockercfgPath(kubeServer.RootDirectory) - klog.V(2).Infof("Using root directory: %v", kubeServer.RootDirectory) - - if kubeDeps.OSInterface == nil { - kubeDeps.OSInterface = kubecontainer.RealOS{} - } - - k, err := createAndInitKubelet(&kubeServer.KubeletConfiguration, - kubeDeps, - &kubeServer.ContainerRuntimeOptions, - kubeServer.ContainerRuntime, - hostname, - hostnameOverridden, - nodeName, - nodeIPs, - kubeServer.ProviderID, - kubeServer.CloudProvider, - kubeServer.CertDirectory, - kubeServer.RootDirectory, - kubeServer.ImageCredentialProviderConfigFile, - kubeServer.ImageCredentialProviderBinDir, - kubeServer.RegisterNode, - kubeServer.RegisterWithTaints, - kubeServer.AllowedUnsafeSysctls, - kubeServer.ExperimentalMounterPath, - kubeServer.KernelMemcgNotification, - kubeServer.ExperimentalCheckNodeCapabilitiesBeforeMount, - kubeServer.ExperimentalNodeAllocatableIgnoreEvictionThreshold, - kubeServer.MinimumGCAge, - kubeServer.MaxPerPodContainerCount, - kubeServer.MaxContainerCount, - kubeServer.MasterServiceNamespace, - kubeServer.RegisterSchedulable, - kubeServer.KeepTerminatedPodVolumes, - kubeServer.NodeLabels, - kubeServer.SeccompProfileRoot, - kubeServer.NodeStatusMaxImages) - if err != nil { - return fmt.Errorf("failed to create kubelet: %v", err) - } - - // NewMainKubelet should have set up a pod source config if one didn't exist - // when the builder was run. This is just a precaution. - if kubeDeps.PodConfig == nil { - return fmt.Errorf("failed to create kubelet, pod source config was nil") - } - podCfg := kubeDeps.PodConfig - - if err := rlimit.SetNumFiles(uint64(kubeServer.MaxOpenFiles)); err != nil { - klog.Errorf("Failed to set rlimit on max file handles: %v", err) - } - - // process pods and exit. - if runOnce { - if _, err := k.RunOnce(podCfg.Updates()); err != nil { - return fmt.Errorf("runonce failed: %v", err) - } - klog.Info("Started kubelet as runonce") - } else { - startKubelet(k, podCfg, &kubeServer.KubeletConfiguration, kubeDeps, kubeServer.EnableCAdvisorJSONEndpoints, kubeServer.EnableServer) - klog.Info("Started kubelet") - } - return nil -} - -func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *kubelet.Dependencies, enableCAdvisorJSONEndpoints, enableServer bool) { - // start the kubelet - go k.Run(podCfg.Updates()) - - // start the kubelet server - if enableServer { - go k.ListenAndServe(net.ParseIP(kubeCfg.Address), uint(kubeCfg.Port), kubeDeps.TLSOptions, kubeDeps.Auth, - enableCAdvisorJSONEndpoints, kubeCfg.EnableDebuggingHandlers, kubeCfg.EnableContentionProfiling, kubeCfg.EnableSystemLogHandler) - - } - if kubeCfg.ReadOnlyPort > 0 { - go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), enableCAdvisorJSONEndpoints) - } - if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPodResources) { - go k.ListenAndServePodResources() - } -} - -func createAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, - kubeDeps *kubelet.Dependencies, - crOptions *config.ContainerRuntimeOptions, - containerRuntime string, - hostname string, - hostnameOverridden bool, - nodeName types.NodeName, - nodeIPs []net.IP, - providerID string, - cloudProvider string, - certDirectory string, - rootDirectory string, - imageCredentialProviderConfigFile string, - imageCredentialProviderBinDir string, - registerNode bool, - registerWithTaints []api.Taint, - allowedUnsafeSysctls []string, - experimentalMounterPath string, - kernelMemcgNotification bool, - experimentalCheckNodeCapabilitiesBeforeMount bool, - experimentalNodeAllocatableIgnoreEvictionThreshold bool, - minimumGCAge metav1.Duration, - maxPerPodContainerCount int32, - maxContainerCount int32, - masterServiceNamespace string, - registerSchedulable bool, - keepTerminatedPodVolumes bool, - nodeLabels map[string]string, - seccompProfileRoot string, - nodeStatusMaxImages int32) (k kubelet.Bootstrap, err error) { - // TODO: block until all sources have delivered at least one update to the channel, or break the sync loop - // up into "per source" synchronizations - - k, err = kubelet.NewMainKubelet(kubeCfg, - kubeDeps, - crOptions, - containerRuntime, - hostname, - hostnameOverridden, - nodeName, - nodeIPs, - providerID, - cloudProvider, - certDirectory, - rootDirectory, - imageCredentialProviderConfigFile, - imageCredentialProviderBinDir, - registerNode, - registerWithTaints, - allowedUnsafeSysctls, - experimentalMounterPath, - kernelMemcgNotification, - experimentalCheckNodeCapabilitiesBeforeMount, - experimentalNodeAllocatableIgnoreEvictionThreshold, - minimumGCAge, - maxPerPodContainerCount, - maxContainerCount, - masterServiceNamespace, - registerSchedulable, - keepTerminatedPodVolumes, - nodeLabels, - seccompProfileRoot, - nodeStatusMaxImages) - if err != nil { - return nil, err - } - - k.BirthCry() - - k.StartGarbageCollection() - - return k, nil -} - -// parseResourceList parses the given configuration map into an API -// ResourceList or returns an error. -func parseResourceList(m map[string]string) (v1.ResourceList, error) { - if len(m) == 0 { - return nil, nil - } - rl := make(v1.ResourceList) - for k, v := range m { - switch v1.ResourceName(k) { - // CPU, memory, local storage, and PID resources are supported. - case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage, pidlimit.PIDs: - q, err := resource.ParseQuantity(v) - if err != nil { - return nil, err - } - if q.Sign() == -1 { - return nil, fmt.Errorf("resource quantity for %q cannot be negative: %v", k, v) - } - rl[v1.ResourceName(k)] = q - default: - return nil, fmt.Errorf("cannot reserve %q resource", k) - } - } - return rl, nil -} - -// BootstrapKubeletConfigController constructs and bootstrap a configuration controller -func BootstrapKubeletConfigController(dynamicConfigDir string, transform dynamickubeletconfig.TransformFunc) (*kubeletconfiginternal.KubeletConfiguration, *dynamickubeletconfig.Controller, error) { - if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { - return nil, nil, fmt.Errorf("failed to bootstrap Kubelet config controller, you must enable the DynamicKubeletConfig feature gate") - } - if len(dynamicConfigDir) == 0 { - return nil, nil, fmt.Errorf("cannot bootstrap Kubelet config controller, --dynamic-config-dir was not provided") - } - - // compute absolute path and bootstrap controller - dir, err := filepath.Abs(dynamicConfigDir) - if err != nil { - return nil, nil, fmt.Errorf("failed to get absolute path for --dynamic-config-dir=%s", dynamicConfigDir) - } - // get the latest KubeletConfiguration checkpoint from disk, or return the default config if no valid checkpoints exist - c := dynamickubeletconfig.NewController(dir, transform) - kc, err := c.Bootstrap() - if err != nil { - return nil, nil, fmt.Errorf("failed to determine a valid configuration, error: %v", err) - } - return kc, c, nil -} diff --git a/cmd/kubelet/app/server_bootstrap_test.go b/cmd/kubelet/app/server_bootstrap_test.go deleted file mode 100644 index c83f604340ac4..0000000000000 --- a/cmd/kubelet/app/server_bootstrap_test.go +++ /dev/null @@ -1,388 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/x509" - "crypto/x509/pkix" - "encoding/json" - "encoding/pem" - "io/ioutil" - "math/big" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "sync" - "testing" - "time" - - certapi "k8s.io/api/certificates/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - restclient "k8s.io/client-go/rest" - certutil "k8s.io/client-go/util/cert" - capihelper "k8s.io/kubernetes/pkg/apis/certificates/v1" - "k8s.io/kubernetes/pkg/controller/certificates/authority" -) - -// Test_buildClientCertificateManager validates that we can build a local client cert -// manager that will use the bootstrap client until we get a valid cert, then use our -// provided identity on subsequent requests. -func Test_buildClientCertificateManager(t *testing.T) { - testDir, err := ioutil.TempDir("", "kubeletcert") - if err != nil { - t.Fatal(err) - } - defer func() { os.RemoveAll(testDir) }() - - serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } - serverCA, err := certutil.NewSelfSignedCACert(certutil.Config{ - CommonName: "the-test-framework", - }, serverPrivateKey) - if err != nil { - t.Fatal(err) - } - server := &csrSimulator{ - t: t, - serverPrivateKey: serverPrivateKey, - serverCA: serverCA, - } - s := httptest.NewServer(server) - defer s.Close() - - config1 := &restclient.Config{ - UserAgent: "FirstClient", - Host: s.URL, - } - config2 := &restclient.Config{ - UserAgent: "SecondClient", - Host: s.URL, - } - - nodeName := types.NodeName("test") - m, err := buildClientCertificateManager(config1, config2, testDir, nodeName) - if err != nil { - t.Fatal(err) - } - defer m.Stop() - r := m.(rotater) - - // get an expired CSR (simulating historical output) - server.backdate = 2 * time.Hour - server.SetExpectUserAgent("FirstClient") - ok, err := r.RotateCerts() - if !ok || err != nil { - t.Fatalf("unexpected rotation err: %t %v", ok, err) - } - if cert := m.Current(); cert != nil { - t.Fatalf("Unexpected cert, should be expired: %#v", cert) - } - fi := getFileInfo(testDir) - if len(fi) != 2 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } - - // if m.Current() == nil, then we try again and get a valid - // client - server.backdate = 0 - server.SetExpectUserAgent("FirstClient") - if ok, err := r.RotateCerts(); !ok || err != nil { - t.Fatalf("unexpected rotation err: %t %v", ok, err) - } - if cert := m.Current(); cert == nil { - t.Fatalf("Unexpected cert, should be valid: %#v", cert) - } - fi = getFileInfo(testDir) - if len(fi) != 2 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } - - // if m.Current() != nil, then we should use the second client - server.SetExpectUserAgent("SecondClient") - if ok, err := r.RotateCerts(); !ok || err != nil { - t.Fatalf("unexpected rotation err: %t %v", ok, err) - } - if cert := m.Current(); cert == nil { - t.Fatalf("Unexpected cert, should be valid: %#v", cert) - } - fi = getFileInfo(testDir) - if len(fi) != 2 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } -} - -func Test_buildClientCertificateManager_populateCertDir(t *testing.T) { - testDir, err := ioutil.TempDir("", "kubeletcert") - if err != nil { - t.Fatal(err) - } - defer func() { os.RemoveAll(testDir) }() - - // when no cert is provided, write nothing to disk - config1 := &restclient.Config{ - UserAgent: "FirstClient", - Host: "http://localhost", - } - config2 := &restclient.Config{ - UserAgent: "SecondClient", - Host: "http://localhost", - } - nodeName := types.NodeName("test") - if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil { - t.Fatal(err) - } - fi := getFileInfo(testDir) - if len(fi) != 0 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } - - // an invalid cert should be ignored - config2.CertData = []byte("invalid contents") - config2.KeyData = []byte("invalid contents") - if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err == nil { - t.Fatal("unexpected non error") - } - fi = getFileInfo(testDir) - if len(fi) != 0 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } - - // an expired client certificate should be written to disk, because the cert manager can - // use config1 to refresh it and the cert manager won't return it for clients. - config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour)) - if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil { - t.Fatal(err) - } - fi = getFileInfo(testDir) - if len(fi) != 2 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } - - // a valid, non-expired client certificate should be written to disk - config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-time.Hour), time.Now().Add(24*time.Hour)) - if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil { - t.Fatal(err) - } - fi = getFileInfo(testDir) - if len(fi) != 2 { - t.Fatalf("Unexpected directory contents: %#v", fi) - } - -} - -func getFileInfo(dir string) map[string]os.FileInfo { - fi := make(map[string]os.FileInfo) - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if path == dir { - return nil - } - fi[path] = info - if !info.IsDir() { - os.Remove(path) - } - return nil - }) - return fi -} - -type rotater interface { - RotateCerts() (bool, error) -} - -func getCSR(req *http.Request) (*certapi.CertificateSigningRequest, error) { - if req.Body == nil { - return nil, nil - } - body, err := ioutil.ReadAll(req.Body) - if err != nil { - return nil, err - } - csr := &certapi.CertificateSigningRequest{} - if err := json.Unmarshal(body, csr); err != nil { - return nil, err - } - return csr, nil -} - -func mustMarshal(obj interface{}) []byte { - data, err := json.Marshal(obj) - if err != nil { - panic(err) - } - return data -} - -type csrSimulator struct { - t *testing.T - - serverPrivateKey *ecdsa.PrivateKey - serverCA *x509.Certificate - backdate time.Duration - - userAgentLock sync.Mutex - expectUserAgent string - - lock sync.Mutex - csr *certapi.CertificateSigningRequest -} - -func (s *csrSimulator) SetExpectUserAgent(a string) { - s.userAgentLock.Lock() - defer s.userAgentLock.Unlock() - s.expectUserAgent = a -} -func (s *csrSimulator) ExpectUserAgent() string { - s.userAgentLock.Lock() - defer s.userAgentLock.Unlock() - return s.expectUserAgent -} - -func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) { - s.lock.Lock() - defer s.lock.Unlock() - t := s.t - - // filter out timeouts as csrSimulator don't support them - q := req.URL.Query() - q.Del("timeout") - q.Del("timeoutSeconds") - q.Del("allowWatchBookmarks") - req.URL.RawQuery = q.Encode() - - t.Logf("Request %q %q %q", req.Method, req.URL, req.UserAgent()) - - if a := s.ExpectUserAgent(); len(a) > 0 && req.UserAgent() != a { - t.Errorf("Unexpected user agent: %s", req.UserAgent()) - } - - switch { - case req.Method == "POST" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests": - csr, err := getCSR(req) - if err != nil { - t.Fatal(err) - } - if csr.Name == "" { - csr.Name = "test-csr" - } - - csr.UID = types.UID("1") - csr.ResourceVersion = "1" - data := mustMarshal(csr) - w.Header().Set("Content-Type", "application/json") - w.Write(data) - - csr = csr.DeepCopy() - csr.ResourceVersion = "2" - ca := &authority.CertificateAuthority{ - Certificate: s.serverCA, - PrivateKey: s.serverPrivateKey, - Backdate: s.backdate, - } - cr, err := capihelper.ParseCSR(csr.Spec.Request) - if err != nil { - t.Fatal(err) - } - der, err := ca.Sign(cr.Raw, authority.PermissiveSigningPolicy{ - TTL: time.Hour, - }) - if err != nil { - t.Fatal(err) - } - csr.Status.Certificate = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) - csr.Status.Conditions = []certapi.CertificateSigningRequestCondition{ - {Type: certapi.CertificateApproved}, - } - s.csr = csr - - case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && (req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&limit=500&resourceVersion=0" || req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr"): - if s.csr == nil { - t.Fatalf("no csr") - } - csr := s.csr.DeepCopy() - - data := mustMarshal(&certapi.CertificateSigningRequestList{ - ListMeta: metav1.ListMeta{ - ResourceVersion: "2", - }, - Items: []certapi.CertificateSigningRequest{ - *csr, - }, - }) - w.Header().Set("Content-Type", "application/json") - w.Write(data) - - case req.Method == "GET" && req.URL.Path == "/apis/certificates.k8s.io/v1/certificatesigningrequests" && req.URL.RawQuery == "fieldSelector=metadata.name%3Dtest-csr&resourceVersion=2&watch=true": - if s.csr == nil { - t.Fatalf("no csr") - } - csr := s.csr.DeepCopy() - - data := mustMarshal(&metav1.WatchEvent{ - Type: "ADDED", - Object: runtime.RawExtension{ - Raw: mustMarshal(csr), - }, - }) - w.Header().Set("Content-Type", "application/json") - w.Write(data) - - default: - t.Fatalf("unexpected request: %s %s", req.Method, req.URL) - } -} - -// genClientCert generates an x509 certificate for testing. Certificate and key -// are returned in PEM encoding. -func genClientCert(t *testing.T, from, to time.Time) ([]byte, []byte) { - key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } - keyRaw, err := x509.MarshalECPrivateKey(key) - if err != nil { - t.Fatal(err) - } - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - t.Fatal(err) - } - cert := &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{Organization: []string{"Acme Co"}}, - NotBefore: from, - NotAfter: to, - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - BasicConstraintsValid: true, - } - certRaw, err := x509.CreateCertificate(rand.Reader, cert, cert, key.Public(), key) - if err != nil { - t.Fatal(err) - } - return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certRaw}), - pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyRaw}) -} diff --git a/cmd/kubelet/app/server_linux.go b/cmd/kubelet/app/server_linux.go deleted file mode 100644 index f6cff9af261cf..0000000000000 --- a/cmd/kubelet/app/server_linux.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "k8s.io/klog/v2" - "k8s.io/utils/inotify" -) - -func watchForLockfileContention(path string, done chan struct{}) error { - watcher, err := inotify.NewWatcher() - if err != nil { - klog.Errorf("unable to create watcher for lockfile: %v", err) - return err - } - if err = watcher.AddWatch(path, inotify.InOpen|inotify.InDeleteSelf); err != nil { - klog.Errorf("unable to watch lockfile: %v", err) - return err - } - go func() { - select { - case ev := <-watcher.Event: - klog.Infof("inotify event: %v", ev) - case err = <-watcher.Error: - klog.Errorf("inotify watcher error: %v", err) - } - close(done) - }() - return nil -} diff --git a/cmd/kubelet/app/server_test.go b/cmd/kubelet/app/server_test.go deleted file mode 100644 index 1db214ab954bc..0000000000000 --- a/cmd/kubelet/app/server_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "testing" -) - -func TestValueOfAllocatableResources(t *testing.T) { - testCases := []struct { - kubeReserved map[string]string - systemReserved map[string]string - errorExpected bool - name string - }{ - { - kubeReserved: map[string]string{"cpu": "200m", "memory": "-150G", "ephemeral-storage": "10Gi"}, - systemReserved: map[string]string{"cpu": "200m", "memory": "15Ki"}, - errorExpected: true, - name: "negative quantity value", - }, - { - kubeReserved: map[string]string{"cpu": "200m", "memory": "150Gi", "ephemeral-storage": "10Gi"}, - systemReserved: map[string]string{"cpu": "200m", "memory": "15Ky"}, - errorExpected: true, - name: "invalid quantity unit", - }, - { - kubeReserved: map[string]string{"cpu": "200m", "memory": "15G", "ephemeral-storage": "10Gi"}, - systemReserved: map[string]string{"cpu": "200m", "memory": "15Ki"}, - errorExpected: false, - name: "Valid resource quantity", - }, - } - - for _, test := range testCases { - _, err1 := parseResourceList(test.kubeReserved) - _, err2 := parseResourceList(test.systemReserved) - if test.errorExpected { - if err1 == nil && err2 == nil { - t.Errorf("%s: error expected", test.name) - } - } else { - if err1 != nil || err2 != nil { - t.Errorf("%s: unexpected error: %v, %v", test.name, err1, err2) - } - } - } -} diff --git a/cmd/kubelet/app/server_unsupported.go b/cmd/kubelet/app/server_unsupported.go deleted file mode 100644 index fd42a8ddd7ca4..0000000000000 --- a/cmd/kubelet/app/server_unsupported.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build !linux - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import "errors" - -func watchForLockfileContention(path string, done chan struct{}) error { - return errors.New("kubelet unsupported in this build") -} diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go deleted file mode 100644 index ca4e759b27c89..0000000000000 --- a/cmd/kubelet/kubelet.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// The kubelet binary is responsible for maintaining a set of containers on a particular host VM. -// It syncs data from both configuration file(s) as well as from a quorum of etcd servers. -// It then queries Docker to see what is currently running. It synchronizes the configuration data, -// with the running set of containers by starting or stopping Docker containers. -package main - -import ( - "math/rand" - "os" - "time" - - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/restclient" - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/kubernetes/cmd/kubelet/app" -) - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := app.NewKubeletCommand() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/kubemark/BUILD b/cmd/kubemark/BUILD deleted file mode 100644 index 1c368d7d5fa93..0000000000000 --- a/cmd/kubemark/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_binary( - name = "kubemark", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["hollow-node.go"], - importpath = "k8s.io/kubernetes/cmd/kubemark", - deps = [ - "//pkg/api/legacyscheme:go_default_library", - "//pkg/apis/core:go_default_library", - "//pkg/cluster/ports:go_default_library", - "//pkg/kubelet/cadvisor/testing:go_default_library", - "//pkg/kubelet/cm:go_default_library", - "//pkg/kubelet/cri/remote:go_default_library", - "//pkg/kubelet/cri/remote/fake:go_default_library", - "//pkg/kubemark:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//pkg/util/sysctl/testing:go_default_library", - "//pkg/util/taints:go_default_library", - "//staging/src/k8s.io/api/core/v1:go_default_library", - "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//staging/src/k8s.io/client-go/kubernetes:go_default_library", - "//staging/src/k8s.io/client-go/rest:go_default_library", - "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library", - "//staging/src/k8s.io/client-go/tools/record:go_default_library", - "//staging/src/k8s.io/component-base/cli/flag:go_default_library", - "//staging/src/k8s.io/component-base/logs:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/restclient:go_default_library", - "//staging/src/k8s.io/component-base/metrics/prometheus/version:go_default_library", - "//staging/src/k8s.io/component-base/version:go_default_library", - "//staging/src/k8s.io/component-base/version/verflag:go_default_library", - "//vendor/github.com/spf13/cobra:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/klog/v2:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kubemark/OWNERS b/cmd/kubemark/OWNERS deleted file mode 100644 index f9874e1757c98..0000000000000 --- a/cmd/kubemark/OWNERS +++ /dev/null @@ -1,12 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - gmarek - - shyamjvs - - sig-scalability-reviewers - - wojtek-t -approvers: - - gmarek - - shyamjvs - - sig-scalability-approvers - - wojtek-t diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go deleted file mode 100644 index 638b545d72bad..0000000000000 --- a/cmd/kubemark/hollow-node.go +++ /dev/null @@ -1,268 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - goflag "flag" - "fmt" - "math/rand" - "os" - "time" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "k8s.io/klog/v2" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/record" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/apis/core" - "k8s.io/kubernetes/pkg/cluster/ports" - cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" - "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubelet/cri/remote" - fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake" - "k8s.io/kubernetes/pkg/kubemark" - fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" - fakesysctl "k8s.io/kubernetes/pkg/util/sysctl/testing" - utiltaints "k8s.io/kubernetes/pkg/util/taints" - fakeexec "k8s.io/utils/exec/testing" -) - -type hollowNodeConfig struct { - KubeconfigPath string - KubeletPort int - KubeletReadOnlyPort int - Morph string - NodeName string - ServerPort int - ContentType string - UseRealProxier bool - ProxierSyncPeriod time.Duration - ProxierMinSyncPeriod time.Duration - NodeLabels map[string]string - RegisterWithTaints []core.Taint -} - -const ( - maxPods = 110 - podsPerCore = 0 -) - -// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy -// and make the config driven. -var knownMorphs = sets.NewString("kubelet", "proxy") - -func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) { - fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.") - fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.") - fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.") - fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.") - fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.") - fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List())) - fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.") - fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.") - fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Period that proxy rules are refreshed in hollow-proxy.") - fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Minimum period that proxy rules are refreshed in hollow-proxy.") - bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels) - fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels") - fs.Var(utiltaints.NewTaintsVar(&c.RegisterWithTaints), "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") -} - -func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) { - clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath) - if err != nil { - return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err) - } - config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig() - if err != nil { - return nil, fmt.Errorf("error while creating kubeconfig: %v", err) - } - config.ContentType = c.ContentType - config.QPS = 10 - config.Burst = 20 - return config, nil -} - -func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubletOptions { - return &kubemark.HollowKubletOptions{ - NodeName: c.NodeName, - KubeletPort: c.KubeletPort, - KubeletReadOnlyPort: c.KubeletReadOnlyPort, - MaxPods: maxPods, - PodsPerCore: podsPerCore, - NodeLabels: c.NodeLabels, - RegisterWithTaints: c.RegisterWithTaints, - } -} - -func main() { - rand.Seed(time.Now().UnixNano()) - - command := newHollowNodeCommand() - - // TODO: once we switch everything over to Cobra commands, we can go back to calling - // cliflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the - // normalize func and add the go flag set by hand. - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) - // cliflag.InitFlags() - logs.InitLogs() - defer logs.FlushLogs() - - if err := command.Execute(); err != nil { - os.Exit(1) - } -} - -// newControllerManagerCommand creates a *cobra.Command object with default parameters -func newHollowNodeCommand() *cobra.Command { - s := &hollowNodeConfig{ - NodeLabels: make(map[string]string), - } - - cmd := &cobra.Command{ - Use: "kubemark", - Long: "kubemark", - Run: func(cmd *cobra.Command, args []string) { - verflag.PrintAndExitIfRequested() - run(s) - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - s.addFlags(cmd.Flags()) - - return cmd -} - -func run(config *hollowNodeConfig) { - // To help debugging, immediately log version - klog.Infof("Version: %+v", version.Get()) - - if !knownMorphs.Has(config.Morph) { - klog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List()) - } - - // create a client to communicate with API server. - clientConfig, err := config.createClientConfigFromFile() - if err != nil { - klog.Fatalf("Failed to create a ClientConfig: %v. Exiting.", err) - } - - client, err := clientset.NewForConfig(clientConfig) - if err != nil { - klog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err) - } - - if config.Morph == "kubelet" { - f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions()) - - heartbeatClientConfig := *clientConfig - heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration - // The timeout is the minimum of the lease duration and status update frequency - leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second - if heartbeatClientConfig.Timeout > leaseTimeout { - heartbeatClientConfig.Timeout = leaseTimeout - } - - heartbeatClientConfig.QPS = float32(-1) - heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig) - if err != nil { - klog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err) - } - - cadvisorInterface := &cadvisortest.Fake{ - NodeName: config.NodeName, - } - containerManager := cm.NewStubContainerManager() - - endpoint, err := fakeremote.GenerateEndpoint() - if err != nil { - klog.Fatalf("Failed to generate fake endpoint %v.", err) - } - fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime() - if err = fakeRemoteRuntime.Start(endpoint); err != nil { - klog.Fatalf("Failed to start fake runtime %v.", err) - } - defer fakeRemoteRuntime.Stop() - runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second) - if err != nil { - klog.Fatalf("Failed to init runtime service %v.", err) - } - - hollowKubelet := kubemark.NewHollowKubelet( - f, c, - client, - heartbeatClient, - cadvisorInterface, - fakeRemoteRuntime.ImageService, - runtimeService, - containerManager, - ) - hollowKubelet.Run() - } - - if config.Morph == "proxy" { - client, err := clientset.NewForConfig(clientConfig) - if err != nil { - klog.Fatalf("Failed to create API Server client: %v", err) - } - iptInterface := fakeiptables.NewFake() - sysctl := fakesysctl.NewFake() - execer := &fakeexec.FakeExec{ - LookPathFunc: func(_ string) (string, error) { return "", errors.New("fake execer") }, - } - eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "kube-proxy", Host: config.NodeName}) - - hollowProxy, err := kubemark.NewHollowProxyOrDie( - config.NodeName, - client, - client.CoreV1(), - iptInterface, - sysctl, - execer, - eventBroadcaster, - recorder, - config.UseRealProxier, - config.ProxierSyncPeriod, - config.ProxierMinSyncPeriod, - ) - if err != nil { - klog.Fatalf("Failed to create hollowProxy instance: %v", err) - } - hollowProxy.Run() - } -} diff --git a/cmd/linkcheck/BUILD b/cmd/linkcheck/BUILD deleted file mode 100644 index 7aa32491ce8cd..0000000000000 --- a/cmd/linkcheck/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "//build:go.bzl", - go_binary = "go_binary_conditional_pure", -) -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_binary( - name = "linkcheck", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["links.go"], - importpath = "k8s.io/kubernetes/cmd/linkcheck", - deps = [ - "//vendor/github.com/mvdan/xurls:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/linkcheck/links.go b/cmd/linkcheck/links.go deleted file mode 100644 index 7a87775722550..0000000000000 --- a/cmd/linkcheck/links.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This tool extracts the links from types.go and .md files, visits the link and -// checks the status code of the response. -// Usage: -// $ linkcheck --root-dir=${ROOT} - -package main - -import ( - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - - "github.com/mvdan/xurls" - flag "github.com/spf13/pflag" -) - -var ( - rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") - fileSuffix = flag.StringSlice("file-suffix", []string{"types.go", ".md"}, "suffix of files to be checked") - // URLs matching the patterns in the regWhiteList won't be checked. Patterns - // of dummy URLs should be added to the list to avoid false alerts. Also, - // patterns of URLs that we don't care about can be added here to improve - // efficiency. - regWhiteList = []*regexp.Regexp{ - regexp.MustCompile(`https://kubernetes-site\.appspot\.com`), - // skip url that doesn't start with an English alphabet, e.g., URLs with IP addresses. - regexp.MustCompile(`https?://[^A-Za-z].*`), - regexp.MustCompile(`https?://localhost.*`), - } - // URLs listed in the fullURLWhiteList won't be checked. This separated from - // the RegWhiteList to improve efficiency. This list includes dummy URLs that - // are hard to be generalized by a regex, and URLs that will cause false alerts. - fullURLWhiteList = map[string]struct{}{ - "http://github.com/some/repo.git": {}, - // This URL returns 404 when visited by this tool, but it works fine if visited by a browser. - "http://stackoverflow.com/questions/ask?tags=kubernetes": {}, - "https://github.com/$YOUR_GITHUB_USERNAME/kubernetes.git": {}, - "https://github.com/$YOUR_GITHUB_USERNAME/kubernetes": {}, - "http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/darwin/amd64/kubectl": {}, - // It seems this server expects certain User-Agent value, it works fine with Chrome, but returns 404 if we issue a plain cURL to it. - "http://supervisord.org/": {}, - "http://kubernetes.io/vX.Y/docs": {}, - "http://kubernetes.io/vX.Y/docs/": {}, - "http://kubernetes.io/vX.Y/": {}, - } - - visitedURLs = map[string]struct{}{} - htmlpreviewReg = regexp.MustCompile(`https://htmlpreview\.github\.io/\?`) - httpOrhttpsReg = regexp.MustCompile(`https?.*`) -) - -func newWalkFunc(invalidLink *bool, client *http.Client) filepath.WalkFunc { - return func(filePath string, info os.FileInfo, initErr error) error { - hasSuffix := false - for _, suffix := range *fileSuffix { - hasSuffix = hasSuffix || strings.HasSuffix(info.Name(), suffix) - } - if !hasSuffix { - return nil - } - - fileBytes, err := ioutil.ReadFile(filePath) - if err != nil { - return err - } - foundInvalid := false - allURLs := xurls.Strict.FindAll(fileBytes, -1) - fmt.Fprintf(os.Stdout, "\nChecking file %s\n", filePath) - URL: - for _, URL := range allURLs { - // Don't check non http/https URL - if !httpOrhttpsReg.Match(URL) { - continue - } - for _, whiteURL := range regWhiteList { - if whiteURL.Match(URL) { - continue URL - } - } - if _, found := fullURLWhiteList[string(URL)]; found { - continue - } - // remove the htmlpreview Prefix - processedURL := htmlpreviewReg.ReplaceAll(URL, []byte{}) - - // check if we have visited the URL. - if _, found := visitedURLs[string(processedURL)]; found { - continue - } - visitedURLs[string(processedURL)] = struct{}{} - - retry := 0 - const maxRetry int = 3 - backoff := 100 - for retry < maxRetry { - fmt.Fprintf(os.Stdout, "Visiting %s\n", string(processedURL)) - // Use verb HEAD to increase efficiency. However, some servers - // do not handle HEAD well, so we need to try a GET to avoid - // false alert. - resp, err := client.Head(string(processedURL)) - // URLs with mock host or mock port will cause error. If we report - // the error here, people need to add the mock URL to the white - // list every time they add a mock URL, which will be a maintenance - // nightmare. Hence, we decide to only report 404 to catch the - // cases where host and port are legit, but path is not, which - // is the most common mistake in our docs. - if err != nil { - break - } - // This header is used in 301, 429 and 503. - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After - // And Go client will follow redirects automatically so the 301 check is probably unnecessary. - if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable { - retryAfter := resp.Header.Get("Retry-After") - if seconds, err := strconv.Atoi(retryAfter); err == nil { - backoff = seconds + 10 - } - fmt.Fprintf(os.Stderr, "Got %d visiting %s, retry after %d seconds.\n", resp.StatusCode, string(URL), backoff) - time.Sleep(time.Duration(backoff) * time.Second) - backoff *= 2 - retry++ - } else if resp.StatusCode == http.StatusNotFound { - // We only check for 404 error for now. 401, 403 errors are hard to handle. - - // We need to try a GET to avoid false alert. - resp, err = client.Get(string(processedURL)) - if err != nil { - break - } - if resp.StatusCode != http.StatusNotFound { - continue URL - } - - foundInvalid = true - fmt.Fprintf(os.Stderr, "Failed: in file %s, Got %d visiting %s\n", filePath, resp.StatusCode, string(URL)) - break - } else { - break - } - } - if retry == maxRetry { - foundInvalid = true - fmt.Fprintf(os.Stderr, "Failed: in file %s, still got 429 visiting %s after %d retries\n", filePath, string(URL), maxRetry) - } - } - if foundInvalid { - *invalidLink = true - } - return nil - } -} - -func main() { - flag.Parse() - - if *rootDir == "" { - flag.Usage() - os.Exit(2) - } - client := http.Client{ - Timeout: time.Duration(5 * time.Second), - } - invalidLink := false - if err := filepath.Walk(*rootDir, newWalkFunc(&invalidLink, &client)); err != nil { - fmt.Fprintf(os.Stderr, "Fail: %v.\n", err) - os.Exit(2) - } - if invalidLink { - os.Exit(1) - } -} diff --git a/cmd/preferredimports/BUILD b/cmd/preferredimports/BUILD deleted file mode 100644 index fd537fa2afb5c..0000000000000 --- a/cmd/preferredimports/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_binary", - "go_library", -) - -go_binary( - name = "preferredimports", - embed = [":go_default_library"], -) - -go_library( - name = "go_default_library", - srcs = ["preferredimports.go"], - importpath = "k8s.io/kubernetes/cmd/preferredimports", - deps = ["//vendor/golang.org/x/crypto/ssh/terminal:go_default_library"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/preferredimports/OWNERS b/cmd/preferredimports/OWNERS deleted file mode 100644 index 1ee6480c5c976..0000000000000 --- a/cmd/preferredimports/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - johnSchnake -approvers: - - dims - - cblecker - - spiffxp diff --git a/cmd/preferredimports/preferredimports.go b/cmd/preferredimports/preferredimports.go deleted file mode 100644 index 8052645d1f3c8..0000000000000 --- a/cmd/preferredimports/preferredimports.go +++ /dev/null @@ -1,263 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// verify that all the imports have our preferred alias(es). -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "go/ast" - "go/build" - "go/format" - "go/parser" - "go/token" - "io/ioutil" - "log" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - - "golang.org/x/crypto/ssh/terminal" -) - -var ( - importAliases = flag.String("import-aliases", "hack/.import-aliases", "json file with import aliases") - confirm = flag.Bool("confirm", false, "update file with the preferred aliases for imports") - regex = flag.String("include-path", "(test/e2e/|test/e2e_node)", "only files with paths matching this regex is touched") - isTerminal = terminal.IsTerminal(int(os.Stdout.Fd())) - logPrefix = "" - aliases map[string]string -) - -type analyzer struct { - fset *token.FileSet // positions are relative to fset - ctx build.Context - failed bool - donePaths map[string]interface{} -} - -func newAnalyzer() *analyzer { - ctx := build.Default - ctx.CgoEnabled = true - - a := &analyzer{ - fset: token.NewFileSet(), - ctx: ctx, - donePaths: make(map[string]interface{}), - } - - return a -} - -// collect extracts test metadata from a file. -func (a *analyzer) collect(dir string) { - if _, ok := a.donePaths[dir]; ok { - return - } - a.donePaths[dir] = nil - - // Create the AST by parsing src. - fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors|parser.ParseComments) - - if err != nil { - fmt.Fprintln(os.Stderr, "ERROR(syntax)", logPrefix, err) - a.failed = true - return - } - - for _, p := range fs { - // returns first error, but a.handleError deals with it - files := a.filterFiles(p.Files) - for _, file := range files { - replacements := make(map[string]string) - pathToFile := a.fset.File(file.Pos()).Name() - for _, imp := range file.Imports { - importPath := strings.Replace(imp.Path.Value, "\"", "", -1) - pathSegments := strings.Split(importPath, "/") - importName := pathSegments[len(pathSegments)-1] - if imp.Name != nil { - importName = imp.Name.Name - } - if alias, ok := aliases[importPath]; ok { - if alias != importName { - if !*confirm { - fmt.Fprintf(os.Stderr, "%sERROR wrong alias for import \"%s\" should be %s in file %s\n", logPrefix, importPath, alias, pathToFile) - a.failed = true - } - replacements[importName] = alias - if imp.Name != nil { - imp.Name.Name = alias - } else { - imp.Name = ast.NewIdent(alias) - } - } - } - } - - if len(replacements) > 0 { - if *confirm { - fmt.Printf("%sReplacing imports with aliases in file %s\n", logPrefix, pathToFile) - for key, value := range replacements { - renameImportUsages(file, key, value) - } - ast.SortImports(a.fset, file) - var buffer bytes.Buffer - if err = format.Node(&buffer, a.fset, file); err != nil { - panic(fmt.Sprintf("Error formatting ast node after rewriting import.\n%s\n", err.Error())) - } - - fileInfo, err := os.Stat(pathToFile) - if err != nil { - panic(fmt.Sprintf("Error stat'ing file: %s\n%s\n", pathToFile, err.Error())) - } - - err = ioutil.WriteFile(pathToFile, buffer.Bytes(), fileInfo.Mode()) - if err != nil { - panic(fmt.Sprintf("Error writing file: %s\n%s\n", pathToFile, err.Error())) - } - } - } - } - } -} - -func renameImportUsages(f *ast.File, old, new string) { - // use this to avoid renaming the package declaration, eg: - // given: package foo; import foo "bar"; foo.Baz, rename foo->qux - // yield: package foo; import qux "bar"; qux.Baz - var pkg *ast.Ident - - // Rename top-level old to new, both unresolved names - // (probably defined in another file) and names that resolve - // to a declaration we renamed. - ast.Inspect(f, func(node ast.Node) bool { - if node == nil { - return false - } - switch id := node.(type) { - case *ast.File: - pkg = id.Name - case *ast.Ident: - if pkg != nil && id == pkg { - return false - } - if id.Name == old { - id.Name = new - } - } - return true - }) -} - -func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File { - var files []*ast.File - for _, f := range fs { - files = append(files, f) - } - return files -} - -type collector struct { - dirs []string - regex *regexp.Regexp -} - -// handlePath walks the filesystem recursively, collecting directories, -// ignoring some unneeded directories (hidden/vendored) that are handled -// specially later. -func (c *collector) handlePath(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - // Ignore hidden directories (.git, .cache, etc) - if len(path) > 1 && path[0] == '.' || - // Staging code is symlinked from vendor/k8s.io, and uses import - // paths as if it were inside of vendor/. It fails typechecking - // inside of staging/, but works when typechecked as part of vendor/. - path == "staging" || - // OS-specific vendor code tends to be imported by OS-specific - // packages. We recursively typecheck imported vendored packages for - // each OS, but don't typecheck everything for every OS. - path == "vendor" || - path == "_output" || - // This is a weird one. /testdata/ is *mostly* ignored by Go, - // and this translates to kubernetes/vendor not working. - // edit/record.go doesn't compile without gopkg.in/yaml.v2 - // in $GOSRC/$GOROOT (both typecheck and the shell script). - path == "pkg/kubectl/cmd/testdata/edit" { - return filepath.SkipDir - } - if c.regex.MatchString(path) { - c.dirs = append(c.dirs, path) - } - } - return nil -} - -func main() { - flag.Parse() - args := flag.Args() - - if len(args) == 0 { - args = append(args, ".") - } - - regex, err := regexp.Compile(*regex) - if err != nil { - log.Fatalf("Error compiling regex: %v", err) - } - c := collector{regex: regex} - for _, arg := range args { - err := filepath.Walk(arg, c.handlePath) - if err != nil { - log.Fatalf("Error walking: %v", err) - } - } - sort.Strings(c.dirs) - - if len(*importAliases) > 0 { - bytes, err := ioutil.ReadFile(*importAliases) - if err != nil { - log.Fatalf("Error reading import aliases: %v", err) - } - err = json.Unmarshal(bytes, &aliases) - if err != nil { - log.Fatalf("Error loading aliases: %v", err) - } - } - if isTerminal { - logPrefix = "\r" // clear status bar when printing - } - fmt.Println("checking-imports: ") - - a := newAnalyzer() - for _, dir := range c.dirs { - if isTerminal { - fmt.Printf("\r\033[0m %-80s", dir) - } - a.collect(dir) - } - fmt.Println() - if a.failed { - os.Exit(1) - } -} diff --git a/cmd/verifydependencies/BUILD b/cmd/verifydependencies/BUILD deleted file mode 100644 index 3dd9b2975b98b..0000000000000 --- a/cmd/verifydependencies/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "go_default_library", - srcs = ["verifydependencies.go"], - importpath = "k8s.io/kubernetes/cmd/verifydependencies", - visibility = ["//visibility:private"], - deps = ["//vendor/gopkg.in/yaml.v2:go_default_library"], -) - -go_binary( - name = "verifydependencies", - embed = [":go_default_library"], - visibility = ["//visibility:public"], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], - visibility = ["//visibility:public"], -) diff --git a/cmd/verifydependencies/OWNERS b/cmd/verifydependencies/OWNERS deleted file mode 100644 index 18ed712ebb1a2..0000000000000 --- a/cmd/verifydependencies/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: -- neolit123 -- justaugustus -approvers: -- yastij -- dims diff --git a/cmd/verifydependencies/verifydependencies.go b/cmd/verifydependencies/verifydependencies.go deleted file mode 100644 index f8f159abadc61..0000000000000 --- a/cmd/verifydependencies/verifydependencies.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// verify that dependencies are up-to-date across different files -package main - -import ( - "flag" - "log" - "strings" - - "bufio" - "io/ioutil" - "os" - "regexp" - - "gopkg.in/yaml.v2" -) - -type dependencies struct { - Dependencies []*dependency `yaml:"dependencies"` -} - -type dependency struct { - Name string `yaml:"name"` - Version string `yaml:"version"` - RefPaths []*refPath `yaml:"refPaths"` -} - -type refPath struct { - Path string `yaml:"path"` - Match string `yaml:"match"` -} - -func main() { - - flag.Parse() - - args := flag.Args() - - if len(args) == 0 { - log.Fatalf("usage: verifydependency ") - } - externalDepsFilePath := args[0] - externalDepsFile, err := ioutil.ReadFile(externalDepsFilePath) - if err != nil { - panic(err) - } - - mismatchErrorMessage := "ERROR: %v indicates that %v should be at version %v, but the following files didn't match:\n\n" + - "%v\n\nif you are changing the version of %v, make sure all of the following files are updated with the newest version including %v\n" + - "then run ./hack/verify-external-dependencies-version.sh\n\n" - externalDeps := &dependencies{} - var pathsToUpdate []string - err = yaml.Unmarshal(externalDepsFile, externalDeps) - if err != nil { - panic(err) - } - - for _, dep := range externalDeps.Dependencies { - for _, refPath := range dep.RefPaths { - func() { - file, err := os.Open(refPath.Path) - if err != nil { - log.Fatalf("error opening file %v : %v", refPath.Path, err) - } - defer file.Close() - matcher := regexp.MustCompile(refPath.Match) - depFileScanner := bufio.NewScanner(file) - var found bool - for depFileScanner.Scan() { - line := depFileScanner.Text() - if matcher.MatchString(line) && strings.Contains(line, dep.Version) { - found = true - break - } - } - if !found { - pathsToUpdate = append(pathsToUpdate, refPath.Path) - } - }() - } - if len(pathsToUpdate) > 0 { - log.Fatalf(mismatchErrorMessage, externalDepsFilePath, dep.Name, dep.Version, strings.Join(pathsToUpdate, "\n"), dep.Name, externalDepsFilePath) - } - } - -}