diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index dbb9bbeda027..bfeb02ef89a4 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -3105,6 +3105,11 @@ "Comment": "v1.10.0", "Rev": "fc32d2f3698e36b93322a3465f63a14e9f0eaead" }, + { + "ImportPath": "k8s.io/kubernetes/cmd/kubeadm/app/features", + "Comment": "v1.10.0", + "Rev": "fc32d2f3698e36b93322a3465f63a14e9f0eaead" + }, { "ImportPath": "k8s.io/kubernetes/cmd/kubelet/app", "Comment": "v1.10.0", diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 7f4b36d6fe6f..ef2e6698c5a7 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -243,6 +243,13 @@ func NewKubeletConfig(k8s config.KubernetesConfig) (string, error) { extraOpts = SetContainerRuntime(extraOpts, k8s.ContainerRuntime) extraFlags := convertToFlags(extraOpts) + + // parses a map of the feature gates for kubelet + _, kubeletFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) + if err != nil { + return "", errors.Wrap(err, "parses feature gate config for kubelet") + } + b := bytes.Buffer{} opts := struct { ExtraOptions string @@ -250,7 +257,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig) (string, error) { ContainerRuntime string }{ ExtraOptions: extraFlags, - FeatureGates: k8s.FeatureGates, + FeatureGates: kubeletFeatureArgs, ContainerRuntime: k8s.ContainerRuntime, } if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil { @@ -334,8 +341,14 @@ func generateConfig(k8s config.KubernetesConfig) (string, error) { return "", errors.Wrap(err, "parsing kubernetes version") } + // parses a map of the feature gates for kubeadm and component + kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) + if err != nil { + return "", errors.Wrap(err, "parses feature gate config for kubeadm and component") + } + // generates a map of component to extra args for apiserver, controller-manager, and scheduler - extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, k8s.FeatureGates) + extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version, componentFeatureArgs) if err != nil { return "", errors.Wrap(err, "generating extra component config for kubeadm") } @@ -349,6 +362,7 @@ func generateConfig(k8s config.KubernetesConfig) (string, error) { EtcdDataDir string NodeName string ExtraArgs []ComponentExtraArgs + FeatureArgs map[string]bool }{ CertDir: util.DefaultCertPath, ServiceCIDR: util.DefaultServiceCIDR, @@ -358,6 +372,7 @@ func generateConfig(k8s config.KubernetesConfig) (string, error) { EtcdDataDir: "/data/minikube", //TODO(r2d4): change to something else persisted NodeName: k8s.NodeName, ExtraArgs: extraComponentConfig, + FeatureArgs: kubeadmFeatureArgs, } b := bytes.Buffer{} diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/kubeadm/templates.go index b683b0e2645a..ab3f8602d79c 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/kubeadm/templates.go @@ -38,6 +38,8 @@ etcd: nodeName: {{.NodeName}} {{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }} {{$val}}{{end}} +{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}} + {{$i}}: {{$val}}{{end}} {{end}}`)) var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(` diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/kubeadm/versions.go index a638a4385743..de4c77cf4051 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions.go @@ -20,11 +20,13 @@ import ( "fmt" "path" "sort" + "strconv" "strings" "github.com/blang/semver" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/util" ) @@ -108,6 +110,48 @@ func NewComponentExtraArgs(opts util.ExtraOptionSlice, version semver.Version, f return kubeadmExtraArgs, nil } +func ParseFeatureArgs(featureGates string) (map[string]bool, string, error) { + kubeadmFeatureArgs := map[string]bool{} + componentFeatureArgs := "" + for _, s := range strings.Split(featureGates, ",") { + if len(s) == 0 { + continue + } + + fg := strings.SplitN(s, "=", 2) + if len(fg) != 2 { + return nil, "", fmt.Errorf("missing value for key \"%v\"", s) + } + + k := strings.TrimSpace(fg[0]) + v := strings.TrimSpace(fg[1]) + + if !Supports(k) { + componentFeatureArgs = fmt.Sprintf("%s%s,", componentFeatureArgs, s) + continue + } + + boolValue, err := strconv.ParseBool(v) + if err != nil { + return nil, "", errors.Wrapf(err, "failed to convert bool value \"%v\"", v) + } + kubeadmFeatureArgs[k] = boolValue + } + componentFeatureArgs = strings.TrimRight(componentFeatureArgs, ",") + return kubeadmFeatureArgs, componentFeatureArgs, nil +} + +// Supports indicates whether a feature name is supported on the +// feature gates for kubeadm +func Supports(featureName string) bool { + for k := range features.InitFeatureGates { + if featureName == string(k) { + return true + } + } + return false +} + func ParseKubernetesVersion(version string) (semver.Version, error) { // Strip leading 'v' prefix from version for semver parsing v, err := semver.Make(version[1:]) diff --git a/pkg/minikube/bootstrapper/kubeadm/versions_test.go b/pkg/minikube/bootstrapper/kubeadm/versions_test.go index a61f3dfc8db4..0d74cb4df11d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions_test.go +++ b/pkg/minikube/bootstrapper/kubeadm/versions_test.go @@ -17,6 +17,7 @@ limitations under the License. package kubeadm import ( + "reflect" "testing" "github.com/blang/semver" @@ -101,3 +102,55 @@ func TestParseKubernetesVersion(t *testing.T) { t.Errorf("Expected: %s, Actual:%s", "1.8.0-alpha.5", version) } } + +func TestParseFeatureArgs(t *testing.T) { + tests := []struct { + description string + featureGates string + expectedKubeadmFeatureArgs map[string]bool + expectedComponentFeatureArgs string + }{ + { + description: "only kubeadm feature", + featureGates: "Auditing=true,SelfHosting=false", + expectedKubeadmFeatureArgs: map[string]bool{ + "Auditing": true, + "SelfHosting": false, + }, + expectedComponentFeatureArgs: "", + }, + { + description: "only component feature", + featureGates: "PodPriority=true,Accelerators=false", + expectedKubeadmFeatureArgs: map[string]bool{}, + expectedComponentFeatureArgs: "PodPriority=true,Accelerators=false", + }, + { + description: "between component and kubeadm feature", + featureGates: "Auditing=true,PodPriority=true,SelfHosting=false,Accelerators=false", + expectedKubeadmFeatureArgs: map[string]bool{ + "Auditing": true, + "SelfHosting": false, + }, + expectedComponentFeatureArgs: "PodPriority=true,Accelerators=false", + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + kubeadm, component, err := ParseFeatureArgs(test.featureGates) + + if err != nil { + t.Fatalf("Error parsing feature args: %s", err) + } + + if !reflect.DeepEqual(kubeadm, test.expectedKubeadmFeatureArgs) { + t.Errorf("Kubeadm Actual: %v, Expected: %v", kubeadm, test.expectedKubeadmFeatureArgs) + } + + if !reflect.DeepEqual(component, test.expectedComponentFeatureArgs) { + t.Errorf("Component Actual: %v, Expected: %v", component, test.expectedComponentFeatureArgs) + } + }) + } +} diff --git a/vendor/k8s.io/kubernetes/cmd/kubeadm/app/features/BUILD b/vendor/k8s.io/kubernetes/cmd/kubeadm/app/features/BUILD new file mode 100644 index 000000000000..c69c0a3a5806 --- /dev/null +++ b/vendor/k8s.io/kubernetes/cmd/kubeadm/app/features/BUILD @@ -0,0 +1,37 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_library( + name = "go_default_library", + srcs = ["features.go"], + importpath = "k8s.io/kubernetes/cmd/kubeadm/app/features", + deps = [ + "//pkg/util/version:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) + +go_test( + name = "go_default_test", + srcs = ["features_test.go"], + embed = [":go_default_library"], + deps = ["//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library"], +) diff --git a/vendor/k8s.io/kubernetes/cmd/kubeadm/app/features/features.go b/vendor/k8s.io/kubernetes/cmd/kubeadm/app/features/features.go new file mode 100644 index 000000000000..f9f5ee281bb2 --- /dev/null +++ b/vendor/k8s.io/kubernetes/cmd/kubeadm/app/features/features.go @@ -0,0 +1,182 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "fmt" + "sort" + "strconv" + "strings" + + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/util/version" +) + +const ( + // HighAvailability is alpha in v1.9 + HighAvailability = "HighAvailability" + + // CoreDNS is alpha in v1.9 + CoreDNS = "CoreDNS" + + // SelfHosting is alpha in v1.8 and v1.9 + SelfHosting = "SelfHosting" + + // StoreCertsInSecrets is alpha in v1.8 and v1.9 + StoreCertsInSecrets = "StoreCertsInSecrets" + + // DynamicKubeletConfig is alpha in v1.9 + DynamicKubeletConfig = "DynamicKubeletConfig" + + // Auditing is beta in 1.8 + Auditing = "Auditing" +) + +var v190 = version.MustParseSemantic("v1.9.0-alpha.1") + +// InitFeatureGates are the default feature gates for the init command +var InitFeatureGates = FeatureList{ + SelfHosting: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, + StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, + // We don't want to advertise this feature gate exists in v1.9 to avoid confusion as it is not yet working + HighAvailability: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190, HiddenInHelpText: true}, + CoreDNS: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}, MinimumVersion: v190}, + DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, + Auditing: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, +} + +// Feature represents a feature being gated +type Feature struct { + utilfeature.FeatureSpec + MinimumVersion *version.Version + HiddenInHelpText bool +} + +// FeatureList represents a list of feature gates +type FeatureList map[string]Feature + +// ValidateVersion ensures that a feature gate list is compatible with the chosen kubernetes version +func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error { + if requestedVersion == "" { + return nil + } + parsedExpVersion, err := version.ParseSemantic(requestedVersion) + if err != nil { + return fmt.Errorf("Error parsing version %s: %v", requestedVersion, err) + } + for k := range requestedFeatures { + if minVersion := allFeatures[k].MinimumVersion; minVersion != nil { + if !parsedExpVersion.AtLeast(minVersion) { + return fmt.Errorf( + "the requested kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum", + requestedVersion, k, minVersion) + } + } + } + return nil +} + +// Enabled indicates whether a feature name has been enabled +func Enabled(featureList map[string]bool, featureName string) bool { + return featureList[string(featureName)] +} + +// Supports indicates whether a feature name is supported on the given +// feature set +func Supports(featureList FeatureList, featureName string) bool { + for k := range featureList { + if featureName == string(k) { + return true + } + } + return false +} + +// Keys returns a slice of feature names for a given feature set +func Keys(featureList FeatureList) []string { + var list []string + for k := range featureList { + list = append(list, string(k)) + } + return list +} + +// KnownFeatures returns a slice of strings describing the FeatureList features. +func KnownFeatures(f *FeatureList) []string { + var known []string + for k, v := range *f { + if v.HiddenInHelpText { + continue + } + + pre := "" + if v.PreRelease != utilfeature.GA { + pre = fmt.Sprintf("%s - ", v.PreRelease) + } + known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default)) + } + sort.Strings(known) + return known +} + +// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a +// map[string]bool of known keys or returns an error. +func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) { + featureGate := map[string]bool{} + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + + arr := strings.SplitN(s, "=", 2) + if len(arr) != 2 { + return nil, fmt.Errorf("missing bool value for feature-gate key:%s", s) + } + + k := strings.TrimSpace(arr[0]) + v := strings.TrimSpace(arr[1]) + + if !Supports(*f, k) { + return nil, fmt.Errorf("unrecognized feature-gate key: %s", k) + } + + boolValue, err := strconv.ParseBool(v) + if err != nil { + return nil, fmt.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k) + } + featureGate[k] = boolValue + } + + ResolveFeatureGateDependencies(featureGate) + + return featureGate, nil +} + +// ResolveFeatureGateDependencies resolve dependencies between feature gates +func ResolveFeatureGateDependencies(featureGate map[string]bool) { + + // if StoreCertsInSecrets enabled, SelfHosting should enabled + if Enabled(featureGate, StoreCertsInSecrets) { + featureGate[SelfHosting] = true + } + + // if HighAvailability enabled, both StoreCertsInSecrets and SelfHosting should enabled + if Enabled(featureGate, HighAvailability) && !Enabled(featureGate, StoreCertsInSecrets) { + featureGate[SelfHosting] = true + featureGate[StoreCertsInSecrets] = true + } +}