diff --git a/docs/security.md b/docs/security.md index abba7215766fd..a1a244c419c97 100644 --- a/docs/security.md +++ b/docs/security.md @@ -34,13 +34,25 @@ This stores the [config.json](https://docs.docker.com/engine/reference/commandli All Pods running on your cluster have access to underlying instance IAM role. Currently permission scope is quite broad. See [iam_roles.md](iam_roles.md) for details and ways to mitigate that. - ## Kubernetes API (this section is a work in progress) Kubernetes has a number of authentication mechanisms: +## Kubelet API + +By default AnonymousAuth on the kubelet is off and so communication between kube-apiserver and kubelet api is not authenticated. In order to switch on authentication; + +```YAML +# In the cluster spec +spec: + kubelet: + anonymousAuth: false +``` + +**Note** on a existing cluster with 'anonymousAuth' unset you would need to first roll out the masters and then update the pools. + ### API Bearer Token The API bearer token is a secret named 'admin'. diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 8165df874488f..1d4f0982d7586 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -198,3 +198,30 @@ func (c *NodeupModelContext) UsesCNI() bool { } return true } + +// UseSecureKubelet checks if the kubelet api should be protected by a client certificate. Note: the settings are be +// in one of three section, master specific kubelet, cluster wide kubelet or the InstanceGroup. Though arguably is +// doesn't make much sense to unset this on a per InstanceGroup level, but hey :) +func (c *NodeupModelContext) UseSecureKubelet() bool { + cluster := &c.Cluster.Spec // just to shorten the typing + group := &c.InstanceGroup.Spec + + // @check if we have anything specific to master kubelet + if c.IsMaster { + if cluster.MasterKubelet != nil && cluster.MasterKubelet.AnonymousAuth != nil && *cluster.MasterKubelet.AnonymousAuth == true { + return true + } + } + + // @check the default settings for master and kubelet + if cluster.Kubelet != nil && cluster.Kubelet.AnonymousAuth != nil && *cluster.Kubelet.AnonymousAuth == false { + return true + } + + // @check on the InstanceGroup itself + if group.Kubelet != nil && group.Kubelet.AnonymousAuth != nil && *group.Kubelet.AnonymousAuth == false { + return true + } + + return false +} diff --git a/nodeup/pkg/model/convenience.go b/nodeup/pkg/model/convenience.go index b8e89ea0d1717..1b58d53d07c9e 100644 --- a/nodeup/pkg/model/convenience.go +++ b/nodeup/pkg/model/convenience.go @@ -17,12 +17,16 @@ limitations under the License. package model import ( + "fmt" + "path/filepath" "strconv" - "github.com/golang/glog" "k8s.io/client-go/pkg/api/v1" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + + "github.com/golang/glog" ) // s is a helper that builds a *string from a string value @@ -35,6 +39,11 @@ func i64(v int64) *int64 { return fi.Int64(v) } +// b returns a pointer to a boolean +func b(v bool) *bool { + return fi.Bool(v) +} + func getProxyEnvVars(proxies *kops.EgressProxySpec) []v1.EnvVar { if proxies == nil { glog.V(8).Info("proxies is == nil, returning empty list") @@ -62,7 +71,54 @@ func getProxyEnvVars(proxies *kops.EgressProxySpec) []v1.EnvVar { } } -// b returns a pointer to a boolean -func b(v bool) *bool { - return fi.Bool(v) +// buildCertificateRequest retrieves the certificate from a keystore +func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error { + cert, err := b.KeyStore.Cert(name) + if err != nil { + return err + } + + serialized, err := cert.AsString() + if err != nil { + return err + } + + location := filepath.Join(b.PathSrvKubernetes(), fmt.Sprintf("%s.pem", name)) + if path != "" { + location = path + } + + c.AddTask(&nodetasks.File{ + Path: location, + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + }) + + return nil +} + +// buildPrivateKeyRequest retrieves a private key from the store +func buildPrivateKeyRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error { + k, err := b.KeyStore.PrivateKey(name) + if err != nil { + return err + } + + serialized, err := k.AsString() + if err != nil { + return err + } + + location := filepath.Join(b.PathSrvKubernetes(), fmt.Sprintf("%s-key.pem", name)) + if path != "" { + location = path + } + + c.AddTask(&nodetasks.File{ + Path: location, + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + }) + + return nil } diff --git a/nodeup/pkg/model/kubeapiserver.go b/nodeup/pkg/model/kubeapiserver.go index 247d51eadd9fc..68385049186ae 100644 --- a/nodeup/pkg/model/kubeapiserver.go +++ b/nodeup/pkg/model/kubeapiserver.go @@ -41,7 +41,7 @@ type KubeAPIServerBuilder struct { var _ fi.ModelBuilder = &KubeAPIServerBuilder{} -// Build is responsible for generating the kubernetes api manifest +// Build is responsible for generating the configuration for the kube-apiserver func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.IsMaster { return nil @@ -71,6 +71,19 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + // @check if we are using secure client certificates for kubelet and grab the certificates + { + if b.UseSecureKubelet() { + name := "kubelet-api" + if err := buildCertificateRequest(c, b.NodeupModelContext, name, ""); err != nil { + return err + } + if err := buildPrivateKeyRequest(c, b.NodeupModelContext, name, ""); err != nil { + return err + } + } + } + // Touch log file, so that docker doesn't create a directory instead { t := &nodetasks.File{ @@ -135,6 +148,7 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication) } +// buildPod is responsible for generating the kube-apiserver pod and thus manifest file func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer := b.Cluster.Spec.KubeAPIServer kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") @@ -150,7 +164,15 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"} kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"} } + + // @check if we are using secure kubelet client certificates + if b.UseSecureKubelet() { + // @note we are making assumption we are using the one's created by the pki model, not custom defined ones + kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.pem") + kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api-key.pem") + } + // build the kube-apiserver flags for the service flags, err := flagbuilder.BuildFlags(b.Cluster.Spec.KubeAPIServer) if err != nil { return nil, fmt.Errorf("error building kube-apiserver flags: %v", err) @@ -228,7 +250,6 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { for _, path := range b.SSLHostPaths() { name := strings.Replace(path, "/", "", -1) - addHostPathMapping(pod, container, name, path) } diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 20ea5cd0f459d..c1323431d9a99 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -18,6 +18,7 @@ package model import ( "fmt" + "path/filepath" "k8s.io/client-go/pkg/api/v1" "k8s.io/kops/nodeup/pkg/distros" @@ -40,7 +41,7 @@ type KubeletBuilder struct { var _ fi.ModelBuilder = &KubeletBuilder{} -// Build is responsible for generating the kubelet config +// Build is responsible for building the kubelet configuration func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { kubeletConfig, err := b.buildKubeletConfig() if err != nil { @@ -270,6 +271,12 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro utils.JsonMergeStruct(c, b.Cluster.Spec.Kubelet) } + // @check if we are using secure kubelet <-> api settings + if b.UseSecureKubelet() { + // @TODO these filenames need to be a constant somewhere + c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + } + if b.InstanceGroup.Spec.Kubelet != nil { utils.JsonMergeStruct(c, b.InstanceGroup.Spec.Kubelet) } diff --git a/pkg/model/pki.go b/pkg/model/pki.go index 9b73d688ac6e6..cb268555e0605 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -32,18 +32,29 @@ type PKIModelBuilder struct { var _ fi.ModelBuilder = &PKIModelBuilder{} -// Build is responsible for generating the pki assets for the cluster +// Build is responsible for generating the various pki assets func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { { t := &fitasks.Keypair{ Name: fi.String("kubelet"), Lifecycle: b.Lifecycle, - Subject: "o=" + user.NodesGroup + ",cn=kubelet", - Type: "client", + + Subject: "o=" + user.NodesGroup + ",cn=kubelet", + Type: "client", } c.AddTask(t) } - + { + // Generate a kubelet client certificate for api to speak securely to kubelets. This change was first + // introduced in https://github.com/kubernetes/kops/pull/2831 where server.cert/key were used. With kubernetes >= 1.7 + // the certificate usage is being checked (obviously the above was server not client certificate) and so now fails + c.AddTask(&fitasks.Keypair{ + Name: fi.String("kubelet-api"), + Lifecycle: b.Lifecycle, + Subject: "cn=kubelet-api", + Type: "client", + }) + } { t := &fitasks.Keypair{ Name: fi.String("kube-scheduler"),