diff --git a/docs/security.md b/docs/security.md index b3637fe310d98..3ef545c25fdfe 100644 --- a/docs/security.md +++ b/docs/security.md @@ -25,13 +25,23 @@ To change the SSH public key on an existing cluster: All Pods running on your cluster have access to underlying instance IAM role. Currently permission scope is quite broad. See [iam_roles.md](iam_roles.md) for details and ways to mitigate that. - ## Kubernetes API (this section is a work in progress) Kubernetes has a number of authentication mechanisms: +## Kubelet API + +By default AnonymousAuth on the kubelet is off and so communication between kube-apiserver and kubelet api is not authenticated. In order to switch on authentication; + +```YAML +# In the cluster spec +spec: + kubelet: + anonymousAuth: false +``` + ### API Bearer Token The API bearer token is a secret named 'admin'. diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index c5f526d1b39c1..7ed24480fb9cf 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -18,6 +18,7 @@ package model import ( "fmt" + "github.com/blang/semver" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" @@ -182,3 +183,30 @@ func (c *NodeupModelContext) UsesCNI() bool { } return true } + +// UseSecureKubelet checks if the kubelet api should be protected by a client certificate. Note: the settings are be +// in one of three section, master specific kubelet, cluster wide kubelet or the InstanceGroup. Though arguably is +// doesn't make much sense to unset this on a per InstanceGroup level, but hey :) +func (c *NodeupModelContext) UseSecureKubelet() bool { + cluster := &c.Cluster.Spec // just to shorten the typing + group := &c.InstanceGroup.Spec + + // @check if we have anything specific to master kubelet + if c.IsMaster { + if cluster.MasterKubelet != nil && cluster.MasterKubelet.AnonymousAuth != nil && *cluster.MasterKubelet.AnonymousAuth == true { + return true + } + } + + // @check the default settings for master and kubelet + if cluster.Kubelet != nil && cluster.Kubelet.AnonymousAuth != nil && *cluster.Kubelet.AnonymousAuth == false { + return true + } + + // @check on the InstanceGroup itself + if group.Kubelet != nil && group.Kubelet.AnonymousAuth != nil && *group.Kubelet.AnonymousAuth == false { + return true + } + + return false +} diff --git a/nodeup/pkg/model/convenience.go b/nodeup/pkg/model/convenience.go index 1f6c130cd7c39..0a219a20376f9 100644 --- a/nodeup/pkg/model/convenience.go +++ b/nodeup/pkg/model/convenience.go @@ -16,7 +16,13 @@ limitations under the License. package model -import "k8s.io/kops/upup/pkg/fi" +import ( + "fmt" + "path/filepath" + + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) // s is a helper that builds a *string from a string value func s(v string) *string { @@ -27,3 +33,55 @@ func s(v string) *string { func i64(v int64) *int64 { return fi.Int64(v) } + +// buildCertificateRequest retrieves the certificate from a keystore +func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error { + cert, err := b.KeyStore.Cert(name) + if err != nil { + return err + } + + serialized, err := cert.AsString() + if err != nil { + return err + } + + location := filepath.Join(b.PathSrvKubernetes(), fmt.Sprintf("%s.pem", name)) + if path != "" { + location = path + } + + c.AddTask(&nodetasks.File{ + Path: location, + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + }) + + return nil +} + +// buildPrivateKeyRequest retrieves a private key from the store +func buildPrivateKeyRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error { + k, err := b.KeyStore.PrivateKey(name) + if err != nil { + return err + } + + serialized, err := k.AsString() + if err != nil { + return err + } + + location := filepath.Join(b.PathSrvKubernetes(), fmt.Sprintf("%s-key.pem", name)) + if path != "" { + location = path + } + + c.AddTask(&nodetasks.File{ + Path: location, + Contents: fi.NewStringResource(serialized), + Type: nodetasks.FileType_File, + }) + + return nil +} diff --git a/nodeup/pkg/model/kubeapiserver.go b/nodeup/pkg/model/kubeapiserver.go index 46cf6ac680e16..50b303181f0c4 100644 --- a/nodeup/pkg/model/kubeapiserver.go +++ b/nodeup/pkg/model/kubeapiserver.go @@ -41,6 +41,7 @@ type KubeAPIServerBuilder struct { var _ fi.ModelBuilder = &KubeAPIServerBuilder{} +// Build is responsible for generating the configuration for the kube-apiserver func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { if !b.IsMaster { return nil @@ -70,6 +71,19 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + // @check if we are using secure client certificates for kubelet and grab the certificates + { + if b.UseSecureKubelet() { + name := "kubelet-api" + if err := buildCertificateRequest(c, b.NodeupModelContext, name, ""); err != nil { + return err + } + if err := buildPrivateKeyRequest(c, b.NodeupModelContext, name, ""); err != nil { + return err + } + } + } + // Touch log file, so that docker doesn't create a directory instead { t := &nodetasks.File{ @@ -127,22 +141,30 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte Type: nodetasks.FileType_File, } c.AddTask(t) + return nil - } else { - return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication) } + + return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication) } +// buildPod is responsible for generating the kube-apiserver pod and thus manifest file func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { kubeAPIServer := b.Cluster.Spec.KubeAPIServer - kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.cert") kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key") - kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv") kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv") + // @check if we are using secure kubelet client certificates + if b.UseSecureKubelet() { + // @note we are making assumption we are using the one's created by the pki model, not custom defined ones + kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.pem") + kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api-key.pem") + } + + // build the kube-apiserver flags for the service flags, err := flagbuilder.BuildFlags(b.Cluster.Spec.KubeAPIServer) if err != nil { return nil, fmt.Errorf("error building kube-apiserver flags: %v", err) @@ -203,6 +225,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { InitialDelaySeconds: 15, TimeoutSeconds: 15, }, + Ports: []v1.ContainerPort{ { Name: "https", @@ -219,7 +242,6 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { for _, path := range b.SSLHostPaths() { name := strings.Replace(path, "/", "", -1) - addHostPathMapping(pod, container, name, path) } diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index 1ecc3f35e7f94..0fe4dc3694fa8 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -18,8 +18,8 @@ package model import ( "fmt" - "github.com/blang/semver" - "github.com/golang/glog" + "path/filepath" + "k8s.io/client-go/pkg/api/v1" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" @@ -29,6 +29,9 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" "k8s.io/kops/upup/pkg/fi/utils" + + "github.com/blang/semver" + "github.com/golang/glog" ) // KubeletBuilder install kubelet @@ -38,6 +41,7 @@ type KubeletBuilder struct { var _ fi.ModelBuilder = &KubeletBuilder{} +// Build is responsible for building the kubelet configuration func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { kubeletConfig, err := b.buildKubeletConfig() if err != nil { @@ -78,7 +82,6 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { // Add kubeconfig { // TODO: Change kubeconfig to be https - kubeconfig, err := b.buildPKIKubeconfig("kubelet") if err != nil { return err @@ -263,6 +266,12 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro utils.JsonMergeStruct(c, b.Cluster.Spec.Kubelet) } + // @check if we are using secure kubelet <-> api settings + if b.UseSecureKubelet() { + // @TODO these filenames need to be a constant somewhere + c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt") + } + if b.InstanceGroup.Spec.Kubelet != nil { utils.JsonMergeStruct(c, b.InstanceGroup.Spec.Kubelet) } diff --git a/pkg/model/pki.go b/pkg/model/pki.go index 55409f6d399d9..b57996a5b0f31 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -30,6 +30,7 @@ type PKIModelBuilder struct { var _ fi.ModelBuilder = &PKIModelBuilder{} +// Build is responsible for generating the various pki assets func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { { // Keypair used by the kubelet @@ -43,6 +44,18 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + { + // Generate a kubelet client certificate for api to speak securely to kubelets. This change was first + // introduced in https://github.com/kubernetes/kops/pull/2831 where server.cert/key were used. With kubernetes >= 1.7 + // the certificate usage is being checked (obviously the above was server not client certificate) and so now fails + c.AddTask(&fitasks.Keypair{ + Name: fi.String("kubelet-api"), + Lifecycle: b.Lifecycle, + Subject: "cn=kubelet-api", + Type: "client", + }) + } + { // Secret used by the kubelet // TODO: Can this be removed... at least from 1.6 on?