From 73041abbf8be87af54ea027dc8a061e77e32ab7a Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 16 Jul 2020 21:27:23 -0700 Subject: [PATCH] Send bootstrap query from nodeup to kops-controller --- cmd/kops-controller/pkg/server/BUILD.bazel | 7 +- cmd/kops-controller/pkg/server/server.go | 62 ++++++++- nodeup/pkg/model/BUILD.bazel | 1 + nodeup/pkg/model/bootstrap_client.go | 46 +++++++ pkg/apis/nodeup/BUILD.bazel | 5 +- pkg/apis/nodeup/bootstrap.go | 29 +++++ upup/pkg/fi/nodeup/command.go | 3 +- upup/pkg/fi/nodeup/nodetasks/BUILD.bazel | 3 + .../fi/nodeup/nodetasks/bootstrap_client.go | 123 ++++++++++++++++++ 9 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 nodeup/pkg/model/bootstrap_client.go create mode 100644 pkg/apis/nodeup/bootstrap.go create mode 100644 upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 831fee41cdb05..9c65dc3f40cc0 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -5,5 +5,10 @@ go_library( srcs = ["server.go"], importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", visibility = ["//visibility:public"], - deps = ["//cmd/kops-controller/pkg/config:go_default_library"], + deps = [ + "//cmd/kops-controller/pkg/config:go_default_library", + "//pkg/apis/nodeup:go_default_library", + "//vendor/github.com/gorilla/mux:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], ) diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 50cc46b617c8a..ad24a90f9a38c 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -18,9 +18,15 @@ package server import ( "crypto/tls" + "encoding/json" + "fmt" "net/http" + "runtime/debug" + "github.com/gorilla/mux" + "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" + "k8s.io/kops/pkg/apis/nodeup" ) type Server struct { @@ -36,12 +42,64 @@ func NewServer(opt *config.Options) (*Server, error) { PreferServerCipherSuites: true, }, } - return &Server{ + + s := &Server{ opt: opt, server: server, - }, nil + } + r := mux.NewRouter() + r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap)) + server.Handler = recovery(r) + + return s, nil } func (s *Server) Start() error { return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) } + +func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { + if r.Body == nil { + klog.Infof("bootstrap %s no body", r.RemoteAddr) + w.WriteHeader(http.StatusBadRequest) + return + } + + // TODO: authenticate request + + req := &nodeup.BootstrapRequest{} + err := json.NewDecoder(r.Body).Decode(req) + if err != nil { + klog.Infof("bootstrap %s decode err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("failed to decode: %v", err))) + return + } + + if req.APIVersion != nodeup.BootstrapAPIVersion { + klog.Infof("bootstrap %s wrong APIVersion", r.RemoteAddr) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("unexpected APIVersion")) + return + } + + w.Header().Set("Content-Type", "application/json") + resp := &nodeup.BootstrapResponse{} + _ = json.NewEncoder(w).Encode(resp) + klog.Infof("bootstrap %s success", r.RemoteAddr) +} + +// recovery is responsible for ensuring we don't exit on a panic. +func recovery(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + + klog.Errorf("failed to handle request: threw exception: %v: %s", err, debug.Stack()) + } + }() + + next.ServeHTTP(w, req) + }) +} diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index ac26b90d5047d..1c393a5055b55 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "architecture.go", + "bootstrap_client.go", "cloudconfig.go", "containerd.go", "context.go", diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go new file mode 100644 index 0000000000000..699a901d54467 --- /dev/null +++ b/nodeup/pkg/model/bootstrap_client.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// SecretBuilder writes secrets +type BootstrapClientBuilder struct { + *NodeupModelContext +} + +func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { + if b.IsMaster || !b.UseKopsControllerForNodeBootstrap() { + return nil + } + + cert, err := b.GetCert(fi.CertificateIDCA) + if err != nil { + return err + } + + bootstrapClient := &nodetasks.BootstrapClient{ + CA: cert, + } + c.AddTask(bootstrapClient) + return nil +} + +var _ fi.ModelBuilder = &BootstrapClientBuilder{} diff --git a/pkg/apis/nodeup/BUILD.bazel b/pkg/apis/nodeup/BUILD.bazel index 36622381b987b..21643fe611128 100644 --- a/pkg/apis/nodeup/BUILD.bazel +++ b/pkg/apis/nodeup/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["config.go"], + srcs = [ + "bootstrap.go", + "config.go", + ], importpath = "k8s.io/kops/pkg/apis/nodeup", visibility = ["//visibility:public"], deps = [ diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go new file mode 100644 index 0000000000000..e2b8119b7b761 --- /dev/null +++ b/pkg/apis/nodeup/bootstrap.go @@ -0,0 +1,29 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeup + +const BootstrapAPIVersion = "bootstrap.kops.k8s.io/v1alpha1" + +// BootstrapRequest is a request from nodeup to kops-controller for bootstrapping a node. +type BootstrapRequest struct { + // APIVersion defines the versioned schema of this representation of a request. + APIVersion string `json:"apiVersion"` +} + +// BootstrapRespose is a response to a BootstrapRequest. +type BootstrapResponse struct { +} diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 022ed5a2ee6c7..f9bbd5d6df177 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -237,6 +237,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } loader := &Loader{} + loader.Builders = append(loader.Builders, &model.BootstrapClientBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.NTPBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.MiscUtilsBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.DirectoryBuilder{NodeupModelContext: modelContext}) @@ -305,7 +306,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { return fmt.Errorf("unsupported target type %q", c.Target) } - context, err := fi.NewContext(target, nil, cloud, keyStore, secretStore, configBase, checkExisting, taskMap) + context, err := fi.NewContext(target, c.cluster, cloud, keyStore, secretStore, configBase, checkExisting, taskMap) if err != nil { klog.Exitf("error building context: %v", err) } diff --git a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel index 33f9f08259789..192664b2f13f8 100644 --- a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel +++ b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "archive.go", "bindmount.go", + "bootstrap_client.go", "chattr.go", "createsdir.go", "file.go", @@ -21,9 +22,11 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apis/kops:go_default_library", + "//pkg/apis/nodeup:go_default_library", "//pkg/backoff:go_default_library", "//pkg/kubeconfig:go_default_library", "//pkg/pki:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/nodeup/cloudinit:go_default_library", "//upup/pkg/fi/nodeup/local:go_default_library", diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go new file mode 100644 index 0000000000000..fc5ca905b6e91 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -0,0 +1,123 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "strconv" + + "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/wellknownports" + "k8s.io/kops/upup/pkg/fi" +) + +type BootstrapClient struct { + // CA is the CA certificate for kops-controller. + CA []byte + + client *http.Client +} + +var _ fi.Task = &BootstrapClient{} +var _ fi.HasName = &BootstrapClient{} + +func (b *BootstrapClient) GetName() *string { + name := "BootstrapClient" + return &name +} + +func (b *BootstrapClient) String() string { + return "BootstrapClient" +} + +func (b *BootstrapClient) Run(c *fi.Context) error { + req := nodeup.BootstrapRequest{ + APIVersion: nodeup.BootstrapAPIVersion, + } + + err := b.queryBootstrap(c, req) + if err != nil { + return err + } + + return nil +} + +func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequest) error { + if b.client == nil { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(b.CA) + + b.client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + MinVersion: tls.VersionTLS12, + }, + }, + } + } + + reqBytes, err := json.Marshal(req) + if err != nil { + return err + } + + bootstrapUrl := url.URL{ + Scheme: "https", + Host: net.JoinHostPort(c.Cluster.Spec.MasterInternalName, strconv.Itoa(wellknownports.KopsControllerPort)), + Path: "/bootstrap", + } + resp, err := b.client.Post(bootstrapUrl.String(), "application/json", bytes.NewReader(reqBytes)) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + detail := "" + if resp.Body != nil { + scanner := bufio.NewScanner(resp.Body) + if scanner.Scan() { + detail = scanner.Text() + } + _ = resp.Body.Close() + } + return fmt.Errorf("bootstrap returned status code %d: %s", resp.StatusCode, detail) + } + + var bootstrapResp nodeup.BootstrapResponse + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + err = json.Unmarshal(body, &bootstrapResp) + if err != nil { + return err + } + + return nil +}