Merge pull request #8577 from justinsb/dump

Capture logs from a kops cluster

cmd/kops/BUILD.bazel

@@ -66,6 +66,7 @@ go_library(
         "//pkg/client/simple:go_default_library",
         "//pkg/cloudinstances:go_default_library",
         "//pkg/commands:go_default_library",
+        "//pkg/dump:go_default_library",
         "//pkg/edit:go_default_library",
         "//pkg/featureflag:go_default_library",
         "//pkg/formatter:go_default_library",
@@ -94,6 +95,7 @@ go_library(
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/cobra/doc:go_default_library",
         "//vendor/github.com/spf13/viper:go_default_library",
+        "//vendor/golang.org/x/crypto/ssh:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kops/toolbox_dump.go

@@ -21,10 +21,21 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
 	"k8s.io/kops/cmd/kops/util"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/dump"
 	"k8s.io/kops/pkg/resources"
 	resourceops "k8s.io/kops/pkg/resources/ops"
 	"k8s.io/kops/upup/pkg/fi/cloudup"
@@ -48,10 +59,14 @@ type ToolboxDumpOptions struct {
 	Output string
 
 	ClusterName string
+
+	Dir        string
+	PrivateKey string
 }
 
 func (o *ToolboxDumpOptions) InitDefaults() {
 	o.Output = OutputYaml
+	o.PrivateKey = "~/.ssh/id_rsa"
 }
 
 func NewCmdToolboxDump(f *util.Factory, out io.Writer) *cobra.Command {
@@ -83,10 +98,14 @@ func NewCmdToolboxDump(f *util.Factory, out io.Writer) *cobra.Command {
 	// Yes please! (@kris-nova)
 	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "output format.  One of: yaml, json")
 
+	cmd.Flags().StringVar(&options.Dir, "dir", options.Dir, "target directory; if specified will collect logs and other information.")
+	cmd.Flags().StringVar(&options.PrivateKey, "private-key", options.PrivateKey, "private key to use for SSH acccess to instances")
+
 	return cmd
 }
 
 func RunToolboxDump(ctx context.Context, f *util.Factory, out io.Writer, options *ToolboxDumpOptions) error {
+
 	clientset, err := f.Clientset()
 	if err != nil {
 		return err
@@ -115,14 +134,85 @@ func RunToolboxDump(ctx context.Context, f *util.Factory, out io.Writer, options
 	if err != nil {
 		return err
 	}
-	dump, err := resources.BuildDump(context.TODO(), cloud, resourceMap)
+	d, err := resources.BuildDump(ctx, cloud, resourceMap)
 	if err != nil {
 		return err
 	}
 
+	if options.Dir != "" {
+		privateKeyPath := options.PrivateKey
+		if strings.HasPrefix(privateKeyPath, "~/") {
+			privateKeyPath = filepath.Join(os.Getenv("HOME"), privateKeyPath[2:])
+		}
+		key, err := ioutil.ReadFile(privateKeyPath)
+		if err != nil {
+			return fmt.Errorf("error reading private key %q: %v", privateKeyPath, err)
+		}
+
+		signer, err := ssh.ParsePrivateKey(key)
+		if err != nil {
+			return fmt.Errorf("error parsing private key %q: %v", privateKeyPath, err)
+		}
+
+		cluster, err := GetCluster(ctx, f, options.ClusterName)
+		if err != nil {
+			return err
+		}
+
+		contextName := cluster.ObjectMeta.Name
+		clientGetter := genericclioptions.NewConfigFlags(true)
+		clientGetter.Context = &contextName
+
+		var nodes corev1.NodeList
+
+		config, err := clientGetter.ToRESTConfig()
+		if err != nil {
+			klog.Warningf("cannot load kubecfg settings for %q: %v", contextName, err)
+		} else {
+			k8sClient, err := kubernetes.NewForConfig(config)
+			if err != nil {
+				klog.Warningf("cannot build kube client for %q: %v", contextName, err)
+			} else {
+
+				nodeList, err := k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+				if err != nil {
+					klog.Warningf("error listing nodes in cluster: %v", err)
+				} else {
+					nodes = *nodeList
+				}
+			}
+		}
+
+		// TODO: We need to find the correct SSH user, ideally per IP
+		sshUser := "ubuntu"
+		sshConfig := &ssh.ClientConfig{
+			User: sshUser,
+			Auth: []ssh.AuthMethod{
+				ssh.PublicKeys(signer),
+			},
+			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		}
+
+		dumper := dump.NewLogDumper(sshConfig, options.Dir)
+
+		var additionalIPs []string
+		for _, instance := range d.Instances {
+			if len(instance.PublicAddresses) != 0 {
+				additionalIPs = append(additionalIPs, instance.PublicAddresses[0])
+				continue
+			}
+
+			klog.Warningf("no public IP for node %q", instance.Name)
+		}
+
+		if err := dumper.DumpAllNodes(ctx, nodes, additionalIPs); err != nil {
+			return fmt.Errorf("error dumping nodes: %v", err)
+		}
+	}
+
 	switch options.Output {
 	case OutputYaml:
-		b, err := kops.ToRawYaml(dump)
+		b, err := kops.ToRawYaml(d)
 		if err != nil {
 			return fmt.Errorf("error marshaling yaml: %v", err)
 		}
@@ -133,7 +223,7 @@ func RunToolboxDump(ctx context.Context, f *util.Factory, out io.Writer, options
 		return nil
 
 	case OutputJSON:
-		b, err := json.MarshalIndent(dump, "", "  ")
+		b, err := json.MarshalIndent(d, "", "  ")
 		if err != nil {
 			return fmt.Errorf("error marshaling json: %v", err)
 		}

docs/cli/kops_toolbox_dump.md

@@ -23,8 +23,10 @@ kops toolbox dump [flags]
 ### Options
 
 ```
-  -h, --help            help for dump
-  -o, --output string   output format.  One of: yaml, json (default "yaml")
+      --dir string           target directory; if specified will collect logs and other information.
+  -h, --help                 help for dump
+  -o, --output string        output format.  One of: yaml, json (default "yaml")
+      --private-key string   private key to use for SSH acccess to instances (default "~/.ssh/id_rsa")
 ```
 
 ### Options inherited from parent commands

hack/.packages

@@ -85,6 +85,7 @@ k8s.io/kops/pkg/commands
 k8s.io/kops/pkg/configbuilder
 k8s.io/kops/pkg/diff
 k8s.io/kops/pkg/dns
+k8s.io/kops/pkg/dump
 k8s.io/kops/pkg/edit
 k8s.io/kops/pkg/featureflag
 k8s.io/kops/pkg/flagbuilder

pkg/dump/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dumper.go"],
+    importpath = "k8s.io/kops/pkg/dump",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/golang.org/x/crypto/ssh:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)