cmd/openshift-install/gather: Recognize "connection refused"

Before this commit, bootstrap machines that failed to come up would
look like [1]:

  level=info msg="Waiting up to 30m0s for the Kubernetes API at https://api.ci-op-6266tp8r-77109.origin-ci-int-aws.dev.rhcloud.com:6443..."
  level=error msg="Attempted to gather ClusterOperator status after installation failure: listing ClusterOperator objects: Get https://api.ci-op-6266tp8r-77109.origin-ci-int-aws.dev.rhcloud.com:6443/apis/config.openshift.io/v1/clusteroperators: dial tcp 3.221.214.197:6443: connect: connection refused"
  level=info msg="Pulling debug logs from the bootstrap machine"
  level=error msg="Attempted to gather debug logs after installation failure: failed to create SSH client, ensure the proper ssh key is in your keyring or specify with --key: dial tcp 3.84.188.207:22: connect: connection refused"
  level=fatal msg="Bootstrap failed to complete: waiting for Kubernetes API: context deadline exceeded"

With this commit, that last error will look like:

  level=error msg="Attempted to gather debug logs after installation failure: failed to connect to the bootstrap machine: dial tcp 3.84.188.207:22: connect: connection refused"

without the unrelated (to this failure mode) distraction about SSH
keys.

[1]: https://prow.svc.ci.openshift.org/view/gcs/origin-ci-test/logs/release-openshift-origin-installer-e2e-aws-upgrade/12076

Updated the commit to match with the latest changes.

cmd/openshift-install/gather.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -142,11 +143,13 @@ func runGatherBootstrapCmd(directory string) error {
 func logGatherBootstrap(bootstrap string, port int, masters []string, directory string) error {
 	logrus.Info("Pulling debug logs from the bootstrap machine")
 	client, err := ssh.NewClient("core", net.JoinHostPort(bootstrap, strconv.Itoa(port)), gatherBootstrapOpts.sshKeys)
-	if err != nil && strings.Contains(err.Error(), "ssh: handshake failed: ssh: unable to authenticate") {
-		return errors.Wrap(err, "failed to create SSH client, ensure the private key is added to your authentication agent (ssh-agent) or specified with the --key parameter")
-	} else if err != nil {
+	if err != nil {
+		if errors.Is(err, syscall.ECONNREFUSED) {
+			return errors.Wrap(err, "failed to connect to the bootstrap machine")
+		}
 		return errors.Wrap(err, "failed to create SSH client")
 	}
+
 	gatherID := time.Now().Format("20060102150405")
 	if err := ssh.Run(client, fmt.Sprintf("/usr/local/bin/installer-gather.sh --id %s %s", gatherID, strings.Join(masters, " "))); err != nil {
 		return errors.Wrap(err, "failed to run remote command")

pkg/gather/ssh/agent.go

@@ -13,13 +13,12 @@ import (
 
 // getAgent attempts to connect to the running SSH agent, returning a newly
 // initialized static agent if that fails.
-func getAgent(keys []string) (agent.Agent, error) {
-	// Attempt to use the existing SSH agent if it's configured and no keys
-	// were explicitly passed.
-	if authSock := os.Getenv("SSH_AUTH_SOCK"); authSock != "" && len(keys) == 0 {
+func getAgent(keys []string) (agent.Agent, string, error) {
+	// Attempt to use the existing SSH agent if it's configured or use the default ssh pair generated.
+	if authSock := os.Getenv("SSH_AUTH_SOCK"); authSock != "" {
 		logrus.Debugf("Using SSH_AUTH_SOCK %s to connect to an existing agent", authSock)
 		if conn, err := net.Dial("unix", authSock); err == nil {
-			return agent.NewClient(conn), nil
+			return agent.NewClient(conn), "agent", nil
 		}
 	}
 
@@ -28,13 +27,10 @@ func getAgent(keys []string) (agent.Agent, error) {
 
 // newAgent initializes an SSH Agent with the keys.
 // If no keys are provided, it loads all the keys from the user's environment.
-func newAgent(keyPaths []string) (agent.Agent, error) {
+func newAgent(keyPaths []string) (agent.Agent, string, error) {
 	keys, err := loadKeys(keyPaths)
 	if err != nil {
-		return nil, err
-	}
-	if len(keys) == 0 {
-		return nil, errors.New("no keys found for SSH agent")
+		return nil, "", err
 	}
 
 	ag := agent.NewKeyring()
@@ -46,9 +42,9 @@ func newAgent(keyPaths []string) (agent.Agent, error) {
 		logrus.Debugf("Added %s to installer's internal agent", name)
 	}
 	if agg := utilerrors.NewAggregate(errs); agg != nil {
-		return nil, agg
+		return nil, "", agg
 	}
-	return ag, nil
+	return ag, "keys", nil
 }
 
 func loadKeys(paths []string) (map[string]interface{}, error) {

pkg/gather/ssh/ssh.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/openshift/installer/pkg/lineprinter"
 	"github.com/pkg/errors"
@@ -20,7 +21,7 @@ import (
 //
 // if keys list is empty, it tries to load the keys from the user's environment.
 func NewClient(user, address string, keys []string) (*ssh.Client, error) {
-	ag, err := getAgent(keys)
+	ag, agentType, err := getAgent(keys)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to initialize the SSH agent")
 	}
@@ -36,6 +37,12 @@ func NewClient(user, address string, keys []string) (*ssh.Client, error) {
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 	})
 	if err != nil {
+		if strings.Contains(err.Error(), "ssh: handshake failed: ssh: unable to authenticate") {
+			if agentType == "agent" {
+				return nil, errors.Wrap(err, "failed to use pre-existing agent, make sure the appropriate keys exist in the agent for authentication")
+			}
+			return nil, errors.Wrap(err, "failed to use the provided keys for authentication")
+		}
 		return nil, err
 	}
 	if err := agent.ForwardToAgent(client, ag); err != nil {