Retry git fetch commands.

prow/pod-utils/clone/clone.go

@@ -34,6 +34,10 @@ import (
 	"k8s.io/test-infra/prow/logrusutil"
 )
 
+type runnable interface {
+	run() (string, string, error)
+}
+
 // Run clones the refs under the prescribed directory and optionally
 // configures the git username and email in the repository as well.
 func Run(refs prowapi.Refs, dir, gitUserName, gitUserEmail, cookiePath string, env []string, oauthToken string) Record {
@@ -47,7 +51,7 @@ func Run(refs prowapi.Refs, dir, gitUserName, gitUserEmail, cookiePath string, e
 
 	// This function runs the provided commands in order, logging them as they run,
 	// aborting early and returning if any command fails.
-	runCommands := func(commands []cloneCommand) error {
+	runCommands := func(commands []runnable) error {
 		for _, command := range commands {
 			formattedCommand, output, err := command.run()
 			log := logrus.WithFields(logrus.Fields{"command": formattedCommand, "output": output})
@@ -142,11 +146,32 @@ func (g *gitCtx) gitCommand(args ...string) cloneCommand {
 	return cloneCommand{dir: g.cloneDir, env: g.env, command: "git", args: args}
 }
 
+var (
+	fetchRetries = []time.Duration{
+		100 * time.Millisecond,
+		200 * time.Millisecond,
+		400 * time.Millisecond,
+		800 * time.Millisecond,
+		2 * time.Second,
+	}
+)
+
+func (g *gitCtx) gitFetch(fetchArgs ...string) retryCommand {
+	args := []string{"fetch"}
+	args = append(args, fetchArgs...)
+
+	return retryCommand{
+		runnable: g.gitCommand(args...),
+		retries:  fetchRetries,
+	}
+}
+
 // commandsForBaseRef returns the list of commands needed to initialize and
 // configure a local git directory, as well as fetch and check out the provided
 // base ref.
-func (g *gitCtx) commandsForBaseRef(refs prowapi.Refs, gitUserName, gitUserEmail, cookiePath string) []cloneCommand {
-	commands := []cloneCommand{{dir: "/", env: g.env, command: "mkdir", args: []string{"-p", g.cloneDir}}}
+func (g *gitCtx) commandsForBaseRef(refs prowapi.Refs, gitUserName, gitUserEmail, cookiePath string) []runnable {
+	var commands []runnable
+	commands = append(commands, cloneCommand{dir: "/", env: g.env, command: "mkdir", args: []string{"-p", g.cloneDir}})
 
 	commands = append(commands, g.gitCommand("init"))
 	if gitUserName != "" {
@@ -160,11 +185,11 @@ func (g *gitCtx) commandsForBaseRef(refs prowapi.Refs, gitUserName, gitUserEmail
 	}
 
 	if refs.CloneDepth > 0 {
-		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, "--tags", "--prune", "--depth", strconv.Itoa(refs.CloneDepth)))
-		commands = append(commands, g.gitCommand("fetch", "--depth", strconv.Itoa(refs.CloneDepth), g.repositoryURI, refs.BaseRef))
+		commands = append(commands, g.gitFetch(g.repositoryURI, "--tags", "--prune", "--depth", strconv.Itoa(refs.CloneDepth)))
+		commands = append(commands, g.gitFetch("--depth", strconv.Itoa(refs.CloneDepth), g.repositoryURI, refs.BaseRef))
 	} else {
-		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, "--tags", "--prune"))
-		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, refs.BaseRef))
+		commands = append(commands, g.gitFetch(g.repositoryURI, "--tags", "--prune"))
+		commands = append(commands, g.gitFetch(g.repositoryURI, refs.BaseRef))
 	}
 	var target string
 	if refs.BaseSHA != "" {
@@ -231,14 +256,14 @@ func (g *gitCtx) gitRevParse() (string, error) {
 // It's recommended that fakeTimestamp be set to the timestamp of the base ref.
 // This enables reproducible timestamps and git tree digests every time the same
 // set of base and pull refs are used.
-func (g *gitCtx) commandsForPullRefs(refs prowapi.Refs, fakeTimestamp int) []cloneCommand {
-	var commands []cloneCommand
+func (g *gitCtx) commandsForPullRefs(refs prowapi.Refs, fakeTimestamp int) []runnable {
+	var commands []runnable
 	for _, prRef := range refs.Pulls {
 		ref := fmt.Sprintf("pull/%d/head", prRef.Number)
 		if prRef.Ref != "" {
 			ref = prRef.Ref
 		}
-		commands = append(commands, g.gitCommand("fetch", g.repositoryURI, ref))
+		commands = append(commands, g.gitFetch(g.repositoryURI, ref))
 		var prCheckout string
 		if prRef.SHA != "" {
 			prCheckout = prRef.SHA
@@ -259,15 +284,39 @@ func (g *gitCtx) commandsForPullRefs(refs prowapi.Refs, fakeTimestamp int) []clo
 	return commands
 }
 
+type retryCommand struct {
+	runnable
+	retries []time.Duration
+}
+
+func (rc retryCommand) run() (string, string, error) {
+	cmd, out, err := rc.runnable.run()
+	if err == nil {
+		return cmd, out, err
+	}
+	for _, dur := range rc.retries {
+		logrus.WithError(err).WithFields(logrus.Fields{
+			"sleep":   dur,
+			"command": cmd,
+		}).Info("Retrying after sleep")
+		time.Sleep(dur)
+		cmd, out, err = rc.runnable.run()
+		if err == nil {
+			break
+		}
+	}
+	return cmd, out, err
+}
+
 type cloneCommand struct {
 	dir     string
 	env     []string
 	command string
 	args    []string
 }
 
-func (c *cloneCommand) run() (string, string, error) {
-	output := bytes.Buffer{}
+func (c cloneCommand) run() (string, string, error) {
+	var output bytes.Buffer
 	cmd := exec.Command(c.command, c.args...)
 	cmd.Dir = c.dir
 	cmd.Env = append(cmd.Env, c.env...)
@@ -277,6 +326,6 @@ func (c *cloneCommand) run() (string, string, error) {
 	return strings.Join(append([]string{c.command}, c.args...), " "), output.String(), err
 }
 
-func (c *cloneCommand) String() string {
+func (c cloneCommand) String() string {
 	return fmt.Sprintf("PWD=%s %s %s %s", c.dir, strings.Join(c.env, " "), c.command, strings.Join(c.env, " "))
 }