From ef8d24607eadc6e388b963f9f5098a5519fc6b72 Mon Sep 17 00:00:00 2001
From: Josh Deprez <jd@buildkite.com>
Date: Thu, 3 Oct 2024 18:10:29 +1000
Subject: [PATCH] Exit 94 if a mirror lock times out

---
 internal/job/checkout.go    | 28 ++++++++++++++++++++++++++--
 internal/job/executor.go    |  4 ++--
 internal/job/shell/shell.go | 10 +++++-----
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/internal/job/checkout.go b/internal/job/checkout.go
index e5caa25f80..1ee331fb32 100644
--- a/internal/job/checkout.go
+++ b/internal/job/checkout.go
@@ -165,11 +165,18 @@ func (e *Executor) CheckoutPhase(ctx context.Context) error {
 				return nil
 			}
 
+			var errLockTimeout ErrTimedOutAcquiringLock
 			switch {
 			case shell.IsExitError(err) && shell.GetExitCode(err) == -1:
 				e.shell.Warningf("Checkout was interrupted by a signal")
 				r.Break()
 
+			case errors.As(err, &errLockTimeout):
+				e.shell.Warningf("Checkout could not acquire the %s lock before timing out", errLockTimeout.Name)
+				r.Break()
+				// 94 chosen by fair die roll
+				return &shell.ExitError{Code: 94, Err: err}
+
 			case errors.Is(err, context.Canceled):
 				e.shell.Warningf("Checkout was cancelled")
 				r.Break()
@@ -304,7 +311,10 @@ func (e *Executor) updateGitMirror(ctx context.Context, repository string) (stri
 	defer canc()
 	mirrorCloneLock, err := e.shell.LockFile(cloneCtx, mirrorDir+".clonelock")
 	if err != nil {
-		return "", err
+		if errors.Is(err, context.DeadlineExceeded) {
+			return "", ErrTimedOutAcquiringLock{Name: "clone", Err: err}
+		}
+		return "", fmt.Errorf("unable to acquire clone lock: %w", err)
 	}
 	defer mirrorCloneLock.Unlock()
 
@@ -343,7 +353,10 @@ func (e *Executor) updateGitMirror(ctx context.Context, repository string) (stri
 	defer canc()
 	mirrorUpdateLock, err := e.shell.LockFile(updateCtx, mirrorDir+".updatelock")
 	if err != nil {
-		return "", err
+		if errors.Is(err, context.DeadlineExceeded) {
+			return "", ErrTimedOutAcquiringLock{Name: "update", Err: err}
+		}
+		return "", fmt.Errorf("unable to acquire update lock: %w", err)
 	}
 	defer mirrorUpdateLock.Unlock()
 
@@ -407,6 +420,17 @@ func (e *Executor) updateGitMirror(ctx context.Context, repository string) (stri
 	return mirrorDir, nil
 }
 
+type ErrTimedOutAcquiringLock struct {
+	Name string
+	Err  error
+}
+
+func (e ErrTimedOutAcquiringLock) Error() string {
+	return fmt.Sprintf("timed out acquiring %s lock: %v", e.Name, e.Err)
+}
+
+func (e ErrTimedOutAcquiringLock) Unwrap() error { return e.Err }
+
 // updateRemoteURL updates the URL for 'origin'. If gitDir == "", it assumes the
 // local repo is in the current directory, otherwise it includes --git-dir.
 // If the remote has changed, it logs some extra information. updateRemoteURL
diff --git a/internal/job/executor.go b/internal/job/executor.go
index 280bb02abd..090339e092 100644
--- a/internal/job/executor.go
+++ b/internal/job/executor.go
@@ -515,8 +515,8 @@ func (e *Executor) runWrappedShellScriptHook(ctx context.Context, hookName strin
 		// so it may inform the Buildkite API
 		if shell.IsExitError(err) {
 			return &shell.ExitError{
-				Code:    exitCode,
-				Message: fmt.Sprintf("The %s hook exited with status %d", hookName, exitCode),
+				Code: exitCode,
+				Err:  fmt.Errorf("The %s hook exited with status %d", hookName, exitCode),
 			}
 		}
 
diff --git a/internal/job/shell/shell.go b/internal/job/shell/shell.go
index 040a7b617c..cde8c198ad 100644
--- a/internal/job/shell/shell.go
+++ b/internal/job/shell/shell.go
@@ -657,11 +657,11 @@ func IsExitError(err error) bool {
 
 // ExitError is an error that carries a shell exit code
 type ExitError struct {
-	Code    int
-	Message string
+	Code int
+	Err  error
 }
 
 // Error returns the string message and fulfils the error interface
-func (ee *ExitError) Error() string {
-	return ee.Message
-}
+func (ee *ExitError) Error() string { return ee.Err.Error() }
+
+func (ee *ExitError) Unwrap() error { return ee.Err }