Skip to content

Commit

Permalink
robustio: copy from cmd/go/internal/robustio (#239)
Browse files Browse the repository at this point in the history
Copied from Go commit b18b05881691861c4279a50010829150f1684fa9.
  • Loading branch information
rogpeppe authored Dec 13, 2023
1 parent fa6a31e commit 2c88e7f
Show file tree
Hide file tree
Showing 5 changed files with 220 additions and 0 deletions.
53 changes: 53 additions & 0 deletions robustio/robustio.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package robustio wraps I/O functions that are prone to failure on Windows,
// transparently retrying errors up to an arbitrary timeout.
//
// Errors are classified heuristically and retries are bounded, so the functions
// in this package do not completely eliminate spurious errors. However, they do
// significantly reduce the rate of failure in practice.
//
// If so, the error will likely wrap one of:
// The functions in this package do not completely eliminate spurious errors,
// but substantially reduce their rate of occurrence in practice.
package robustio

// Rename is like os.Rename, but on Windows retries errors that may occur if the
// file is concurrently read or overwritten.
//
// (See golang.org/issue/31247 and golang.org/issue/32188.)
func Rename(oldpath, newpath string) error {
return rename(oldpath, newpath)
}

// ReadFile is like os.ReadFile, but on Windows retries errors that may
// occur if the file is concurrently replaced.
//
// (See golang.org/issue/31247 and golang.org/issue/32188.)
func ReadFile(filename string) ([]byte, error) {
return readFile(filename)
}

// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur
// if an executable file in the directory has recently been executed.
//
// (See golang.org/issue/19491.)
func RemoveAll(path string) error {
return removeAll(path)
}

// IsEphemeralError reports whether err is one of the errors that the functions
// in this package attempt to mitigate.
//
// Errors considered ephemeral include:
// - syscall.ERROR_ACCESS_DENIED
// - syscall.ERROR_FILE_NOT_FOUND
// - internal/syscall/windows.ERROR_SHARING_VIOLATION
//
// This set may be expanded in the future; programs must not rely on the
// non-ephemerality of any given error.
func IsEphemeralError(err error) bool {
return isEphemeralError(err)
}
21 changes: 21 additions & 0 deletions robustio/robustio_darwin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package robustio

import (
"errors"
"syscall"
)

const errFileNotFound = syscall.ENOENT

// isEphemeralError returns true if err may be resolved by waiting.
func isEphemeralError(err error) bool {
var errno syscall.Errno
if errors.As(err, &errno) {
return errno == errFileNotFound
}
return false
}
91 changes: 91 additions & 0 deletions robustio/robustio_flaky.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build windows || darwin

package robustio

import (
"errors"
"math/rand"
"os"
"syscall"
"time"
)

const arbitraryTimeout = 2000 * time.Millisecond

// retry retries ephemeral errors from f up to an arbitrary timeout
// to work around filesystem flakiness on Windows and Darwin.
func retry(f func() (err error, mayRetry bool)) error {
var (
bestErr error
lowestErrno syscall.Errno
start time.Time
nextSleep time.Duration = 1 * time.Millisecond
)
for {
err, mayRetry := f()
if err == nil || !mayRetry {
return err
}

var errno syscall.Errno
if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) {
bestErr = err
lowestErrno = errno
} else if bestErr == nil {
bestErr = err
}

if start.IsZero() {
start = time.Now()
} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
break
}
time.Sleep(nextSleep)
nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
}

return bestErr
}

// rename is like os.Rename, but retries ephemeral errors.
//
// On Windows it wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
// MOVEFILE_REPLACE_EXISTING.
//
// Windows also provides a different system call, ReplaceFile,
// that provides similar semantics, but perhaps preserves more metadata. (The
// documentation on the differences between the two is very sparse.)
//
// Empirical error rates with MoveFileEx are lower under modest concurrency, so
// for now we're sticking with what the os package already provides.
func rename(oldpath, newpath string) (err error) {
return retry(func() (err error, mayRetry bool) {
err = os.Rename(oldpath, newpath)
return err, isEphemeralError(err)
})
}

// readFile is like os.ReadFile, but retries ephemeral errors.
func readFile(filename string) ([]byte, error) {
var b []byte
err := retry(func() (err error, mayRetry bool) {
b, err = os.ReadFile(filename)

// Unlike in rename, we do not retry errFileNotFound here: it can occur
// as a spurious error, but the file may also genuinely not exist, so the
// increase in robustness is probably not worth the extra latency.
return err, isEphemeralError(err) && !errors.Is(err, errFileNotFound)
})
return b, err
}

func removeAll(path string) error {
return retry(func() (err error, mayRetry bool) {
err = os.RemoveAll(path)
return err, isEphemeralError(err)
})
}
27 changes: 27 additions & 0 deletions robustio/robustio_other.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build !windows && !darwin

package robustio

import (
"os"
)

func rename(oldpath, newpath string) error {
return os.Rename(oldpath, newpath)
}

func readFile(filename string) ([]byte, error) {
return os.ReadFile(filename)
}

func removeAll(path string) error {
return os.RemoveAll(path)
}

func isEphemeralError(err error) bool {
return false
}
28 changes: 28 additions & 0 deletions robustio/robustio_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package robustio

import (
"errors"
"syscall"

"github.com/rogpeppe/go-internal/internal/syscall/windows"
)

const errFileNotFound = syscall.ERROR_FILE_NOT_FOUND

// isEphemeralError returns true if err may be resolved by waiting.
func isEphemeralError(err error) bool {
var errno syscall.Errno
if errors.As(err, &errno) {
switch errno {
case syscall.ERROR_ACCESS_DENIED,
syscall.ERROR_FILE_NOT_FOUND,
windows.ERROR_SHARING_VIOLATION:
return true
}
}
return false
}

0 comments on commit 2c88e7f

Please sign in to comment.