Prevent overwriting existing host_uuid file

In some circumstances, multiple Teleport processes may be trying
to write the host_uuid file in the same data directory simultaneously.
The last of the writers would win, and any process using a host
UUID that did not match what ended up on disk could get into a perpertual
state of being unable to connect to the cluster.

To avoid the raciness, the host_uuid file writing process is no
longer a blind upsert. Instead, special care is taken to ensure
that there can only be a single writer, and that any subsequent
updates to the file are aborted and the first value written is
used instead.

lib/utils/utils.go

@@ -37,6 +37,7 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/google/renameio/v2"
 	"github.com/google/uuid"
 	"github.com/gravitational/trace"
 	log "github.com/sirupsen/logrus"
@@ -512,29 +513,75 @@ func WriteHostUUID(dataDir string, id string) error {
 // ReadOrMakeHostUUID looks for a hostid file in the data dir. If present,
 // returns the UUID from it, otherwise generates one
 func ReadOrMakeHostUUID(dataDir string) (string, error) {
-	id, err := ReadHostUUID(dataDir)
-	if err == nil {
+	hostUUIDFileName := GetHostUUIDPath(dataDir)
+	hostUUIDLock := hostUUIDFileName + ".lock"
+
+	iterationLimit := 3
+
+	for i := 0; i < iterationLimit; i++ {
+		id, err := ReadHostUUID(dataDir)
+		if err == nil {
+			return id, nil
+		}
+		if !trace.IsNotFound(err) {
+			return "", trace.Wrap(err)
+		}
+
+		// Checking error instead of the usual uuid.New() in case uuid generation
+		// fails due to not enough randomness. It's been known to happen happen when
+		// Teleport starts very early in the node initialization cycle and /dev/urandom
+		// isn't ready yet.
+		rawID, err := uuid.NewRandom()
+		if err != nil {
+			return "", trace.BadParameter("" +
+				"Teleport failed to generate host UUID. " +
+				"This may happen if randomness source is not fully initialized when the node is starting up. " +
+				"Please try restarting Teleport again.")
+		}
+
+		id = rawID.String()
+
+		writeFile := func() (string, error) {
+			f, err := os.OpenFile(hostUUIDLock, os.O_WRONLY|os.O_CREATE, 0o400)
+			if err != nil {
+				return "", trace.Wrap(err)
+			}
+
+			unlock, err := FSTryWriteLock(f.Name())
+			if err != nil {
+				return "", trace.Wrap(err)
+			}
+
+			defer func() {
+				os.Remove(hostUUIDLock)
+				unlock()
+			}()
+
+			if read, err := ReadHostUUID(dataDir); err == nil {
+				return read, nil
+			}
+
+			if err := renameio.WriteFile(hostUUIDFileName, []byte(id), 0o400); err != nil {
+				return "", trace.Wrap(err)
+			}
+
+			return id, nil
+		}
+
+		id, err = writeFile()
+		if err != nil {
+			if errors.Is(err, fs.ErrPermission) || errors.Is(err, ErrUnsuccessfulLockTry) {
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+
+			return "", trace.Wrap(err)
+		}
+
 		return id, nil
 	}
-	if !trace.IsNotFound(err) {
-		return "", trace.Wrap(err)
-	}
-	// Checking error instead of the usual uuid.New() in case uuid generation
-	// fails due to not enough randomness. It's been known to happen happen when
-	// Teleport starts very early in the node initialization cycle and /dev/urandom
-	// isn't ready yet.
-	rawID, err := uuid.NewRandom()
-	if err != nil {
-		return "", trace.BadParameter("" +
-			"Teleport failed to generate host UUID. " +
-			"This may happen if randomness source is not fully initialized when the node is starting up. " +
-			"Please try restarting Teleport again.")
-	}
-	id = rawID.String()
-	if err = WriteHostUUID(dataDir, id); err != nil {
-		return "", trace.Wrap(err)
-	}
-	return id, nil
+
+	return "", trace.LimitExceeded("failed to obtain host uuid")
 }
 
 // StringSliceSubset returns true if b is a subset of a.

lib/utils/utils_test.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -31,6 +32,7 @@ import (
 	"github.com/gravitational/trace"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	errgroup "golang.org/x/sync/errgroup"
 
 	"github.com/gravitational/teleport/api/utils/keys"
 	"github.com/gravitational/teleport/lib/fixtures"
@@ -42,6 +44,32 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+func TestReadOrMakeHostUUID(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	var wg errgroup.Group
+	concurrency := 4
+	ids := make([]string, concurrency)
+	barrier := make(chan struct{})
+
+	for i := 0; i < concurrency; i++ {
+		wg.Go(func() error {
+			<-barrier
+			id, err := ReadOrMakeHostUUID(dir)
+			ids[i] = id
+			return err
+		})
+	}
+
+	close(barrier)
+
+	require.NoError(t, wg.Wait())
+	require.Equal(t, slices.Repeat([]string{ids[0]}, concurrency), ids)
+	require.Equal(t, ids[0], ids[1])
+}
+
 func TestHostUUIDIdempotent(t *testing.T) {
 	t.Parallel()