wal: failover write path code

failover_manager.go contains the failoverManager (which implements
wal.Manager) for the write path, and helper classes. dirProber monitors
the primary dir when failed over to use the secondary, to decide when to
failback to the primary. failoverMonitor uses the latency and error seen
by the current LogWriter, and probing state, to decide when to switch to
a different dir.

Informs #3230

Informs CRDB-35401

open.go

@@ -322,19 +322,18 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	})
 	walManager := &wal.StandaloneManager{}
 	err = walManager.Init(wal.Options{
-		Primary:                        wal.Dir{FS: opts.FS, Dirname: walDirname},
-		Secondary:                      wal.Dir{},
-		MinUnflushedWALNum:             wal.NumWAL(d.mu.versions.minUnflushedLogNum),
-		MaxNumRecyclableLogs:           opts.MemTableStopWritesThreshold + 1,
-		NoSyncOnClose:                  opts.NoSyncOnClose,
-		BytesPerSync:                   opts.WALBytesPerSync,
-		PreallocateSize:                d.walPreallocateSize,
-		MinSyncInterval:                opts.WALMinSyncInterval,
-		FsyncLatency:                   d.mu.log.metrics.fsyncLatency,
-		QueueSemChan:                   d.commit.logSyncQSem,
-		ElevatedWriteStallThresholdLag: 0,
-		Logger:                         opts.Logger,
-		EventListener:                  walEventListenerAdaptor{l: opts.EventListener},
+		Primary:              wal.Dir{FS: opts.FS, Dirname: walDirname},
+		Secondary:            wal.Dir{},
+		MinUnflushedWALNum:   wal.NumWAL(d.mu.versions.minUnflushedLogNum),
+		MaxNumRecyclableLogs: opts.MemTableStopWritesThreshold + 1,
+		NoSyncOnClose:        opts.NoSyncOnClose,
+		BytesPerSync:         opts.WALBytesPerSync,
+		PreallocateSize:      d.walPreallocateSize,
+		MinSyncInterval:      opts.WALMinSyncInterval,
+		FsyncLatency:         d.mu.log.metrics.fsyncLatency,
+		QueueSemChan:         d.commit.logSyncQSem,
+		Logger:               opts.Logger,
+		EventListener:        walEventListenerAdaptor{l: opts.EventListener},
 	})
 	if err != nil {
 		return nil, err

vfs/errorfs/errorfs.go

@@ -83,6 +83,8 @@ const (
 	OpFileSync
 	// OpFileSyncData describes a file sync operation.
 	OpFileSyncData
+	// OpFileSyncTo describes a file sync operation.
+	OpFileSyncTo
 	// OpFileFlush describes a file flush operation.
 	OpFileFlush
 )
@@ -92,7 +94,7 @@ func (o OpKind) ReadOrWrite() OpReadWrite {
 	switch o {
 	case OpOpen, OpOpenDir, OpList, OpStat, OpGetDiskUsage, OpFileRead, OpFileReadAt, OpFileStat:
 		return OpIsRead
-	case OpCreate, OpLink, OpRemove, OpRemoveAll, OpRename, OpReuseForWrite, OpMkdirAll, OpLock, OpFileClose, OpFileWrite, OpFileWriteAt, OpFileSync, OpFileSyncData, OpFileFlush, OpFilePreallocate:
+	case OpCreate, OpLink, OpRemove, OpRemoveAll, OpRename, OpReuseForWrite, OpMkdirAll, OpLock, OpFileClose, OpFileWrite, OpFileWriteAt, OpFileSync, OpFileSyncData, OpFileSyncTo, OpFileFlush, OpFilePreallocate:
 		return OpIsWrite
 	default:
 		panic(fmt.Sprintf("unrecognized op %v\n", o))
@@ -535,7 +537,9 @@ func (f *errorFile) SyncData() error {
 }
 
 func (f *errorFile) SyncTo(length int64) (fullSync bool, err error) {
-	// TODO(jackson): Consider error injection.
+	if err := f.inj.MaybeError(Op{Kind: OpFileSyncTo, Path: f.path}); err != nil {
+		return false, err
+	}
 	return f.file.SyncTo(length)
 }