Custom Fallback TOML Config

This commit provides setting a new env var `CL_CHAIN_FALLBACK` as a path to a custom `fallback.toml`. This allows
plugins to define their own set of fallback options apart from the core node which override the default fallback
options.

core/chains/evm/config/toml/defaults.go

@@ -3,8 +3,8 @@ package toml
 import (
 	"bytes"
 	"embed"
+	"errors"
 	"fmt"
-	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -19,7 +19,6 @@ import (
 )
 
 var (
-
 	//go:embed defaults/*.toml
 	defaultsFS   embed.FS
 	fallback     Chain
@@ -33,48 +32,83 @@ var (
 )
 
 func init() {
-	// read the defaults first
+	// read all default configs
+	initReadDefaults()
+
+	// overrides should be applied as:
+	// fallback.toml(defaults dir) <- fallback.toml(env CL_CHAIN_FALLBACK) <- ChainSpecific.toml(env CL_CHAIN_DEFAULTS)
+	//
+	// the custom fallback gets processed and overrides the default fallback
+	if path := env.CustomFallback.Get(); path != "" {
+		_, chain, err := readConfig(path, os.ReadFile, true)
+		if err != nil {
+			if !errors.Is(err, errFallbackConfig) {
+				log.Fatalf("custom fallback config error: %s", err.Error())
+			}
+
+			fallback = chain
+		}
+	}
+
+	// check for and apply any overrides
+	initApplyEVMOverrides()
+}
 
+var (
+	errRead           = errors.New("error reading file")
+	errDecode         = errors.New("error in TOML decoding")
+	errMissingChainID = errors.New("missing ChainID")
+	errNonNilChainID  = errors.New("fallback ChainID must be nil")
+	errFallbackConfig = errors.New("fallback config")
+)
+
+func initReadDefaults() {
+	// read the defaults first
 	fes, err := defaultsFS.ReadDir("defaults")
 	if err != nil {
-		log.Fatalf("failed to read defaults/: %v", err)
+		log.Fatalf("failed to read defaults: %v", err)
 	}
+
 	for _, fe := range fes {
-		path := filepath.Join("defaults", fe.Name())
-		b, err2 := defaultsFS.ReadFile(path)
-		if err2 != nil {
-			log.Fatalf("failed to read %q: %v", path, err2)
+		if fe.IsDir() {
+			// Skip directories
+			continue
 		}
-		var config = struct {
-			ChainID *big.Big
-			Chain
-		}{}
 
-		if err3 := cconfig.DecodeTOML(bytes.NewReader(b), &config); err3 != nil {
-			log.Fatalf("failed to decode %q: %v", path, err3)
-		}
-		if fe.Name() == "fallback.toml" {
-			if config.ChainID != nil {
-				log.Fatalf("fallback ChainID must be nil, not: %s", config.ChainID)
+		// read the file to bytes
+		path := filepath.Join("defaults", fe.Name())
+		chainID, chain, err := readConfig(path, defaultsFS.ReadFile, fe.Name() == "fallback.toml")
+		if err != nil {
+			if errors.Is(err, errFallbackConfig) {
+				fallback = chain
+
+				continue
 			}
-			fallback = config.Chain
-			continue
-		}
-		if config.ChainID == nil {
-			log.Fatalf("missing ChainID: %s", path)
+
+			log.Fatal(err.Error())
 		}
-		DefaultIDs = append(DefaultIDs, config.ChainID)
-		id := config.ChainID.String()
+
+		// add ChainID to set of default IDs
+		DefaultIDs = append(DefaultIDs, chainID)
+
+		// ChainID as a default should not be duplicated
+		id := chainID.String()
 		if _, ok := defaults[id]; ok {
 			log.Fatalf("%q contains duplicate ChainID: %s", path, id)
 		}
-		defaults[id] = config.Chain
+
+		// set default lookups
+		defaults[id] = chain
 		defaultNames[id] = strings.ReplaceAll(strings.TrimSuffix(fe.Name(), ".toml"), "_", " ")
 	}
+
+	// sort default IDs in numeric order
 	slices.SortFunc(DefaultIDs, func(a, b *big.Big) int {
 		return a.Cmp(b)
 	})
+}
 
+func initApplyEVMOverrides() {
 	// read the custom defaults overrides
 	dir := env.CustomDefaults.Get()
 	if dir == "" {
@@ -98,39 +132,54 @@ func init() {
 			continue
 		}
 
+		// read the file to bytes
 		path := evmDir + "/" + entry.Name()
-		file, err := os.Open(path)
+		chainID, chain, err := readConfig(path, os.ReadFile, false)
 		if err != nil {
-			log.Fatalf("error opening file (name: %v) in custom defaults override directory: %v", entry.Name(), err)
+			log.Fatalf("custom defaults override failure (%s): %s", entry.Name(), err.Error())
 		}
 
-		// Read file contents
-		b, err := io.ReadAll(file)
-		file.Close()
-		if err != nil {
-			log.Fatalf("error reading file (name: %v) contents in custom defaults override directory: %v", entry.Name(), err)
+		// ChainID as a default should not be duplicated
+		id := chainID.String()
+		if _, ok := customDefaults[id]; ok {
+			log.Fatalf("%q contains duplicate ChainID: %s", path, id)
 		}
 
-		var config = struct {
-			ChainID *big.Big
-			Chain
-		}{}
+		// set default lookups
+		customDefaults[id] = chain
+	}
+}
 
-		if err := cconfig.DecodeTOML(bytes.NewReader(b), &config); err != nil {
-			log.Fatalf("failed to decode %q in custom defaults override directory: %v", path, err)
-		}
+func readConfig(path string, reader func(name string) ([]byte, error), fallback bool) (*big.Big, Chain, error) {
+	bts, err := reader(path)
+	if err != nil {
+		return nil, Chain{}, fmt.Errorf("%w: %s", errRead, err.Error())
+	}
 
-		if config.ChainID == nil {
-			log.Fatalf("missing ChainID in: %s in custom defaults override directory. exiting", path)
-		}
+	var config = struct {
+		ChainID *big.Big
+		Chain
+	}{}
 
-		id := config.ChainID.String()
+	// decode from toml to a chain config
+	if err := cconfig.DecodeTOML(bytes.NewReader(bts), &config); err != nil {
+		return nil, Chain{}, fmt.Errorf("%w %q: %v", errDecode, path, err)
+	}
 
-		if _, ok := customDefaults[id]; ok {
-			log.Fatalf("%q contains duplicate ChainID: %s", path, id)
+	if fallback {
+		if config.ChainID != nil {
+			return nil, Chain{}, fmt.Errorf("%w: found: %s", errNonNilChainID, config.ChainID)
 		}
-		customDefaults[id] = config.Chain
+
+		return nil, config.Chain, errFallbackConfig
 	}
+
+	// ensure ChainID is set
+	if config.ChainID == nil {
+		return nil, Chain{}, fmt.Errorf("%w: %s", errMissingChainID, path)
+	}
+
+	return config.ChainID, config.Chain, nil
 }
 
 // DefaultsNamed returns the default Chain values, optionally for the given chainID, as well as a name if the chainID is known.

core/config/env/env.go

@@ -21,6 +21,7 @@ var (
 	// Migrations env vars
 	EVMChainIDNotNullMigration0195 = "CL_EVM_CHAINID_NOT_NULL_MIGRATION_0195"
 	CustomDefaults                 = Var("CL_CHAIN_DEFAULTS")
+	CustomFallback                 = Var("CL_CHAIN_FALLBACK")
 )
 
 // LOOPP commands and vars

testdata/scripts/node/validate/defaults-override.txtar

@@ -17,10 +17,113 @@ env CL_CHAIN_DEFAULTS=default_overrides3
 ! exec chainlink node -c config.toml -s secrets.toml validate
 stderr 'contains duplicate ChainID'
 
+# test with fallback override
+env CL_CHAIN_DEFAULTS=default_overrides
+env CL_CHAIN_FALLBACK=default_overrides/fallback.toml
+exec chainlink node -c config.toml -s secrets.toml validate
+! cmp stdout out.txt
+
 -- default_overrides/evm/Ethereum_Mainnet.toml --
 ChainID = '1'
 FinalityTagEnabled = false
 
+-- default_overrides/fallback.toml --
+AutoCreateKey = true
+BlockBackfillDepth = 1000000
+BlockBackfillSkip = false
+FinalityDepth = 50
+FinalityTagEnabled = false
+LogBackfillBatchSize = 1000
+LogPollInterval = '15s'
+LogKeepBlocksDepth = 100000
+LogPrunePageSize = 0
+BackupLogPollerBlockDelay = 100
+MinContractPayment = '.00001 link'
+MinIncomingConfirmations = 3
+NonceAutoSync = true
+NoNewHeadsThreshold = '3m'
+RPCDefaultBatchSize = 250
+RPCBlockQueryDelay = 1
+FinalizedBlockOffset = 0
+NoNewFinalizedHeadsThreshold = '0'
+LogBroadcasterEnabled = true
+
+[Transactions]
+ForwardersEnabled = false
+MaxInFlight = 16
+MaxQueued = 250
+ReaperInterval = '1h'
+ReaperThreshold = '168h'
+ResendAfterThreshold = '1m'
+
+[Transactions.AutoPurge]
+Enabled = false
+
+[BalanceMonitor]
+Enabled = true
+
+[GasEstimator]
+Mode = 'BlockHistory'
+PriceDefault = '20 gwei'
+PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether'
+PriceMin = '1 gwei'
+LimitDefault = 500_000
+LimitMax = 500_000
+LimitMultiplier = '1'
+LimitTransfer = 21_000
+BumpMin = '5 gwei'
+BumpPercent = 20
+BumpThreshold = 3
+EIP1559DynamicFees = false
+FeeCapDefault = '100 gwei'
+TipCapDefault = '1'
+TipCapMin = '1'
+EstimateLimit = false
+
+[GasEstimator.BlockHistory]
+BatchSize = 25
+BlockHistorySize = 8
+CheckInclusionBlocks = 12
+CheckInclusionPercentile = 90
+TransactionPercentile = 60
+
+[GasEstimator.FeeHistory]
+CacheTimeout = '10s'
+
+[HeadTracker]
+HistoryDepth = 100
+MaxBufferSize = 3
+SamplingInterval = '1s'
+FinalityTagBypass = true
+MaxAllowedFinalityDepth = 10000
+PersistenceEnabled = true
+
+[NodePool]
+PollFailureThreshold = 5
+PollInterval = '10s'
+SelectionMode = 'HighestHead'
+SyncThreshold = 5
+LeaseDuration = '0s'
+NodeIsSyncingEnabled = false
+FinalizedBlockPollInterval = '5s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m'
+NewHeadsPollInterval = '0s'
+
+[OCR]
+ContractConfirmations = 4
+ContractTransmitterTransmitTimeout = '10s'
+DatabaseTimeout = '10s'
+DeltaCOverride = '168h'
+DeltaCJitterOverride = '1h'
+ObservationGracePeriod = '1s'
+
+[OCR2.Automation]
+GasLimit = 5400000
+
+[Workflow]
+GasLimitDefault = 400_000
+
 -- default_overrides2/Ethereum_Mainnet.toml --
 ChainID = '1'
 FinalityTagEnabled = false