Build Silkworm RpcDaemon settings from Erigon ones (#10002)

This PR introduces support for customising Silkworm RpcDaemon settings
in Erigon++.

Common RPC settings between Erigon and Silkworm are simply translated
from the existing Erigon command-line options. They include:
- `--http.addr`
- `--http.port`
- `--http.compression`
- `--http.corsdomain`
- `--http.api`
- `--ws`
- `--ws.compression`

Moreover, the following Silkworm-specific command-line options are
added:
- `--silkworm.verbosity`
- `--silkworm.contexts`
- `--silkworm.rpc.log`
- `--silkworm.rpc.log.maxsize`
- `--silkworm.rpc.log.maxfiles`
- `--silkworm.rpc.log.response`
- `--silkworm.rpc.workers`
- `--silkworm.rpc.compatibility`

Default values cover the common usages of Erigon++ experimental
features, yet such options can be useful for testing some corner cases
or collecting information.

Finally, this PR adds a new `LogDirPath` function to `logging` module
just to determine the log dir path used by Erigon and put there also
Silkworm RPC interface logs, when enabled.

cmd/utils/flags.go

@@ -61,6 +61,7 @@ import (
 	"github.com/ledgerwatch/erigon/p2p/netutil"
 	"github.com/ledgerwatch/erigon/params"
 	"github.com/ledgerwatch/erigon/rpc/rpccfg"
+	"github.com/ledgerwatch/erigon/turbo/logging"
 )
 
 // These are all the command line flags we support.
@@ -864,12 +865,52 @@ var (
 	}
 	SilkwormRpcDaemonFlag = cli.BoolFlag{
 		Name:  "silkworm.rpc",
-		Usage: "Enable embedded Silkworm RPC daemon",
+		Usage: "Enable embedded Silkworm RPC service",
 	}
 	SilkwormSentryFlag = cli.BoolFlag{
 		Name:  "silkworm.sentry",
 		Usage: "Enable embedded Silkworm Sentry service",
 	}
+	SilkwormVerbosityFlag = cli.StringFlag{
+		Name:  "silkworm.verbosity",
+		Usage: "Set the log level for Silkworm console logs",
+		Value: log.LvlInfo.String(),
+	}
+	SilkwormNumContextsFlag = cli.UintFlag{
+		Name:  "silkworm.contexts",
+		Usage: "Number of I/O contexts used in embedded Silkworm RPC and Sentry services (zero means use default in Silkworm)",
+		Value: 0,
+	}
+	SilkwormRpcLogEnabledFlag = cli.BoolFlag{
+		Name:  "silkworm.rpc.log",
+		Usage: "Enable interface log for embedded Silkworm RPC service",
+		Value: false,
+	}
+	SilkwormRpcLogMaxFileSizeFlag = cli.UintFlag{
+		Name:  "silkworm.rpc.log.maxsize",
+		Usage: "Max interface log file size in MB for embedded Silkworm RPC service",
+		Value: 1,
+	}
+	SilkwormRpcLogMaxFilesFlag = cli.UintFlag{
+		Name:  "silkworm.rpc.log.maxfiles",
+		Usage: "Max interface log files for embedded Silkworm RPC service",
+		Value: 100,
+	}
+	SilkwormRpcLogDumpResponseFlag = cli.BoolFlag{
+		Name:  "silkworm.rpc.log.response",
+		Usage: "Dump responses in interface logs for embedded Silkworm RPC service",
+		Value: false,
+	}
+	SilkwormRpcNumWorkersFlag = cli.UintFlag{
+		Name:  "silkworm.rpc.workers",
+		Usage: "Number of worker threads used in embedded Silkworm RPC service (zero means use default in Silkworm)",
+		Value: 0,
+	}
+	SilkwormRpcJsonCompatibilityFlag = cli.BoolFlag{
+		Name:  "silkworm.rpc.compatibility",
+		Usage: "Preserve JSON-RPC compatibility using embedded Silkworm RPC service",
+		Value: true,
+	}
 
 	BeaconAPIFlag = cli.StringSliceFlag{
 		Name:  "beacon.api",
@@ -1598,6 +1639,15 @@ func setSilkworm(ctx *cli.Context, cfg *ethconfig.Config) {
 	cfg.SilkwormExecution = ctx.Bool(SilkwormExecutionFlag.Name)
 	cfg.SilkwormRpcDaemon = ctx.Bool(SilkwormRpcDaemonFlag.Name)
 	cfg.SilkwormSentry = ctx.Bool(SilkwormSentryFlag.Name)
+	cfg.SilkwormVerbosity = ctx.String(SilkwormVerbosityFlag.Name)
+	cfg.SilkwormNumContexts = uint32(ctx.Uint64(SilkwormNumContextsFlag.Name))
+	cfg.SilkwormRpcLogEnabled = ctx.Bool(SilkwormRpcLogEnabledFlag.Name)
+	cfg.SilkwormRpcLogDirPath = logging.LogDirPath(ctx)
+	cfg.SilkwormRpcLogMaxFileSize = uint16(ctx.Uint64(SilkwormRpcLogMaxFileSizeFlag.Name))
+	cfg.SilkwormRpcLogMaxFiles = uint16(ctx.Uint(SilkwormRpcLogMaxFilesFlag.Name))
+	cfg.SilkwormRpcLogDumpResponse = ctx.Bool(SilkwormRpcLogDumpResponseFlag.Name)
+	cfg.SilkwormRpcNumWorkers = uint32(ctx.Uint64(SilkwormRpcNumWorkersFlag.Name))
+	cfg.SilkwormRpcJsonCompatibility = ctx.Bool(SilkwormRpcJsonCompatibilityFlag.Name)
 }
 
 // CheckExclusive verifies that only a single instance of the provided flags was

eth/backend.go

@@ -352,7 +352,11 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
 	backend.gasPrice, _ = uint256.FromBig(config.Miner.GasPrice)
 
 	if config.SilkwormExecution || config.SilkwormRpcDaemon || config.SilkwormSentry {
-		backend.silkworm, err = silkworm.New(config.Dirs.DataDir, mdbx.Version())
+		logLevel, err := log.LvlFromString(config.SilkwormVerbosity)
+		if err != nil {
+			return nil, err
+		}
+		backend.silkworm, err = silkworm.New(config.Dirs.DataDir, mdbx.Version(), config.SilkwormNumContexts, logLevel)
 		if err != nil {
 			return nil, err
 		}
@@ -945,7 +949,27 @@ func (s *Ethereum) Init(stack *node.Node, config *ethconfig.Config, chainConfig
 	s.apiList = jsonrpc.APIList(chainKv, ethRpcClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, s.agg, &httpRpcCfg, s.engine, s.logger)
 
 	if config.SilkwormRpcDaemon && httpRpcCfg.Enabled {
-		silkwormRPCDaemonService := silkworm.NewRpcDaemonService(s.silkworm, chainKv)
+		interface_log_settings := silkworm.RpcInterfaceLogSettings{
+			Enabled:         config.SilkwormRpcLogEnabled,
+			ContainerFolder: config.SilkwormRpcLogDirPath,
+			MaxFileSizeMB:   config.SilkwormRpcLogMaxFileSize,
+			MaxFiles:        config.SilkwormRpcLogMaxFiles,
+			DumpResponse:    config.SilkwormRpcLogDumpResponse,
+		}
+		settings := silkworm.RpcDaemonSettings{
+			EthLogSettings:       interface_log_settings,
+			EthAPIHost:           httpRpcCfg.HttpListenAddress,
+			EthAPIPort:           httpRpcCfg.HttpPort,
+			EthAPISpec:           httpRpcCfg.API,
+			NumWorkers:           config.SilkwormRpcNumWorkers,
+			CORSDomains:          httpRpcCfg.HttpCORSDomain,
+			JWTFilePath:          httpRpcCfg.JWTSecretPath,
+			JSONRPCCompatibility: config.SilkwormRpcJsonCompatibility,
+			WebSocketEnabled:     httpRpcCfg.WebsocketEnabled,
+			WebSocketCompression: httpRpcCfg.WebsocketCompression,
+			HTTPCompression:      httpRpcCfg.HttpCompression,
+		}
+		silkwormRPCDaemonService := silkworm.NewRpcDaemonService(s.silkworm, chainKv, settings)
 		s.silkwormRPCDaemonService = &silkwormRPCDaemonService
 	} else {
 		go func() {

eth/ethconfig/config.go

@@ -251,9 +251,18 @@ type Config struct {
 	ForcePartialCommit bool
 
 	// Embedded Silkworm support
-	SilkwormExecution bool
-	SilkwormRpcDaemon bool
-	SilkwormSentry    bool
+	SilkwormExecution            bool
+	SilkwormRpcDaemon            bool
+	SilkwormSentry               bool
+	SilkwormVerbosity            string
+	SilkwormNumContexts          uint32
+	SilkwormRpcLogEnabled        bool
+	SilkwormRpcLogDirPath        string
+	SilkwormRpcLogMaxFileSize    uint16
+	SilkwormRpcLogMaxFiles       uint16
+	SilkwormRpcLogDumpResponse   bool
+	SilkwormRpcNumWorkers        uint32
+	SilkwormRpcJsonCompatibility bool
 
 	DisableTxPoolGossip bool
 }

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/erigontech/mdbx-go v0.27.24
-	github.com/erigontech/silkworm-go v0.15.1
+	github.com/erigontech/silkworm-go v0.18.0
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0
 )

go.sum

@@ -263,8 +263,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erigontech/mdbx-go v0.27.24 h1:jNsRE/4jC1F3S5SpAbmgT5jrEkfrdFk2MKEL9toVPxo=
 github.com/erigontech/mdbx-go v0.27.24/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4=
-github.com/erigontech/silkworm-go v0.15.1 h1:1hGntrpa8e6MNEnVi0p4A063TNnRgldItjl3xP9v1t4=
-github.com/erigontech/silkworm-go v0.15.1/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
+github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA=
+github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
 github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0=
 github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=

turbo/cli/default_flags.go

@@ -171,6 +171,14 @@ var DefaultFlags = []cli.Flag{
 	&utils.SilkwormExecutionFlag,
 	&utils.SilkwormRpcDaemonFlag,
 	&utils.SilkwormSentryFlag,
+	&utils.SilkwormVerbosityFlag,
+	&utils.SilkwormNumContextsFlag,
+	&utils.SilkwormRpcLogEnabledFlag,
+	&utils.SilkwormRpcLogMaxFileSizeFlag,
+	&utils.SilkwormRpcLogMaxFilesFlag,
+	&utils.SilkwormRpcLogDumpResponseFlag,
+	&utils.SilkwormRpcNumWorkersFlag,
+	&utils.SilkwormRpcJsonCompatibilityFlag,
 
 	&utils.BeaconAPIFlag,
 	&utils.BeaconApiAddrFlag,

turbo/logging/logging.go

@@ -14,6 +14,21 @@ import (
 	"github.com/ledgerwatch/erigon-lib/common/metrics"
 )
 
+// Determine the log dir path based on the given urfave context
+func LogDirPath(ctx *cli.Context) string {
+	dirPath := ""
+	if !ctx.Bool(LogDirDisableFlag.Name) {
+		dirPath = ctx.String(LogDirPathFlag.Name)
+		if dirPath == "" {
+			datadir := ctx.String("datadir")
+			if datadir != "" {
+				dirPath = filepath.Join(datadir, "logs")
+			}
+		}
+	}
+	return dirPath
+}
+
 // SetupLoggerCtx performs the logging setup according to the parameters
 // containted in the given urfave context. It returns either root logger,
 // if rootHandler argument is set to true, or a newly created logger.
@@ -216,7 +231,6 @@ func initSeparatedLogging(
 	mux := log.MultiHandler(consoleHandler, log.LvlFilterHandler(dirLevel, userLog))
 	logger.SetHandler(mux)
 	logger.Info("logging to file system", "log dir", dirPath, "file prefix", filePrefix, "log level", dirLevel, "json", dirJson)
-	return
 }
 
 func tryGetLogLevel(s string) (log.Lvl, error) {

turbo/silkworm/silkworm.go

@@ -8,49 +8,73 @@ import (
 	silkworm_go "github.com/erigontech/silkworm-go"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon/consensus"
+	"github.com/ledgerwatch/log/v3"
 )
 
 type Silkworm = silkworm_go.Silkworm
+type SilkwormLogLevel = silkworm_go.SilkwormLogLevel
 type SentrySettings = silkworm_go.SentrySettings
+type RpcDaemonSettings = silkworm_go.RpcDaemonSettings
+type RpcInterfaceLogSettings = silkworm_go.RpcInterfaceLogSettings
 type MappedHeaderSnapshot = silkworm_go.MappedHeaderSnapshot
 type MappedBodySnapshot = silkworm_go.MappedBodySnapshot
 type MappedTxnSnapshot = silkworm_go.MappedTxnSnapshot
 type MappedChainSnapshot = silkworm_go.MappedChainSnapshot
 
-var New = silkworm_go.New
 var NewMemoryMappedRegion = silkworm_go.NewMemoryMappedRegion
 var NewMappedHeaderSnapshot = silkworm_go.NewMappedHeaderSnapshot
 var NewMappedBodySnapshot = silkworm_go.NewMappedBodySnapshot
 var NewMappedTxnSnapshot = silkworm_go.NewMappedTxnSnapshot
 
 var ErrInterrupted = silkworm_go.ErrInterrupted
 
+func New(dataDirPath string, libMdbxVersion string, numIOContexts uint32, logLevel log.Lvl) (*Silkworm, error) {
+	var logVerbosity SilkwormLogLevel
+	switch logLevel {
+	case log.LvlCrit:
+		logVerbosity = silkworm_go.LogLevelCritical
+	case log.LvlError:
+		logVerbosity = silkworm_go.LogLevelError
+	case log.LvlWarn:
+		logVerbosity = silkworm_go.LogLevelWarning
+	case log.LvlInfo:
+		logVerbosity = silkworm_go.LogLevelInfo
+	case log.LvlDebug:
+		logVerbosity = silkworm_go.LogLevelDebug
+	case log.LvlTrace:
+		logVerbosity = silkworm_go.LogLevelTrace
+	}
+	return silkworm_go.New(dataDirPath, libMdbxVersion, numIOContexts, logVerbosity)
+}
+
 type RpcDaemonService struct {
 	silkworm *Silkworm
 	db       kv.RoDB
+	settings RpcDaemonSettings
 }
 
-func NewRpcDaemonService(s *Silkworm, db kv.RoDB) RpcDaemonService {
+func NewRpcDaemonService(s *Silkworm, db kv.RoDB, settings RpcDaemonSettings) RpcDaemonService {
 	return RpcDaemonService{
 		silkworm: s,
 		db:       db,
+		settings: settings,
 	}
 }
 
 func (service RpcDaemonService) Start() error {
-	return service.silkworm.StartRpcDaemon(service.db.CHandle())
+	return service.silkworm.StartRpcDaemon(service.db.CHandle(), service.settings)
 }
 
 func (service RpcDaemonService) Stop() error {
 	return service.silkworm.StopRpcDaemon()
 }
 
 type SentryService struct {
-	silkworm *silkworm_go.Silkworm
-	settings silkworm_go.SentrySettings
+	silkworm *Silkworm
+	settings SentrySettings
 }
 
-func NewSentryService(s *Silkworm, settings silkworm_go.SentrySettings) SentryService {
+func NewSentryService(s *Silkworm, settings SentrySettings) SentryService {
 	return SentryService{
 		silkworm: s,
 		settings: settings,