From e9de3a2776ae463c7f3b28cea10b5855c69ad004 Mon Sep 17 00:00:00 2001 From: Urvi Date: Mon, 17 Jun 2024 16:34:54 -0700 Subject: [PATCH] Add shared functions to get the stellar-core build version and protocol version and update ledgerexporter and captivecorebackend to use these shared functions. Implement a version check for protocol 21 when creating a new captive-core instance. --- .../ledgerexporter/internal/config.go | 21 +-- .../ledgerexporter/internal/config_test.go | 83 +----------- ingest/ledgerbackend/captive_core_backend.go | 37 +++--- .../captive_core_backend_test.go | 33 +++++ ingest/ledgerbackend/stellar_core_version.go | 63 +++++++++ .../stellar_core_version_test.go | 123 ++++++++++++++++++ .../internal/ingest/db_integration_test.go | 2 + services/horizon/internal/ingest/main_test.go | 2 + 8 files changed, 247 insertions(+), 117 deletions(-) create mode 100644 ingest/ledgerbackend/stellar_core_version.go create mode 100644 ingest/ledgerbackend/stellar_core_version_test.go diff --git a/exp/services/ledgerexporter/internal/config.go b/exp/services/ledgerexporter/internal/config.go index ff0ffbe2bf..50a5adf541 100644 --- a/exp/services/ledgerexporter/internal/config.go +++ b/exp/services/ledgerexporter/internal/config.go @@ -5,8 +5,6 @@ import ( _ "embed" "fmt" "os" - "os/exec" - "strings" "github.com/stellar/go/historyarchive" "github.com/stellar/go/ingest/ledgerbackend" @@ -192,26 +190,11 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb }, nil } -// By default, it points to exec.Command, overridden for testing purpose -var execCommand = exec.Command - -// Executes the "stellar-core version" command and parses its output to extract -// the core version -// The output of the "version" command is expected to be a multi-line string where the -// first line is the core version in format "vX.Y.Z-*". func (c *Config) setCoreVersionInfo() (err error) { - versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version") - versionOutput, err := versionCmd.Output() + c.CoreVersion, err = ledgerbackend.GetCoreBuildVersionFunc(c.StellarCoreConfig.StellarCoreBinaryPath) if err != nil { - return fmt.Errorf("failed to execute stellar-core version command: %w", err) - } - - // Split the output into lines - rows := strings.Split(string(versionOutput), "\n") - if len(rows) == 0 || len(rows[0]) == 0 { - return fmt.Errorf("stellar-core version not found") + return fmt.Errorf("failed to set stellar-core version: %w", err) } - c.CoreVersion = rows[0] logger.Infof("stellar-core version: %s", c.CoreVersion) return nil } diff --git a/exp/services/ledgerexporter/internal/config_test.go b/exp/services/ledgerexporter/internal/config_test.go index 37dc574012..07c0b78dba 100644 --- a/exp/services/ledgerexporter/internal/config_test.go +++ b/exp/services/ledgerexporter/internal/config_test.go @@ -3,17 +3,15 @@ package ledgerexporter import ( "context" "fmt" - "os" - "os/exec" "testing" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/datastore" "github.com/stretchr/testify/require" "github.com/stellar/go/historyarchive" - "github.com/stellar/go/support/errors" ) func TestNewConfig(t *testing.T) { @@ -95,7 +93,7 @@ func TestDefaultCaptiveCoreBin(t *testing.T) { RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}) require.NoError(t, err) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("/test/default/stellar-core") require.NoError(t, err) require.Equal(t, ccConfig.BinaryPath, "/test/default/stellar-core") @@ -116,7 +114,7 @@ func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -137,7 +135,7 @@ func TestValidCaptiveCoreManualNetwork(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, "test") require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"}) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -157,7 +155,7 @@ func TestValidCaptiveCoreOverridenToml(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -179,7 +177,7 @@ func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"}) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -438,72 +436,3 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) { } mockArchive.AssertExpectations(t) } - -var cmdOut = "" - -func fakeExecCommand(command string, args ...string) *exec.Cmd { - cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...) - cmd := exec.Command(os.Args[0], cs...) - cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut) - return cmd -} - -func init() { - execCommand = fakeExecCommand -} - -func TestExecCmdHelperProcess(t *testing.T) { - if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" { - return - } - fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT")) - os.Exit(0) -} - -func TestSetCoreVersionInfo(t *testing.T) { - execCommand = fakeExecCommand - tests := []struct { - name string - commandOutput string - expectedError error - expectedCoreVer string - }{ - { - name: "version found", - commandOutput: "v20.2.0-2-g6e73c0a88\n" + - "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" + - "soroban-env-host: \n" + - " curr:\n" + - " package version: 20.2.0\n" + - " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" + - " ledger protocol version: 20\n" + - " pre-release version: 0\n" + - " rs-stellar-xdr:\n" + - " package version: 20.1.0\n" + - " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" + - " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n", - expectedError: nil, - expectedCoreVer: "v20.2.0-2-g6e73c0a88", - }, - { - name: "core version not found", - commandOutput: "", - expectedError: errors.New("stellar-core version not found"), - expectedCoreVer: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config := Config{} - cmdOut = tt.commandOutput - err := config.setCoreVersionInfo() - - if tt.expectedError != nil { - require.EqualError(t, err, tt.expectedError.Error()) - } else { - require.NoError(t, err) - require.Equal(t, tt.expectedCoreVer, config.CoreVersion) - } - }) - } -} diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index 9dfc82ac02..f3aaa17a6f 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -6,8 +6,6 @@ import ( "fmt" "net/http" "os" - "os/exec" - "strings" "sync" "time" @@ -22,6 +20,8 @@ import ( "github.com/stellar/go/xdr" ) +var minProtocolVersionSupported uint = 21 + // Ensure CaptiveStellarCore implements LedgerBackend var _ LedgerBackend = (*CaptiveStellarCore)(nil) @@ -168,6 +168,17 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) { config.Log.SetLevel(logrus.InfoLevel) } + protocolVersion, err := GetCoreProtocolVersionFunc(config.BinaryPath) + if err != nil { + return nil, fmt.Errorf("error determining stellar-core protocol version: %w", err) + } + + if protocolVersion < minProtocolVersionSupported { + return nil, fmt.Errorf("stellar-core version not supported. Installed stellar-core version is at protocol %d, but minimum "+ + "required version is %d. Please upgrade stellar-core to a version that supports protocol version %d or higher", + protocolVersion, minProtocolVersionSupported, minProtocolVersionSupported) + } + parentCtx := config.Context if parentCtx == nil { parentCtx = context.Background() @@ -249,28 +260,12 @@ func (c *CaptiveStellarCore) coreVersionMetric() float64 { return float64(info.Info.ProtocolVersion) } -// By default, it points to exec.Command, overridden for testing purpose -var execCommand = exec.Command - -// Executes the "stellar-core version" command and parses its output to extract -// the core version -// The output of the "version" command is expected to be a multi-line string where the -// first line is the core version in format "vX.Y.Z-*". func (c *CaptiveStellarCore) setCoreVersion() { - versionCmd := execCommand(c.config.BinaryPath, "version") - versionOutput, err := versionCmd.Output() + var err error + c.captiveCoreVersion, err = GetCoreBuildVersionFunc(c.config.BinaryPath) if err != nil { - c.config.Log.Errorf("failed to execute stellar-core version command: %s", err) + c.config.Log.Errorf("Failed to set stellar-core version: %s", err) } - - // Split the output into lines - rows := strings.Split(string(versionOutput), "\n") - if len(rows) == 0 || len(rows[0]) == 0 { - c.config.Log.Error("stellar-core version not found") - return - } - - c.captiveCoreVersion = rows[0] c.config.Log.Infof("stellar-core version: %s", c.captiveCoreVersion) } diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index a367f560f1..01a58af212 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -151,6 +151,8 @@ func TestCaptiveNew(t *testing.T) { networkPassphrase := network.PublicNetworkPassphrase historyURLs := []string{server.URL} + GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ BinaryPath: executablePath, @@ -169,6 +171,35 @@ func TestCaptiveNew(t *testing.T) { assert.Equal(t, "uatest", userAgent) } +func TestCaptiveNewUnsupportedProtocolVersion(t *testing.T) { + storagePath, err := os.MkdirTemp("", "captive-core-*") + require.NoError(t, err) + defer os.RemoveAll(storagePath) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + executablePath := "/etc/stellar-core" + networkPassphrase := network.PublicNetworkPassphrase + historyURLs := []string{server.URL} + + GetCoreProtocolVersionFunc = func(string) (uint, error) { return 20, nil } + + _, err = NewCaptive( + CaptiveCoreConfig{ + BinaryPath: executablePath, + NetworkPassphrase: networkPassphrase, + HistoryArchiveURLs: historyURLs, + StoragePath: storagePath, + UserAgent: "uatest", + }, + ) + + assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 20, but minimum required version is 21. Please upgrade stellar-core to a version that supports protocol version 21 or higher") +} + func TestCaptivePrepareRange(t *testing.T) { metaChan := make(chan metaResult, 100) @@ -984,6 +1015,8 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) { captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{}) assert.NoError(t, err) + GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ BinaryPath: executablePath, diff --git a/ingest/ledgerbackend/stellar_core_version.go b/ingest/ledgerbackend/stellar_core_version.go new file mode 100644 index 0000000000..4ef26b9cde --- /dev/null +++ b/ingest/ledgerbackend/stellar_core_version.go @@ -0,0 +1,63 @@ +package ledgerbackend + +import ( + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// By default, it points to exec.Command, overridden for testing purpose +var ExecCommand = exec.Command +var GetCoreProtocolVersionFunc = GetCoreProtocolVersion +var GetCoreBuildVersionFunc = GetCoreBuildVersion + +// GetCoreBuildVersion executes the "stellar-core version" command and parses its output to extract +// the core version +// The output of the "version" command is expected to be a multi-line string where the +// first line is the core version in format "vX.Y.Z-*". +func GetCoreBuildVersion(coreBinaryPath string) (string, error) { + versionCmd := ExecCommand(coreBinaryPath, "version") + versionOutput, err := versionCmd.Output() + if err != nil { + return "", fmt.Errorf("failed to execute stellar-core version command: %w", err) + } + + // Split the output into lines + rows := strings.Split(string(versionOutput), "\n") + if len(rows) == 0 || len(rows[0]) == 0 { + return "", fmt.Errorf("stellar-core version not found") + } + + return rows[0], nil +} + +// GetCoreProtocolVersion retrieves the ledger protocol version from the specified stellar-core binary. +// It executes the "stellar-core version" command and parses the output to extract the protocol version. +func GetCoreProtocolVersion(coreBinaryPath string) (uint, error) { + if coreBinaryPath == "" { + return 0, fmt.Errorf("stellar-core binary path is empty") + } + + versionBytes, err := ExecCommand(coreBinaryPath, "version").Output() + if err != nil { + return 0, fmt.Errorf("error executing stellar-core version command (%s): %w", coreBinaryPath, err) + } + + versionRows := strings.Split(string(versionBytes), "\n") + re := regexp.MustCompile(`^\s*ledger protocol version: (\d*)`) + var ledgerProtocolStrings []string + for _, line := range versionRows { + ledgerProtocolStrings = re.FindStringSubmatch(line) + if len(ledgerProtocolStrings) == 2 { + val, err := strconv.Atoi(ledgerProtocolStrings[1]) + if err != nil { + return 0, fmt.Errorf("error parsing protocol version from stellar-core output: %w", err) + } + return uint(val), nil + } + } + + return 0, fmt.Errorf("error parsing protocol version from stellar-core output") +} diff --git a/ingest/ledgerbackend/stellar_core_version_test.go b/ingest/ledgerbackend/stellar_core_version_test.go new file mode 100644 index 0000000000..c4225dc29a --- /dev/null +++ b/ingest/ledgerbackend/stellar_core_version_test.go @@ -0,0 +1,123 @@ +package ledgerbackend + +import ( + "fmt" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +var fakeExecCmdOut = "" + +func fakeExecCommand(command string, args ...string) *exec.Cmd { + cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+fakeExecCmdOut) + return cmd +} + +func init() { + ExecCommand = fakeExecCommand +} + +func TestExecCmdHelperProcess(t *testing.T) { + if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" { + return + } + fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT")) + os.Exit(0) +} + +func TestGetCoreBuildVersion(t *testing.T) { + tests := []struct { + name string + commandOutput string + expectedError error + expectedCoreVer string + }{ + { + name: "core build version found", + commandOutput: "v20.2.0-2-g6e73c0a88\n" + + "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" + + "soroban-env-host: \n" + + " curr:\n" + + " package version: 20.2.0\n" + + " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" + + " ledger protocol version: 20\n" + + " pre-release version: 0\n" + + " rs-stellar-xdr:\n" + + " package version: 20.1.0\n" + + " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" + + " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n", + expectedError: nil, + expectedCoreVer: "v20.2.0-2-g6e73c0a88", + }, + { + name: "core build version not found", + commandOutput: "", + expectedError: fmt.Errorf("stellar-core version not found"), + expectedCoreVer: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeExecCmdOut = tt.commandOutput + coreVersion, err := GetCoreBuildVersion("") + + if tt.expectedError != nil { + require.EqualError(t, err, tt.expectedError.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedCoreVer, coreVersion) + } + }) + } +} + +func TestGetCoreProtcolVersion(t *testing.T) { + tests := []struct { + name string + commandOutput string + expectedError error + expectedProtocolVersion uint + }{ + { + name: "core protocol version found", + commandOutput: "v20.2.0-2-g6e73c0a88\n" + + "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" + + "soroban-env-host: \n" + + " curr:\n" + + " package version: 20.2.0\n" + + " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" + + " ledger protocol version: 21\n" + + " pre-release version: 0\n" + + " rs-stellar-xdr:\n" + + " package version: 20.1.0\n" + + " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" + + " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n", + expectedError: nil, + expectedProtocolVersion: 21, + }, + { + name: "core protocol version not found", + commandOutput: "", + expectedError: fmt.Errorf("error parsing protocol version from stellar-core output"), + expectedProtocolVersion: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeExecCmdOut = tt.commandOutput + coreVersion, err := GetCoreProtocolVersion("/usr/bin/stellar-core") + + if tt.expectedError != nil { + require.EqualError(t, err, tt.expectedError.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedProtocolVersion, coreVersion) + } + }) + } +} diff --git a/services/horizon/internal/ingest/db_integration_test.go b/services/horizon/internal/ingest/db_integration_test.go index 606cd9fb2b..61998649f0 100644 --- a/services/horizon/internal/ingest/db_integration_test.go +++ b/services/horizon/internal/ingest/db_integration_test.go @@ -100,6 +100,8 @@ func (s *DBTestSuite) SetupTest() { s.checkpointHash = xdr.Hash{1, 2, 3} s.ledgerBackend = &ledgerbackend.MockDatabaseBackend{} s.historyAdapter = &mockHistoryArchiveAdapter{} + ledgerbackend.GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + var err error sIface, err := NewSystem(Config{ HistorySession: s.tt.HorizonSession(), diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go index 3c7c587aa2..3efcd73e3b 100644 --- a/services/horizon/internal/ingest/main_test.go +++ b/services/horizon/internal/ingest/main_test.go @@ -96,6 +96,8 @@ func TestNewSystem(t *testing.T) { CheckpointFrequency: 64, } + ledgerbackend.GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + sIface, err := NewSystem(config) assert.NoError(t, err) system := sIface.(*system)