diff --git a/ingest/CHANGELOG.md b/ingest/CHANGELOG.md index 1aab4ae542..ed168de74e 100644 --- a/ingest/CHANGELOG.md +++ b/ingest/CHANGELOG.md @@ -2,8 +2,11 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). - -## Unreleased +### Stellar Core Protocol 21 Configuration Update: +* BucketlistDB is now the default database for stellar-core, replacing the experimental option. As a result, the `EXPERIMENTAL_BUCKETLIST_DB` configuration parameter has been deprecated. +* A new mandatory parameter, `DEPRECATED_SQL_LEDGER_STATE`, has been added with a default value of false which equivalent to `EXPERIMENTAL_BUCKETLIST_DB` being set to true. +* The parameter `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` has been renamed to `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`. +* The parameter `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF` has been renamed to `BUCKETLIST_DB_INDEX_CUTOFF`. ### New Features * Support for Soroban and Protocol 20! diff --git a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg index 6d23046f56..5af59efaf9 100644 --- a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg @@ -5,8 +5,6 @@ FAILURE_SAFETY=1 HTTP_PORT=11626 PEER_PORT=11725 -EXPERIMENTAL_BUCKETLIST_DB=true - [[HOME_DOMAINS]] HOME_DOMAIN="publicnode.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/configs/captive-core-testnet.cfg b/ingest/ledgerbackend/configs/captive-core-testnet.cfg index 3f0dec5095..9abeecc8f5 100644 --- a/ingest/ledgerbackend/configs/captive-core-testnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-testnet.cfg @@ -2,8 +2,6 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015" UNSAFE_QUORUM=true FAILURE_SAFETY=1 -EXPERIMENTAL_BUCKETLIST_DB=true - [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg new file mode 100644 index 0000000000..9040d6dc57 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg @@ -0,0 +1,22 @@ +# Generated file, do not edit +BUCKETLIST_DB_INDEX_CUTOFF = 20 +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15 +DATABASE = "sqlite3://stellar.db" +DEPRECATED_SQL_LEDGER_STATE = false +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg new file mode 100644 index 0000000000..12fe5e36a5 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg @@ -0,0 +1,21 @@ +# Generated file, do not edit +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DATABASE = "sqlite3://stellar.db" +DEPRECATED_SQL_LEDGER_STATE = false +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg b/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg new file mode 100644 index 0000000000..965fa5aaa6 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg @@ -0,0 +1,19 @@ +# Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg index 53838f6165..ec37e504fc 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg @@ -1,7 +1,7 @@ # Generated file, do not edit +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 DATABASE = "sqlite3://stellar.db" -EXPERIMENTAL_BUCKETLIST_DB = true -EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DEPRECATED_SQL_LEDGER_STATE = false FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg index 8c66b4a5ad..fa25d69b98 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = true diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg index df307a3a00..4a971a19d2 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg index 124abc435b..30159c150a 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg index d7b41a0b81..eab3b67bf8 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg index 9eca1ccad1..9c5fd26769 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "/var/stellar-core/test.log" diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg index 57a5e7ff2c..b8aeff746c 100644 --- a/ingest/ledgerbackend/testdata/expected-online-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 6789 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg index 278c681595..8ecc0fe958 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 11626 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg index f2227376a9..3960c1b546 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = true diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg index 89e1762757..6b3830e6ff 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 11626 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg index 1b65c5f318..93bb61b8be 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 6789 LOG_FILE_PATH = "/var/stellar-core/test.log" diff --git a/ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg b/ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg new file mode 100644 index 0000000000..e01a92cc61 --- /dev/null +++ b/ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg @@ -0,0 +1,12 @@ +BUCKETLIST_DB_INDEX_CUTOFF = 20 +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15 + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" diff --git a/ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg b/ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg new file mode 100644 index 0000000000..958019998c --- /dev/null +++ b/ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg @@ -0,0 +1,11 @@ +DEPRECATED_SQL_LEDGER_STATE = false + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" diff --git a/ingest/ledgerbackend/testdata/sample-appendix-on-disk.cfg b/ingest/ledgerbackend/testdata/sample-appendix-on-disk.cfg new file mode 100644 index 0000000000..3ca36b223b --- /dev/null +++ b/ingest/ledgerbackend/testdata/sample-appendix-on-disk.cfg @@ -0,0 +1,11 @@ +DEPRECATED_SQL_LEDGER_STATE = true + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index b8b6634a78..b7f41b03a7 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -21,6 +21,8 @@ var ( //go:embed configs/captive-core-testnet.cfg TestnetDefaultConfig []byte + + defaultBucketListDBPageSize uint = 12 ) const ( @@ -95,9 +97,9 @@ type captiveCoreTomlValues struct { Validators []Validator `toml:"VALIDATORS,omitempty"` HistoryEntries map[string]History `toml:"-"` QuorumSetEntries map[string]QuorumSet `toml:"-"` - UseBucketListDB bool `toml:"EXPERIMENTAL_BUCKETLIST_DB,omitempty"` - BucketListDBPageSizeExp *uint `toml:"EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"` - BucketListDBCutoff *uint `toml:"EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF,omitempty"` + BucketListDBPageSizeExp *uint `toml:"BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"` + BucketListDBCutoff *uint `toml:"BUCKETLIST_DB_INDEX_CUTOFF,omitempty"` + DeprecatedSqlLedgerState *bool `toml:"DEPRECATED_SQL_LEDGER_STATE,omitempty"` EnableSorobanDiagnosticEvents *bool `toml:"ENABLE_SOROBAN_DIAGNOSTIC_EVENTS,omitempty"` TestingMinimumPersistentEntryLifetime *uint `toml:"TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME,omitempty"` TestingSorobanHighLimitOverride *bool `toml:"TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE,omitempty"` @@ -446,14 +448,15 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { c.Database = "sqlite3://stellar.db" } - if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); !def && params.UseDB { - c.UseBucketListDB = true - } - - if c.UseBucketListDB && !c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { - n := uint(12) - c.BucketListDBPageSizeExp = &n // Set default page size to 4KB + deprecatedSqlLedgerState := false + if !params.UseDB { + deprecatedSqlLedgerState = true + } else { + if !c.tree.Has("BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { + c.BucketListDBPageSizeExp = &defaultBucketListDBPageSize + } } + c.DeprecatedSqlLedgerState = &deprecatedSqlLedgerState if !c.tree.Has("NETWORK_PASSPHRASE") { c.NetworkPassphrase = params.NetworkPassphrase @@ -543,10 +546,14 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error { ) } - if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); def && !params.UseDB { - return fmt.Errorf( - "BucketListDB enabled in captive core config file, requires Horizon flag --captive-core-use-db", - ) + if c.tree.Has("DEPRECATED_SQL_LEDGER_STATE") { + if params.UseDB && *c.DeprecatedSqlLedgerState { + return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to true, indicating stellar-core on-disk mode," + + " in which DEPRECATED_SQL_LEDGER_STATE must be set to false") + } else if !params.UseDB && !*c.DeprecatedSqlLedgerState { + return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to false, indicating stellar-core in-memory mode," + + " in which DEPRECATED_SQL_LEDGER_STATE must be set to true") + } } homeDomainSet := map[string]HomeDomain{} diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index 39b8473d4a..42412b85f1 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -2,9 +2,7 @@ package ledgerbackend import ( "io/ioutil" - "os" "path/filepath" - "strconv" "testing" "github.com/stretchr/testify/assert" @@ -28,6 +26,7 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { peerPort *uint logPath *string expectedError string + inMemory bool }{ { name: "mismatching NETWORK_PASSPHRASE", @@ -203,6 +202,19 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { appendPath: filepath.Join("testdata", "appendix-with-bucket-dir-path.cfg"), expectedError: "could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead", }, + { + name: "invalid DEPRECATED_SQL_LEDGER_STATE on-disk", + appendPath: filepath.Join("testdata", "sample-appendix-on-disk.cfg"), + expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to true, indicating " + + "stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false", + }, + { + name: "invalid DEPRECATED_SQL_LEDGER_STATE in-memory", + appendPath: filepath.Join("testdata", "sample-appendix-in-memory.cfg"), + expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to false, indicating " + + "stellar-core in-memory mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to true", + inMemory: true, + }, } { t.Run(testCase.name, func(t *testing.T) { params := CaptiveCoreTomlParams{ @@ -212,6 +224,7 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { PeerPort: testCase.peerPort, LogPath: testCase.logPath, Strict: true, + UseDB: !testCase.inMemory, } _, err := NewCaptiveCoreTomlFromFile(testCase.appendPath, params) assert.EqualError(t, err, testCase.expectedError) @@ -219,18 +232,6 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { } } -func checkTestingAboveProtocol19() bool { - str := os.Getenv("HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL") - if str == "" { - return false - } - version, err := strconv.ParseUint(str, 10, 32) - if err != nil { - return false - } - return uint32(version) > 19 -} - func TestGenerateConfig(t *testing.T) { for _, testCase := range []struct { name string @@ -352,6 +353,22 @@ func TestGenerateConfig(t *testing.T) { expectedPath: filepath.Join("testdata", "expected-online-with-appendix-minimum-persistent-entry.cfg"), logPath: nil, }, + { + name: "default BucketlistDB config", + mode: stellarCoreRunnerModeOnline, + appendPath: filepath.Join("testdata", "sample-appendix.cfg"), + expectedPath: filepath.Join("testdata", "expected-default-bucketlistdb-core.cfg"), + useDB: true, + logPath: nil, + }, + { + name: "BucketlistDB config in appendix", + mode: stellarCoreRunnerModeOnline, + appendPath: filepath.Join("testdata", "sample-appendix-bucketlistdb.cfg"), + expectedPath: filepath.Join("testdata", "expected-bucketlistdb-core.cfg"), + useDB: true, + logPath: nil, + }, } { t.Run(testCase.name, func(t *testing.T) { var err error @@ -385,6 +402,29 @@ func TestGenerateConfig(t *testing.T) { } } +func TestGenerateCoreConfigInMemory(t *testing.T) { + appendPath := filepath.Join("testdata", "sample-appendix.cfg") + expectedPath := filepath.Join("testdata", "expected-in-mem-core.cfg") + var err error + var captiveCoreToml *CaptiveCoreToml + params := CaptiveCoreTomlParams{ + NetworkPassphrase: "Public Global Stellar Network ; September 2015", + HistoryArchiveURLs: []string{"http://localhost:1170"}, + Strict: false, + UseDB: false, + } + captiveCoreToml, err = NewCaptiveCoreTomlFromFile(appendPath, params) + assert.NoError(t, err) + + configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOnline) + assert.NoError(t, err) + + expectedByte, err := ioutil.ReadFile(expectedPath) + assert.NoError(t, err) + + assert.Equal(t, string(expectedByte), string(configBytes)) +} + func TestHistoryArchiveURLTrailingSlash(t *testing.T) { httpPort := uint(8000) peerPort := uint(8000) @@ -461,6 +501,9 @@ func TestDBConfigDefaultsToSqlite(t *testing.T) { toml := CaptiveCoreToml{} require.NoError(t, toml.unmarshal(configBytes, true)) assert.Equal(t, toml.Database, "sqlite3://stellar.db") + assert.Equal(t, *toml.DeprecatedSqlLedgerState, false) + assert.Equal(t, *toml.BucketListDBPageSizeExp, defaultBucketListDBPageSize) + assert.Equal(t, toml.BucketListDBCutoff, (*uint)(nil)) } func TestNonDBConfigDoesNotUpdateDatabase(t *testing.T) { diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 7a89844062..6f127b9968 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -10,8 +10,12 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Change ingestion filtering logic to store transactions if any filter matches on it. ([5303](https://github.com/stellar/go/pull/5303)) - The previous behaviour was to store a tx only if both asset and account filters match together. So even if a tx matched an account filter but failed to match an asset filter, it would not be stored by Horizon. -## 2.30.0 +- Captive-core configuration parameters updated to align with [stellar-core v21](https://github.com/stellar/stellar-core/issues/3811) ([5333](https://github.com/stellar/go/pull/5333)) + - BucketlistDB is now the default database for stellar-core, deprecating `EXPERIMENTAL_BUCKETLIST_DB`. + - A new mandatory parameter `DEPRECATED_SQL_LEDGER_STATE` (default: false) is required by stellar-core on its configuration toml file. if the toml provided by `CAPTIVE_CORE_CONFIG_PATH` does not have this new setting, Horizon will add it automatically, therefore, no action required. + - If using `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` or `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF`, they have been renamed to `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` and `BUCKETLIST_DB_INDEX_CUTOFF` respectively. +## 2.30.0 **This release adds support for Protocol 21** ### Added diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg index 02c59d7057..275599bacd 100644 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ b/services/horizon/docker/captive-core-integration-tests.cfg @@ -4,8 +4,6 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -EXPERIMENTAL_BUCKETLIST_DB=true - ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg index cf4d514975..8d76504de2 100644 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg @@ -1,11 +1,10 @@ +EXPERIMENTAL_BUCKETLIST_DB = true PEER_PORT=11725 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -EXPERIMENTAL_BUCKETLIST_DB=true - ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg index 4744fd390e..44820f5933 100644 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg @@ -2,7 +2,6 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-standalone.cfg b/services/horizon/docker/captive-core-standalone.cfg index f5042a14df..d54b0ecae1 100644 --- a/services/horizon/docker/captive-core-standalone.cfg +++ b/services/horizon/docker/captive-core-standalone.cfg @@ -3,8 +3,6 @@ PEER_PORT=11725 UNSAFE_QUORUM=true FAILURE_SAFETY=0 -EXPERIMENTAL_BUCKETLIST_DB=true - [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg index f2b20b4927..fe23e94e8d 100644 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-classic-integration-tests.cfg @@ -1,3 +1,4 @@ +DEPRECATED_SQL_LEDGER_STATE=false ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true NETWORK_PASSPHRASE="Standalone Network ; February 2017" @@ -13,7 +14,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" -EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg index 0d5ec5cc43..414ad9c1eb 100644 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-integration-tests.cfg @@ -1,3 +1,4 @@ +DEPRECATED_SQL_LEDGER_STATE=false ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true NETWORK_PASSPHRASE="Standalone Network ; February 2017" @@ -13,7 +14,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" -EXPERIMENTAL_BUCKETLIST_DB=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/stellar-core-pubnet.cfg b/services/horizon/docker/stellar-core-pubnet.cfg index 7f250b10c7..94b29c4b4e 100644 --- a/services/horizon/docker/stellar-core-pubnet.cfg +++ b/services/horizon/docker/stellar-core-pubnet.cfg @@ -9,8 +9,6 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 -EXPERIMENTAL_BUCKETLIST_DB=true - [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/stellar-core-standalone.cfg b/services/horizon/docker/stellar-core-standalone.cfg index 41c8a377aa..a2b7e806c9 100644 --- a/services/horizon/docker/stellar-core-standalone.cfg +++ b/services/horizon/docker/stellar-core-standalone.cfg @@ -14,7 +14,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.internal port=5641 dbname=stellar" -EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-testnet.cfg b/services/horizon/docker/stellar-core-testnet.cfg index 55c5cea462..cf8546a3e9 100644 --- a/services/horizon/docker/stellar-core-testnet.cfg +++ b/services/horizon/docker/stellar-core-testnet.cfg @@ -12,8 +12,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=100 -EXPERIMENTAL_BUCKETLIST_DB=true - [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg index cedbbc65ab..5a702711fe 100644 --- a/services/horizon/docker/verify-range/captive-core-pubnet.cfg +++ b/services/horizon/docker/verify-range/captive-core-pubnet.cfg @@ -2,8 +2,6 @@ PEER_PORT=11725 FAILURE_SAFETY=1 -EXPERIMENTAL_BUCKETLIST_DB=true - [[HOME_DOMAINS]] HOME_DOMAIN="stellar.org" QUALITY="HIGH" diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg index 91bb190131..6139f9f528 100644 --- a/services/horizon/docker/verify-range/stellar-core.cfg +++ b/services/horizon/docker/verify-range/stellar-core.cfg @@ -4,8 +4,6 @@ LOG_FILE_PATH="" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 -EXPERIMENTAL_BUCKETLIST_DB=true - AUTOMATIC_MAINTENANCE_COUNT=0 NODE_NAMES=[