Bump core to the stable v21 build. (#5323)

* Bump core to the stable v21 build. * Try using 'unsafe-stellar-core' docker image * Revert "Try using 'unsafe-stellar-core' docker image" This reverts commit 19ac62f. * Add DEPRECATED_SQL_LEDGER_STATE=false to (most) of the Core configs * Try using `EXPERIMENTAL_BUCKETLIST_DB` for compatibility reasons * Update horizon.yml force all integration tests to have HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB=true by default * captive core use_db flag default to true for integration tests should always be set to true, that flag is deprecated and defaulted to true, there is no test paths validating it being false. * #5295: dealing with cc use db assumptions in tests * #5295: fixing captive core use_db flag assumptions in tests --------- Co-authored-by: shawn <[email protected]> Co-authored-by: Shawn Reuland <[email protected]>
stellar · Jun 4, 2024 · 89068a9 · 89068a9
1 parent f03edd0
commit 89068a9
Show file tree

Hide file tree

Showing 24 changed files with 46 additions and 9 deletions.
diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml
@@ -32,11 +32,12 @@ jobs:
     env:
       HORIZON_INTEGRATION_TESTS_ENABLED: true
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
-      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1812.rc1.a10329cca.focal
-      PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal
+      HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true
+      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
+      PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21
       PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
-      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1812.rc1.a10329cca.focal
-      PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal
+      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
+      PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21
       PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
       PGHOST: localhost
       PGPORT: 5432
@@ -125,7 +126,7 @@ jobs:
     name: Test (and push) verify-range image
     runs-on: ubuntu-22.04
     env:
-      STELLAR_CORE_VERSION: 19.14.0-1500.5664eff4e.focal
+      STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal
       CAPTIVE_CORE_STORAGE_PATH: /tmp
     steps:
       - uses: actions/checkout@v3
@@ -155,7 +156,7 @@ jobs:
     name: Test and push the Ledger Exporter images
     runs-on: ubuntu-latest
     env:
-      STELLAR_CORE_VERSION: 21.0.0-1812.rc1.a10329cca.focal
+      STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal
     steps:
       - uses: actions/checkout@v3
         with:

diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg
@@ -3,6 +3,8 @@ DATABASE = "sqlite3:///cc/stellar.db"
 
 FAILURE_SAFETY=1
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 # WARNING! Do not use this config in production. Quorum sets should
 # be carefully selected manually.
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"

diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg
@@ -4,6 +4,8 @@ DATABASE = "sqlite3:///cc/stellar.db"
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[HOME_DOMAINS]]
 HOME_DOMAIN="testnet.stellar.org"
 QUALITY="HIGH"

diff --git a/exp/tools/dump-ledger-state/Dockerfile b/exp/tools/dump-ledger-state/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:22.04
 
-ENV STELLAR_CORE_VERSION=19.14.0-1500.5664eff4e.focal
+ENV STELLAR_CORE_VERSION=21.0.0-1872.c6f474133.focal
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl wget gnupg apt-utils

diff --git a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg
@@ -8,6 +8,8 @@ UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 CATCHUP_RECENT=8640
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [HISTORY.cache]
 get="cp /opt/stellar/history-cache/{0} {1}"
 

diff --git a/exp/tools/dump-ledger-state/stellar-core.cfg b/exp/tools/dump-ledger-state/stellar-core.cfg
@@ -6,6 +6,8 @@ DATABASE="postgresql://dbname=core host=localhost user=circleci"
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
 CATCHUP_RECENT=1
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [HISTORY.cache]
 get="cp /opt/stellar/history-cache/{0} {1}"
 

diff --git a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg
@@ -5,6 +5,8 @@ FAILURE_SAFETY=1
 HTTP_PORT=11626
 PEER_PORT=11725
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[HOME_DOMAINS]]
 HOME_DOMAIN="publicnode.org"
 QUALITY="HIGH"

diff --git a/ingest/ledgerbackend/configs/captive-core-testnet.cfg b/ingest/ledgerbackend/configs/captive-core-testnet.cfg
@@ -2,6 +2,8 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[HOME_DOMAINS]]
 HOME_DOMAIN="testnet.stellar.org"
 QUALITY="HIGH"

diff --git a/services/horizon/docker/captive-core-classic-integration-tests.cfg b/services/horizon/docker/captive-core-classic-integration-tests.cfg
@@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[VALIDATORS]]
 NAME="local_core"
 HOME_DOMAIN="core.local"

diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg
@@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true
 # Lower the TTL of persistent ledger entries
 # so that ledger entry extension/restoring becomes testeable

diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg
@@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true
 # Lower the TTL of persistent ledger entries
 # so that ledger entry extension/restoring becomes testeable

diff --git a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg
@@ -1,5 +1,7 @@
 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[VALIDATORS]]
 NAME="local_core"
 HOME_DOMAIN="core.local"

diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg
@@ -2,6 +2,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true
 TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true
+EXPERIMENTAL_BUCKETLIST_DB=true
 
 [[VALIDATORS]]
 NAME="local_core"

diff --git a/services/horizon/docker/captive-core-standalone.cfg b/services/horizon/docker/captive-core-standalone.cfg
@@ -3,6 +3,8 @@ PEER_PORT=11725
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[VALIDATORS]]
 NAME="local_core"
 HOME_DOMAIN="core.local"

diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg
@@ -13,6 +13,7 @@ UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar"
+EXPERIMENTAL_BUCKETLIST_DB=true
 
 [QUORUM_SET]
 THRESHOLD_PERCENT=100

diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg
@@ -13,6 +13,7 @@ UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar"
+EXPERIMENTAL_BUCKETLIST_DB=true
 
 # Lower the TTL of persistent ledger entries
 # so that ledger entry extension/restoring becomes testeable

diff --git a/services/horizon/docker/stellar-core-pubnet.cfg b/services/horizon/docker/stellar-core-pubnet.cfg
@@ -9,6 +9,8 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
 CATCHUP_RECENT=100
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [HISTORY.cache]
 get="cp /opt/stellar/history-cache/{0} {1}"
 

diff --git a/services/horizon/docker/stellar-core-standalone.cfg b/services/horizon/docker/stellar-core-standalone.cfg
@@ -14,6 +14,7 @@ UNSAFE_QUORUM=true
 FAILURE_SAFETY=0
 
 DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.internal port=5641 dbname=stellar"
+EXPERIMENTAL_BUCKETLIST_DB=true
 
 [QUORUM_SET]
 THRESHOLD_PERCENT=100

diff --git a/services/horizon/docker/stellar-core-testnet.cfg b/services/horizon/docker/stellar-core-testnet.cfg
@@ -12,6 +12,8 @@ UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 CATCHUP_RECENT=100
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [HISTORY.cache]
 get="cp /opt/stellar/history-cache/{0} {1}"
 

diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg
@@ -2,6 +2,8 @@ PEER_PORT=11725
 
 FAILURE_SAFETY=1
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 [[HOME_DOMAINS]]
 HOME_DOMAIN="stellar.org"
 QUALITY="HIGH"

diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg
@@ -4,6 +4,8 @@ LOG_FILE_PATH=""
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
 CATCHUP_RECENT=100
 
+EXPERIMENTAL_BUCKETLIST_DB=true
+
 AUTOMATIC_MAINTENANCE_COUNT=0
 
 NODE_NAMES=[

diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go
@@ -86,6 +86,7 @@ func Test_createCaptiveCoreDefaultConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tt.config.CaptiveCoreTomlParams.UseDB = true
 			e := setCaptiveCoreConfiguration(&tt.config,
 				ApplyOptions{RequireCaptiveCoreFullConfig: true})
 			if tt.errStr == "" {
@@ -192,6 +193,7 @@ func Test_createCaptiveCoreConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tt.config.CaptiveCoreTomlParams.UseDB = true
 			e := setCaptiveCoreConfiguration(&tt.config,
 				ApplyOptions{RequireCaptiveCoreFullConfig: tt.requireCaptiveCoreConfig})
 			if tt.errStr == "" {

diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go
@@ -505,7 +505,7 @@ func TestDeprecatedOutputs(t *testing.T) {
 		stdLog.SetOutput(os.Stderr)
 
 		testConfig := integration.GetTestConfig()
-		testConfig.HorizonIngestParameters = map[string]string{"captive-core-use-db": "false"}
+		testConfig.HorizonIngestParameters = map[string]string{"captive-core-use-db": "true"}
 		test := integration.NewTest(t, *testConfig)
 		err := test.StartHorizon()
 		assert.NoError(t, err)

diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go
@@ -441,7 +441,7 @@ func (i *Test) getDefaultIngestArgs(postgres *dbtest.DB) map[string]string {
 		"stellar-core-binary-path":  i.coreConfig.binaryPath,
 		"captive-core-config-path":  i.coreConfig.configPath,
 		"captive-core-http-port":    "21626",
-		"captive-core-use-db":       strconv.FormatBool(i.coreConfig.useDB),
+		"captive-core-use-db":       "true",
 		"captive-core-storage-path": i.coreConfig.storagePath,
 		"ingest":                    "true"})
 }