From 89068a97ead78fbce9168f53ce6898a866b85815 Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Tue, 4 Jun 2024 14:02:03 -0400 Subject: [PATCH] Bump core to the stable v21 build. (#5323) * Bump core to the stable v21 build. * Try using 'unsafe-stellar-core' docker image * Revert "Try using 'unsafe-stellar-core' docker image" This reverts commit 19ac62ffeb2fad774ab3212dd3fac79e854d5989. * Add DEPRECATED_SQL_LEDGER_STATE=false to (most) of the Core configs * Try using `EXPERIMENTAL_BUCKETLIST_DB` for compatibility reasons * Update horizon.yml force all integration tests to have HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB=true by default * captive core use_db flag default to true for integration tests should always be set to true, that flag is deprecated and defaulted to true, there is no test paths validating it being false. * #5295: dealing with cc use db assumptions in tests * #5295: fixing captive core use_db flag assumptions in tests --------- Co-authored-by: shawn Co-authored-by: Shawn Reuland --- .github/workflows/horizon.yml | 13 +++++++------ .../build/ledgerexporter/captive-core-pubnet.cfg | 2 ++ .../build/ledgerexporter/captive-core-testnet.cfg | 2 ++ exp/tools/dump-ledger-state/Dockerfile | 2 +- .../dump-ledger-state/stellar-core-testnet.cfg | 2 ++ exp/tools/dump-ledger-state/stellar-core.cfg | 2 ++ .../ledgerbackend/configs/captive-core-pubnet.cfg | 2 ++ .../ledgerbackend/configs/captive-core-testnet.cfg | 2 ++ .../captive-core-classic-integration-tests.cfg | 2 ++ .../docker/captive-core-integration-tests.cfg | 2 ++ .../captive-core-integration-tests.soroban-rpc.cfg | 2 ++ ...ore-reingest-range-classic-integration-tests.cfg | 2 ++ ...aptive-core-reingest-range-integration-tests.cfg | 1 + services/horizon/docker/captive-core-standalone.cfg | 2 ++ .../stellar-core-classic-integration-tests.cfg | 1 + .../docker/stellar-core-integration-tests.cfg | 1 + services/horizon/docker/stellar-core-pubnet.cfg | 2 ++ services/horizon/docker/stellar-core-standalone.cfg | 1 + services/horizon/docker/stellar-core-testnet.cfg | 2 ++ .../docker/verify-range/captive-core-pubnet.cfg | 2 ++ .../horizon/docker/verify-range/stellar-core.cfg | 2 ++ services/horizon/internal/flags_test.go | 2 ++ .../horizon/internal/integration/parameters_test.go | 2 +- .../internal/test/integration/integration.go | 2 +- 24 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index ddf0da7f11..fe7b2df7a8 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -32,11 +32,12 @@ jobs: env: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} - PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1812.rc1.a10329cca.focal - PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal + HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true + PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal + PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 - PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1812.rc1.a10329cca.focal - PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal + PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal + PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PGHOST: localhost PGPORT: 5432 @@ -125,7 +126,7 @@ jobs: name: Test (and push) verify-range image runs-on: ubuntu-22.04 env: - STELLAR_CORE_VERSION: 19.14.0-1500.5664eff4e.focal + STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal CAPTIVE_CORE_STORAGE_PATH: /tmp steps: - uses: actions/checkout@v3 @@ -155,7 +156,7 @@ jobs: name: Test and push the Ledger Exporter images runs-on: ubuntu-latest env: - STELLAR_CORE_VERSION: 21.0.0-1812.rc1.a10329cca.focal + STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal steps: - uses: actions/checkout@v3 with: diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg index d891626756..6379725b8d 100644 --- a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg +++ b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg @@ -3,6 +3,8 @@ DATABASE = "sqlite3:///cc/stellar.db" FAILURE_SAFETY=1 +EXPERIMENTAL_BUCKETLIST_DB=true + # WARNING! Do not use this config in production. Quorum sets should # be carefully selected manually. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg index 0cd9b2f496..9c7dadc527 100644 --- a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg +++ b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg @@ -4,6 +4,8 @@ DATABASE = "sqlite3:///cc/stellar.db" UNSAFE_QUORUM=true FAILURE_SAFETY=1 +EXPERIMENTAL_BUCKETLIST_DB=true + [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" QUALITY="HIGH" diff --git a/exp/tools/dump-ledger-state/Dockerfile b/exp/tools/dump-ledger-state/Dockerfile index 91b258f0a8..5ffcb9c0a2 100644 --- a/exp/tools/dump-ledger-state/Dockerfile +++ b/exp/tools/dump-ledger-state/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 -ENV STELLAR_CORE_VERSION=19.14.0-1500.5664eff4e.focal +ENV STELLAR_CORE_VERSION=21.0.0-1872.c6f474133.focal ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl wget gnupg apt-utils diff --git a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg index 5162295622..a02221e795 100644 --- a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg +++ b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg @@ -8,6 +8,8 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=8640 +EXPERIMENTAL_BUCKETLIST_DB=true + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/exp/tools/dump-ledger-state/stellar-core.cfg b/exp/tools/dump-ledger-state/stellar-core.cfg index 0b4e454681..0d97346ce6 100644 --- a/exp/tools/dump-ledger-state/stellar-core.cfg +++ b/exp/tools/dump-ledger-state/stellar-core.cfg @@ -6,6 +6,8 @@ DATABASE="postgresql://dbname=core host=localhost user=circleci" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=1 +EXPERIMENTAL_BUCKETLIST_DB=true + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg index 5af59efaf9..6d23046f56 100644 --- a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg @@ -5,6 +5,8 @@ FAILURE_SAFETY=1 HTTP_PORT=11626 PEER_PORT=11725 +EXPERIMENTAL_BUCKETLIST_DB=true + [[HOME_DOMAINS]] HOME_DOMAIN="publicnode.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/configs/captive-core-testnet.cfg b/ingest/ledgerbackend/configs/captive-core-testnet.cfg index 9abeecc8f5..3f0dec5095 100644 --- a/ingest/ledgerbackend/configs/captive-core-testnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-testnet.cfg @@ -2,6 +2,8 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015" UNSAFE_QUORUM=true FAILURE_SAFETY=1 +EXPERIMENTAL_BUCKETLIST_DB=true + [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" QUALITY="HIGH" diff --git a/services/horizon/docker/captive-core-classic-integration-tests.cfg b/services/horizon/docker/captive-core-classic-integration-tests.cfg index ed8ac3ed73..2f95a0ee54 100644 --- a/services/horizon/docker/captive-core-classic-integration-tests.cfg +++ b/services/horizon/docker/captive-core-classic-integration-tests.cfg @@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 +EXPERIMENTAL_BUCKETLIST_DB=true + [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg index 275599bacd..02c59d7057 100644 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ b/services/horizon/docker/captive-core-integration-tests.cfg @@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 +EXPERIMENTAL_BUCKETLIST_DB=true + ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg index 9a7ad9d769..cf4d514975 100644 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg @@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 +EXPERIMENTAL_BUCKETLIST_DB=true + ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg index 4902cf8d15..735f58b739 100644 --- a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg @@ -1,5 +1,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true +EXPERIMENTAL_BUCKETLIST_DB=true + [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg index 44820f5933..4744fd390e 100644 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg @@ -2,6 +2,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true +EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-standalone.cfg b/services/horizon/docker/captive-core-standalone.cfg index d54b0ecae1..f5042a14df 100644 --- a/services/horizon/docker/captive-core-standalone.cfg +++ b/services/horizon/docker/captive-core-standalone.cfg @@ -3,6 +3,8 @@ PEER_PORT=11725 UNSAFE_QUORUM=true FAILURE_SAFETY=0 +EXPERIMENTAL_BUCKETLIST_DB=true + [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg index e27cfe14ed..f2b20b4927 100644 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-classic-integration-tests.cfg @@ -13,6 +13,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" +EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg index 594a35b244..0d5ec5cc43 100644 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-integration-tests.cfg @@ -13,6 +13,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" +EXPERIMENTAL_BUCKETLIST_DB=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/stellar-core-pubnet.cfg b/services/horizon/docker/stellar-core-pubnet.cfg index 94b29c4b4e..7f250b10c7 100644 --- a/services/horizon/docker/stellar-core-pubnet.cfg +++ b/services/horizon/docker/stellar-core-pubnet.cfg @@ -9,6 +9,8 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 +EXPERIMENTAL_BUCKETLIST_DB=true + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/stellar-core-standalone.cfg b/services/horizon/docker/stellar-core-standalone.cfg index a2b7e806c9..41c8a377aa 100644 --- a/services/horizon/docker/stellar-core-standalone.cfg +++ b/services/horizon/docker/stellar-core-standalone.cfg @@ -14,6 +14,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.internal port=5641 dbname=stellar" +EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-testnet.cfg b/services/horizon/docker/stellar-core-testnet.cfg index cf8546a3e9..55c5cea462 100644 --- a/services/horizon/docker/stellar-core-testnet.cfg +++ b/services/horizon/docker/stellar-core-testnet.cfg @@ -12,6 +12,8 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=100 +EXPERIMENTAL_BUCKETLIST_DB=true + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg index 5a702711fe..cedbbc65ab 100644 --- a/services/horizon/docker/verify-range/captive-core-pubnet.cfg +++ b/services/horizon/docker/verify-range/captive-core-pubnet.cfg @@ -2,6 +2,8 @@ PEER_PORT=11725 FAILURE_SAFETY=1 +EXPERIMENTAL_BUCKETLIST_DB=true + [[HOME_DOMAINS]] HOME_DOMAIN="stellar.org" QUALITY="HIGH" diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg index 6139f9f528..91bb190131 100644 --- a/services/horizon/docker/verify-range/stellar-core.cfg +++ b/services/horizon/docker/verify-range/stellar-core.cfg @@ -4,6 +4,8 @@ LOG_FILE_PATH="" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 +EXPERIMENTAL_BUCKETLIST_DB=true + AUTOMATIC_MAINTENANCE_COUNT=0 NODE_NAMES=[ diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index a30ea3a404..65d1da524c 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -86,6 +86,7 @@ func Test_createCaptiveCoreDefaultConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.config.CaptiveCoreTomlParams.UseDB = true e := setCaptiveCoreConfiguration(&tt.config, ApplyOptions{RequireCaptiveCoreFullConfig: true}) if tt.errStr == "" { @@ -192,6 +193,7 @@ func Test_createCaptiveCoreConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.config.CaptiveCoreTomlParams.UseDB = true e := setCaptiveCoreConfiguration(&tt.config, ApplyOptions{RequireCaptiveCoreFullConfig: tt.requireCaptiveCoreConfig}) if tt.errStr == "" { diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index eb22e9a068..133950d6f3 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -505,7 +505,7 @@ func TestDeprecatedOutputs(t *testing.T) { stdLog.SetOutput(os.Stderr) testConfig := integration.GetTestConfig() - testConfig.HorizonIngestParameters = map[string]string{"captive-core-use-db": "false"} + testConfig.HorizonIngestParameters = map[string]string{"captive-core-use-db": "true"} test := integration.NewTest(t, *testConfig) err := test.StartHorizon() assert.NoError(t, err) diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index d755d00252..e12f77f693 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -441,7 +441,7 @@ func (i *Test) getDefaultIngestArgs(postgres *dbtest.DB) map[string]string { "stellar-core-binary-path": i.coreConfig.binaryPath, "captive-core-config-path": i.coreConfig.configPath, "captive-core-http-port": "21626", - "captive-core-use-db": strconv.FormatBool(i.coreConfig.useDB), + "captive-core-use-db": "true", "captive-core-storage-path": i.coreConfig.storagePath, "ingest": "true"}) }