From 91d56a6de74a26eef13b94394e6c0df3aa0bb219 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Thu, 28 Jul 2022 16:33:50 +0100 Subject: [PATCH] Add --console flag for captive core 19.3.0 (#4487) * Add --console flag for captive core * Update Changelog * bump core to latest 19.3.0 * mutex around a global variable to avoid data race Co-authored-by: MonsieurNicolas --- .github/workflows/horizon.yml | 10 +++++----- ingest/ledgerbackend/stellar_core_runner.go | 2 +- services/horizon/CHANGELOG.md | 7 +++++++ .../docker-compose.integration-tests.yml | 2 +- .../horizon/internal/ingest/filters/main.go | 18 ++++++++++++++++-- .../integration/ingestion_filtering_test.go | 8 ++++---- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index 7c3e94d5b1..80bc0b12d2 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -34,10 +34,10 @@ jobs: env: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} - PROTOCOL_19_CORE_DEBIAN_PKG_VERSION: 19.2.0-966.d18d54aa3.focal - PROTOCOL_19_CORE_DOCKER_IMG: stellar/stellar-core:19.2.0-966.d18d54aa3.focal - PROTOCOL_18_CORE_DEBIAN_PKG_VERSION: 19.2.0-966.d18d54aa3.focal - PROTOCOL_18_CORE_DOCKER_IMG: stellar/stellar-core:19.2.0-966.d18d54aa3.focal + PROTOCOL_19_CORE_DEBIAN_PKG_VERSION: 19.3.0-1006.9ce6dc4e9.focal + PROTOCOL_19_CORE_DOCKER_IMG: stellar/stellar-core:19.3.0-1006.9ce6dc4e9.focal + PROTOCOL_18_CORE_DEBIAN_PKG_VERSION: 19.3.0-1006.9ce6dc4e9.focal + PROTOCOL_18_CORE_DOCKER_IMG: stellar/stellar-core:19.3.0-1006.9ce6dc4e9.focal PGHOST: localhost PGPORT: 5432 PGUSER: postgres @@ -97,7 +97,7 @@ jobs: name: Test (and push) verify-range image runs-on: ubuntu-latest env: - STELLAR_CORE_VERSION: 19.2.0-966.d18d54aa3.focal + STELLAR_CORE_VERSION: 19.3.0-1006.9ce6dc4e9.focal CAPTIVE_CORE_STORAGE_PATH: /tmp steps: - uses: actions/checkout@v3 diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index 876caf1355..b67c96032a 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -235,7 +235,7 @@ func (r *stellarCoreRunner) getLogLineWriter() io.Writer { } func (r *stellarCoreRunner) createCmd(params ...string) *exec.Cmd { - allParams := append([]string{"--conf", r.getConfFileName()}, params...) + allParams := append([]string{"--conf", r.getConfFileName(), "--console"}, params...) cmd := exec.Command(r.executablePath, allParams...) cmd.Dir = r.storagePath cmd.Stdout = r.getLogLineWriter() diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index a68a7bfd5b..4cbb6ee45e 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -7,6 +7,13 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). **Upgrading to this version from <= v2.8.3 will trigger a state rebuild. During this process (which will take at least 10 minutes), Horizon will not ingest new ledgers.** +### Breaking Changes + +* Update core version to 19.3.0 ([4485](https://github.com/stellar/go/pull/4485)). +* Pass `--console` to captive core. This is due to a breaking change in stellar-core 19.3.0 ([4487](https://github.com/stellar/go/pull/4487)). + +### Changes + * Run postgres autovacuum on `history_trades_60000` table more frequently. ([4412](https://github.com/stellar/go/pull/4412)). * Change `protocols/horizon.Transaction.AccountSequence` to `int64` from `string`. ([4409](https://github.com/stellar/go/pull/4409)). * Add missing signer key type names. ([4429](https://github.com/stellar/go/pull/4429)). diff --git a/services/horizon/docker/docker-compose.integration-tests.yml b/services/horizon/docker/docker-compose.integration-tests.yml index f15c69e432..d6dbc1360e 100644 --- a/services/horizon/docker/docker-compose.integration-tests.yml +++ b/services/horizon/docker/docker-compose.integration-tests.yml @@ -14,7 +14,7 @@ services: # Note: Please keep the image pinned to an immutable tag matching the Captive Core version. # This avoid implicit updates which break compatibility between # the Core container and captive core. - image: ${CORE_IMAGE:-stellar/stellar-core:19.2.0-966.d18d54aa3.focal} + image: ${CORE_IMAGE:-stellar/stellar-core:19.3.0-1006.9ce6dc4e9.focal} depends_on: - core-postgres restart: on-failure diff --git a/services/horizon/internal/ingest/filters/main.go b/services/horizon/internal/ingest/filters/main.go index 94e762d0ac..06532e5848 100644 --- a/services/horizon/internal/ingest/filters/main.go +++ b/services/horizon/internal/ingest/filters/main.go @@ -2,6 +2,7 @@ package filters import ( "context" + "sync" "time" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -13,9 +14,22 @@ var ( // the filter config cache will be checked against latest from db at most once per each of this interval. //lint:ignore ST1011, don't need the linter warn on literal assignment - FilterConfigCheckIntervalSeconds time.Duration = 100 + filterConfigCheckIntervalSeconds time.Duration = 100 + filterConfigCheckIntervalSecondsLock sync.RWMutex ) +func GetFilterConfigCheckIntervalSeconds() time.Duration { + filterConfigCheckIntervalSecondsLock.RLock() + defer filterConfigCheckIntervalSecondsLock.RUnlock() + return filterConfigCheckIntervalSeconds +} + +func SetFilterConfigCheckIntervalSeconds(t time.Duration) { + filterConfigCheckIntervalSecondsLock.Lock() + defer filterConfigCheckIntervalSecondsLock.Unlock() + filterConfigCheckIntervalSeconds = t +} + var ( LOG = log.WithFields(log.F{ "filters": "load", @@ -43,7 +57,7 @@ func NewFilters() Filters { // rebuild the list on expiration time interval. Method is NOT thread-safe. func (f *filtersCache) GetFilters(filterQ history.QFilter, ctx context.Context) []processors.LedgerTransactionFilterer { // only attempt to refresh filter config cache state at configured interval limit - if time.Now().Unix() < (f.lastFilterConfigCheckUnixEpoch + int64(FilterConfigCheckIntervalSeconds.Seconds())) { + if time.Now().Unix() < (f.lastFilterConfigCheckUnixEpoch + int64(GetFilterConfigCheckIntervalSeconds().Seconds())) { return f.convertCacheToList() } diff --git a/services/horizon/internal/integration/ingestion_filtering_test.go b/services/horizon/internal/integration/ingestion_filtering_test.go index 8eca0f3aa1..47cfc1ccbc 100644 --- a/services/horizon/internal/integration/ingestion_filtering_test.go +++ b/services/horizon/internal/integration/ingestion_filtering_test.go @@ -48,7 +48,7 @@ func TestFilteringAccountWhiteList(t *testing.T) { tt.NoError(err) // Setup a whitelisted account rule, force refresh of filter configs to be quick - filters.FilterConfigCheckIntervalSeconds = 1 + filters.SetFilterConfigCheckIntervalSeconds(1) expectedAccountFilter := hProtocol.AccountFilterConfig{ Whitelist: []string{whitelistedAccount.GetAccountID()}, @@ -64,7 +64,7 @@ func TestFilteringAccountWhiteList(t *testing.T) { tt.Equal(expectedAccountFilter.Enabled, accountFilter.Enabled) // Ensure the latest filter configs are reloaded by the ingestion state machine processor - time.Sleep(time.Duration(filters.FilterConfigCheckIntervalSeconds) * time.Second) + time.Sleep(time.Duration(filters.GetFilterConfigCheckIntervalSeconds()) * time.Second) // Make sure that when using a non-whitelisted account, the transaction is not stored txResp = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), @@ -123,7 +123,7 @@ func TestFilteringAssetWhiteList(t *testing.T) { tt.NoError(err) // Setup a whitelisted asset rule, force refresh of filters to be quick - filters.FilterConfigCheckIntervalSeconds = 1 + filters.SetFilterConfigCheckIntervalSeconds(1) asset, err := whitelistedAsset.ToXDR() tt.NoError(err) @@ -141,7 +141,7 @@ func TestFilteringAssetWhiteList(t *testing.T) { tt.Equal(expectedAssetFilter.Enabled, assetFilter.Enabled) // Ensure the latest filter configs are reloaded by the ingestion state machine processor - time.Sleep(time.Duration(filters.FilterConfigCheckIntervalSeconds) * time.Second) + time.Sleep(time.Duration(filters.GetFilterConfigCheckIntervalSeconds()) * time.Second) // Make sure that when using a non-whitelisted asset, the transaction is not stored txResp = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(),