From d59b3924a78821cb1548f56815dabf7bd7df274c Mon Sep 17 00:00:00 2001 From: tamirms Date: Wed, 14 Oct 2020 17:18:22 +0100 Subject: [PATCH] Move exp/ingest package to top level and rename expingest to ingest (#3118) The new Horizon ingestion implementation has been running in production for over 6 months. We are confident that we can remove the experimental warning from the ingestion implementation. --- exp/services/captivecore/internal/api.go | 2 +- exp/services/captivecore/internal/api_test.go | 2 +- exp/services/captivecore/internal/server.go | 2 +- .../captivecore/internal/server_test.go | 2 +- exp/services/captivecore/main.go | 2 +- exp/tools/dump-ledger-state/main.go | 2 +- .../adapters/history_archive_adapter.go | 2 +- .../adapters/history_archive_adapter_test.go | 0 {exp/ingest => ingest}/adapters/mocks.go | 2 +- {exp/ingest => ingest}/doc.go | 0 {exp/ingest => ingest}/doc_test.go | 4 +- {exp/ingest => ingest}/errors/main.go | 0 {exp/ingest => ingest}/io/change.go | 0 {exp/ingest => ingest}/io/errors.go | 0 .../io/genesis_ledger_state_reader.go | 0 .../io/genesis_ledger_state_reader_test.go | 0 .../io/ledger_change_reader.go | 2 +- .../io/ledger_change_reader_test.go | 2 +- .../io/ledger_entry_change_cache.go | 2 +- .../io/ledger_entry_change_cache_test.go | 0 .../io/ledger_transaction.go | 0 .../io/ledger_transaction_reader.go | 2 +- .../io/ledger_transaction_test.go | 0 {exp/ingest => ingest}/io/main.go | 0 {exp/ingest => ingest}/io/memory_temp_set.go | 0 .../io/memory_temp_set_test.go | 0 .../io/mock_change_processor.go | 0 .../io/mock_change_reader.go | 0 {exp/ingest => ingest}/io/processors.go | 0 {exp/ingest => ingest}/io/processors_test.go | 0 .../io/single_ledger_state_reader.go | 0 .../io/single_ledger_state_reader_test.go | 0 .../io/stats_change_processor.go | 0 .../io/stats_change_processor_test.go | 0 .../io/stats_ledger_transaction_processor.go | 0 ...stats_ledger_transaction_processor_test.go | 0 .../ledgerbackend/captive_core_backend.go | 0 .../captive_core_backend_test.go | 0 .../ledgerbackend/database_backend.go | 0 .../ledgerbackend/hash_order_test.go | 0 .../ledgerbackend/history_archive_backend.go | 0 .../history_archive_backend_test.go | 0 .../ledgerbackend/ledger_backend.go | 0 .../ledgerbackend/mock_database_backend.go | 0 .../ledgerbackend/remote_captive_core.go | 0 .../ledgerbackend/stellar_core_runner.go | 0 .../stellar_core_runner_posix.go | 0 .../stellar_core_runner_windows.go | 0 {exp/ingest => ingest}/verify/main.go | 4 +- {exp/ingest => ingest}/verify/main_test.go | 4 +- services/horizon/CHANGELOG.md | 4 ++ services/horizon/cmd/db.go | 10 ++-- services/horizon/cmd/ingest.go | 14 ++--- .../horizon/internal/action_offers_test.go | 4 +- .../horizon/internal/actions_account_test.go | 4 +- .../horizon/internal/actions_data_test.go | 4 +- .../horizon/internal/actions_effects_test.go | 4 +- .../horizon/internal/actions_payment_test.go | 4 +- .../internal/actions_transaction_test.go | 4 +- services/horizon/internal/app.go | 20 +++---- .../internal/db2/history/fee_bump_scenario.go | 2 +- .../horizon/internal/db2/history/key_value.go | 4 +- .../mock_transactions_batch_insert_builder.go | 2 +- .../transaction_batch_insert_builder.go | 2 +- .../transaction_batch_insert_builder_test.go | 2 +- .../internal/db2/history/transaction_test.go | 2 +- services/horizon/internal/docs/admin.md | 50 +++++++++--------- services/horizon/internal/httpx/middleware.go | 4 +- .../internal/{expingest => ingest}/TESTING.md | 0 .../{expingest => ingest}/build_state_test.go | 10 ++-- .../database_backend_test.go | 4 +- .../db_integration_test.go | 10 ++-- .../fake_ledger_backend.go | 4 +- .../internal/{expingest => ingest}/fsm.go | 12 ++--- .../{expingest => ingest}/group_processors.go | 4 +- .../group_processors_test.go | 12 ++--- .../ingest_history_range_state_test.go | 8 +-- .../{expingest => ingest}/init_state_test.go | 16 +++--- .../logger_change_reader_test.go | 4 +- .../logging_change_reader.go | 4 +- .../internal/{expingest => ingest}/main.go | 12 ++--- .../{expingest => ingest}/main_test.go | 8 +-- .../mock_orderbook_graph.go | 2 +- .../{expingest => ingest}/orderbook.go | 2 +- .../{expingest => ingest}/orderbook_test.go | 2 +- .../{expingest => ingest}/parallel.go | 2 +- .../{expingest => ingest}/parallel_test.go | 2 +- .../{expingest => ingest}/processor_runner.go | 10 ++-- .../processor_runner_test.go | 10 ++-- .../processors/account_data_processor.go | 4 +- .../accounts_data_processor_test.go | 2 +- .../processors/accounts_processor.go | 4 +- .../processors/accounts_processor_test.go | 2 +- .../processors/asset_stats_processor.go | 4 +- .../processors/asset_stats_processor_test.go | 2 +- .../processors/asset_stats_set.go | 0 .../processors/asset_stats_set_test.go | 0 .../claimable_balances_processor.go | 4 +- .../claimable_balances_processor_test.go | 2 +- .../processors/effects_processor.go | 2 +- .../processors/effects_processor_test.go | 2 +- .../processors/ledgers_processor.go | 2 +- .../processors/ledgers_processor_test.go | 2 +- .../{expingest => ingest}/processors/main.go | 2 +- .../processors/offers_processor.go | 4 +- .../processors/offers_processor_test.go | 4 +- .../processors/operations_processor.go | 2 +- .../processors/operations_processor_test.go | 2 +- .../processors/participants_processor.go | 2 +- .../processors/participants_processor_test.go | 2 +- .../processors/participants_test.go | 2 +- .../processors/signer_processor_test.go | 4 +- .../processors/signers_processor.go | 4 +- .../processors/trades_processor.go | 2 +- .../processors/trades_processor_test.go | 2 +- .../transaction_operation_wrapper_test.go | 2 +- .../processors/transactions_processor.go | 2 +- .../processors/transactions_processor_test.go | 0 .../processors/trust_lines_processor.go | 4 +- .../processors/trust_lines_processor_test.go | 4 +- .../resume_state_test.go | 8 +-- .../sample_changes_test.go | 6 +-- .../{expingest => ingest}/stress_test.go | 6 +-- .../testdata/sample-changes.xdr | Bin .../internal/{expingest => ingest}/verify.go | 8 +-- .../verify_range_state_test.go | 6 +-- services/horizon/internal/init.go | 26 ++++----- services/horizon/internal/middleware_test.go | 22 ++++---- .../internal/test/transactions/main.go | 2 +- tools/archive-reader/archive_reader.go | 2 +- 130 files changed, 240 insertions(+), 236 deletions(-) rename {exp/ingest => ingest}/adapters/history_archive_adapter.go (98%) rename {exp/ingest => ingest}/adapters/history_archive_adapter_test.go (100%) rename {exp/ingest => ingest}/adapters/mocks.go (94%) rename {exp/ingest => ingest}/doc.go (100%) rename {exp/ingest => ingest}/doc_test.go (97%) rename {exp/ingest => ingest}/errors/main.go (100%) rename {exp/ingest => ingest}/io/change.go (100%) rename {exp/ingest => ingest}/io/errors.go (100%) rename {exp/ingest => ingest}/io/genesis_ledger_state_reader.go (100%) rename {exp/ingest => ingest}/io/genesis_ledger_state_reader_test.go (100%) rename {exp/ingest => ingest}/io/ledger_change_reader.go (98%) rename {exp/ingest => ingest}/io/ledger_change_reader_test.go (99%) rename {exp/ingest => ingest}/io/ledger_entry_change_cache.go (99%) rename {exp/ingest => ingest}/io/ledger_entry_change_cache_test.go (100%) rename {exp/ingest => ingest}/io/ledger_transaction.go (100%) rename {exp/ingest => ingest}/io/ledger_transaction_reader.go (98%) rename {exp/ingest => ingest}/io/ledger_transaction_test.go (100%) rename {exp/ingest => ingest}/io/main.go (100%) rename {exp/ingest => ingest}/io/memory_temp_set.go (100%) rename {exp/ingest => ingest}/io/memory_temp_set_test.go (100%) rename {exp/ingest => ingest}/io/mock_change_processor.go (100%) rename {exp/ingest => ingest}/io/mock_change_reader.go (100%) rename {exp/ingest => ingest}/io/processors.go (100%) rename {exp/ingest => ingest}/io/processors_test.go (100%) rename {exp/ingest => ingest}/io/single_ledger_state_reader.go (100%) rename {exp/ingest => ingest}/io/single_ledger_state_reader_test.go (100%) rename {exp/ingest => ingest}/io/stats_change_processor.go (100%) rename {exp/ingest => ingest}/io/stats_change_processor_test.go (100%) rename {exp/ingest => ingest}/io/stats_ledger_transaction_processor.go (100%) rename {exp/ingest => ingest}/io/stats_ledger_transaction_processor_test.go (100%) rename {exp/ingest => ingest}/ledgerbackend/captive_core_backend.go (100%) rename {exp/ingest => ingest}/ledgerbackend/captive_core_backend_test.go (100%) rename {exp/ingest => ingest}/ledgerbackend/database_backend.go (100%) rename {exp/ingest => ingest}/ledgerbackend/hash_order_test.go (100%) rename {exp/ingest => ingest}/ledgerbackend/history_archive_backend.go (100%) rename {exp/ingest => ingest}/ledgerbackend/history_archive_backend_test.go (100%) rename {exp/ingest => ingest}/ledgerbackend/ledger_backend.go (100%) rename {exp/ingest => ingest}/ledgerbackend/mock_database_backend.go (100%) rename {exp/ingest => ingest}/ledgerbackend/remote_captive_core.go (100%) rename {exp/ingest => ingest}/ledgerbackend/stellar_core_runner.go (100%) rename {exp/ingest => ingest}/ledgerbackend/stellar_core_runner_posix.go (100%) rename {exp/ingest => ingest}/ledgerbackend/stellar_core_runner_windows.go (100%) rename {exp/ingest => ingest}/verify/main.go (98%) rename {exp/ingest => ingest}/verify/main_test.go (98%) rename services/horizon/internal/{expingest => ingest}/TESTING.md (100%) rename services/horizon/internal/{expingest => ingest}/build_state_test.go (98%) rename services/horizon/internal/{expingest => ingest}/database_backend_test.go (93%) rename services/horizon/internal/{expingest => ingest}/db_integration_test.go (94%) rename services/horizon/internal/{expingest => ingest}/fake_ledger_backend.go (98%) rename services/horizon/internal/{expingest => ingest}/fsm.go (98%) rename services/horizon/internal/{expingest => ingest}/group_processors.go (95%) rename services/horizon/internal/{expingest => ingest}/group_processors_test.go (90%) rename services/horizon/internal/{expingest => ingest}/ingest_history_range_state_test.go (99%) rename services/horizon/internal/{expingest => ingest}/init_state_test.go (95%) rename services/horizon/internal/{expingest => ingest}/logger_change_reader_test.go (96%) rename services/horizon/internal/{expingest => ingest}/logging_change_reader.go (95%) rename services/horizon/internal/{expingest => ingest}/main.go (98%) rename services/horizon/internal/{expingest => ingest}/main_test.go (98%) rename services/horizon/internal/{expingest => ingest}/mock_orderbook_graph.go (98%) rename services/horizon/internal/{expingest => ingest}/orderbook.go (99%) rename services/horizon/internal/{expingest => ingest}/orderbook_test.go (99%) rename services/horizon/internal/{expingest => ingest}/parallel.go (99%) rename services/horizon/internal/{expingest => ingest}/parallel_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processor_runner.go (97%) rename services/horizon/internal/{expingest => ingest}/processor_runner_test.go (98%) rename services/horizon/internal/{expingest => ingest}/processors/account_data_processor.go (96%) rename services/horizon/internal/{expingest => ingest}/processors/accounts_data_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/accounts_processor.go (95%) rename services/horizon/internal/{expingest => ingest}/processors/accounts_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/asset_stats_processor.go (98%) rename services/horizon/internal/{expingest => ingest}/processors/asset_stats_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/asset_stats_set.go (100%) rename services/horizon/internal/{expingest => ingest}/processors/asset_stats_set_test.go (100%) rename services/horizon/internal/{expingest => ingest}/processors/claimable_balances_processor.go (96%) rename services/horizon/internal/{expingest => ingest}/processors/claimable_balances_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/effects_processor.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/effects_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/ledgers_processor.go (97%) rename services/horizon/internal/{expingest => ingest}/processors/ledgers_processor_test.go (98%) rename services/horizon/internal/{expingest => ingest}/processors/main.go (85%) rename services/horizon/internal/{expingest => ingest}/processors/offers_processor.go (97%) rename services/horizon/internal/{expingest => ingest}/processors/offers_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/operations_processor.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/operations_processor_test.go (98%) rename services/horizon/internal/{expingest => ingest}/processors/participants_processor.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/participants_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/participants_test.go (98%) rename services/horizon/internal/{expingest => ingest}/processors/signer_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/signers_processor.go (97%) rename services/horizon/internal/{expingest => ingest}/processors/trades_processor.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/trades_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/transaction_operation_wrapper_test.go (99%) rename services/horizon/internal/{expingest => ingest}/processors/transactions_processor.go (96%) rename services/horizon/internal/{expingest => ingest}/processors/transactions_processor_test.go (100%) rename services/horizon/internal/{expingest => ingest}/processors/trust_lines_processor.go (96%) rename services/horizon/internal/{expingest => ingest}/processors/trust_lines_processor_test.go (99%) rename services/horizon/internal/{expingest => ingest}/resume_state_test.go (98%) rename services/horizon/internal/{expingest => ingest}/sample_changes_test.go (98%) rename services/horizon/internal/{expingest => ingest}/stress_test.go (97%) rename services/horizon/internal/{expingest => ingest}/testdata/sample-changes.xdr (100%) rename services/horizon/internal/{expingest => ingest}/verify.go (99%) rename services/horizon/internal/{expingest => ingest}/verify_range_state_test.go (99%) diff --git a/exp/services/captivecore/internal/api.go b/exp/services/captivecore/internal/api.go index 5740c5a472..9b70c19188 100644 --- a/exp/services/captivecore/internal/api.go +++ b/exp/services/captivecore/internal/api.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" ) diff --git a/exp/services/captivecore/internal/api_test.go b/exp/services/captivecore/internal/api_test.go index 5819c8cddc..223f5e9bb4 100644 --- a/exp/services/captivecore/internal/api_test.go +++ b/exp/services/captivecore/internal/api_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/suite" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" ) diff --git a/exp/services/captivecore/internal/server.go b/exp/services/captivecore/internal/server.go index 39bc92718b..b451a35c93 100644 --- a/exp/services/captivecore/internal/server.go +++ b/exp/services/captivecore/internal/server.go @@ -4,7 +4,7 @@ import ( "encoding/json" "net/http" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" supporthttp "github.com/stellar/go/support/http" "github.com/stellar/go/support/http/httpdecode" supportlog "github.com/stellar/go/support/log" diff --git a/exp/services/captivecore/internal/server_test.go b/exp/services/captivecore/internal/server_test.go index 0aab9606e6..1923767ee4 100644 --- a/exp/services/captivecore/internal/server_test.go +++ b/exp/services/captivecore/internal/server_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/suite" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" ) diff --git a/exp/services/captivecore/main.go b/exp/services/captivecore/main.go index ac3ab6c5ea..c9677520fe 100644 --- a/exp/services/captivecore/main.go +++ b/exp/services/captivecore/main.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" - "github.com/stellar/go/exp/ingest/ledgerbackend" "github.com/stellar/go/exp/services/captivecore/internal" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/config" supporthttp "github.com/stellar/go/support/http" diff --git a/exp/tools/dump-ledger-state/main.go b/exp/tools/dump-ledger-state/main.go index d15528168b..23deb734d6 100644 --- a/exp/tools/dump-ledger-state/main.go +++ b/exp/tools/dump-ledger-state/main.go @@ -11,8 +11,8 @@ import ( "strconv" "time" - "github.com/stellar/go/exp/ingest/io" "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" diff --git a/exp/ingest/adapters/history_archive_adapter.go b/ingest/adapters/history_archive_adapter.go similarity index 98% rename from exp/ingest/adapters/history_archive_adapter.go rename to ingest/adapters/history_archive_adapter.go index 430e46f1cf..a97d4f76ef 100644 --- a/exp/ingest/adapters/history_archive_adapter.go +++ b/ingest/adapters/history_archive_adapter.go @@ -3,8 +3,8 @@ package adapters import ( "context" - "github.com/stellar/go/exp/ingest/io" "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) diff --git a/exp/ingest/adapters/history_archive_adapter_test.go b/ingest/adapters/history_archive_adapter_test.go similarity index 100% rename from exp/ingest/adapters/history_archive_adapter_test.go rename to ingest/adapters/history_archive_adapter_test.go diff --git a/exp/ingest/adapters/mocks.go b/ingest/adapters/mocks.go similarity index 94% rename from exp/ingest/adapters/mocks.go rename to ingest/adapters/mocks.go index d820e44cf4..254e1f3b9c 100644 --- a/exp/ingest/adapters/mocks.go +++ b/ingest/adapters/mocks.go @@ -3,7 +3,7 @@ package adapters import ( "context" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/xdr" "github.com/stretchr/testify/mock" ) diff --git a/exp/ingest/doc.go b/ingest/doc.go similarity index 100% rename from exp/ingest/doc.go rename to ingest/doc.go diff --git a/exp/ingest/doc_test.go b/ingest/doc_test.go similarity index 97% rename from exp/ingest/doc_test.go rename to ingest/doc_test.go index 9c69eaba91..5c8b7a2bd7 100644 --- a/exp/ingest/doc_test.go +++ b/ingest/doc_test.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/xdr" ) diff --git a/exp/ingest/errors/main.go b/ingest/errors/main.go similarity index 100% rename from exp/ingest/errors/main.go rename to ingest/errors/main.go diff --git a/exp/ingest/io/change.go b/ingest/io/change.go similarity index 100% rename from exp/ingest/io/change.go rename to ingest/io/change.go diff --git a/exp/ingest/io/errors.go b/ingest/io/errors.go similarity index 100% rename from exp/ingest/io/errors.go rename to ingest/io/errors.go diff --git a/exp/ingest/io/genesis_ledger_state_reader.go b/ingest/io/genesis_ledger_state_reader.go similarity index 100% rename from exp/ingest/io/genesis_ledger_state_reader.go rename to ingest/io/genesis_ledger_state_reader.go diff --git a/exp/ingest/io/genesis_ledger_state_reader_test.go b/ingest/io/genesis_ledger_state_reader_test.go similarity index 100% rename from exp/ingest/io/genesis_ledger_state_reader_test.go rename to ingest/io/genesis_ledger_state_reader_test.go diff --git a/exp/ingest/io/ledger_change_reader.go b/ingest/io/ledger_change_reader.go similarity index 98% rename from exp/ingest/io/ledger_change_reader.go rename to ingest/io/ledger_change_reader.go index 36cbe6d3d8..4c089f82c3 100644 --- a/exp/ingest/io/ledger_change_reader.go +++ b/ingest/io/ledger_change_reader.go @@ -3,7 +3,7 @@ package io import ( "io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" ) // ChangeReader provides convenient, streaming access to a sequence of Changes. diff --git a/exp/ingest/io/ledger_change_reader_test.go b/ingest/io/ledger_change_reader_test.go similarity index 99% rename from exp/ingest/io/ledger_change_reader_test.go rename to ingest/io/ledger_change_reader_test.go index 61668b9df9..6784c65653 100644 --- a/exp/ingest/io/ledger_change_reader_test.go +++ b/ingest/io/ledger_change_reader_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/xdr" ) diff --git a/exp/ingest/io/ledger_entry_change_cache.go b/ingest/io/ledger_entry_change_cache.go similarity index 99% rename from exp/ingest/io/ledger_entry_change_cache.go rename to ingest/io/ledger_entry_change_cache.go index ebdcfd24a0..d438e80099 100644 --- a/exp/ingest/io/ledger_entry_change_cache.go +++ b/ingest/io/ledger_entry_change_cache.go @@ -3,7 +3,7 @@ package io import ( "sync" - ingesterrors "github.com/stellar/go/exp/ingest/errors" + ingesterrors "github.com/stellar/go/ingest/errors" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) diff --git a/exp/ingest/io/ledger_entry_change_cache_test.go b/ingest/io/ledger_entry_change_cache_test.go similarity index 100% rename from exp/ingest/io/ledger_entry_change_cache_test.go rename to ingest/io/ledger_entry_change_cache_test.go diff --git a/exp/ingest/io/ledger_transaction.go b/ingest/io/ledger_transaction.go similarity index 100% rename from exp/ingest/io/ledger_transaction.go rename to ingest/io/ledger_transaction.go diff --git a/exp/ingest/io/ledger_transaction_reader.go b/ingest/io/ledger_transaction_reader.go similarity index 98% rename from exp/ingest/io/ledger_transaction_reader.go rename to ingest/io/ledger_transaction_reader.go index 7f73bba187..34a693a992 100644 --- a/exp/ingest/io/ledger_transaction_reader.go +++ b/ingest/io/ledger_transaction_reader.go @@ -4,7 +4,7 @@ import ( "encoding/hex" "io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/exp/ingest/io/ledger_transaction_test.go b/ingest/io/ledger_transaction_test.go similarity index 100% rename from exp/ingest/io/ledger_transaction_test.go rename to ingest/io/ledger_transaction_test.go diff --git a/exp/ingest/io/main.go b/ingest/io/main.go similarity index 100% rename from exp/ingest/io/main.go rename to ingest/io/main.go diff --git a/exp/ingest/io/memory_temp_set.go b/ingest/io/memory_temp_set.go similarity index 100% rename from exp/ingest/io/memory_temp_set.go rename to ingest/io/memory_temp_set.go diff --git a/exp/ingest/io/memory_temp_set_test.go b/ingest/io/memory_temp_set_test.go similarity index 100% rename from exp/ingest/io/memory_temp_set_test.go rename to ingest/io/memory_temp_set_test.go diff --git a/exp/ingest/io/mock_change_processor.go b/ingest/io/mock_change_processor.go similarity index 100% rename from exp/ingest/io/mock_change_processor.go rename to ingest/io/mock_change_processor.go diff --git a/exp/ingest/io/mock_change_reader.go b/ingest/io/mock_change_reader.go similarity index 100% rename from exp/ingest/io/mock_change_reader.go rename to ingest/io/mock_change_reader.go diff --git a/exp/ingest/io/processors.go b/ingest/io/processors.go similarity index 100% rename from exp/ingest/io/processors.go rename to ingest/io/processors.go diff --git a/exp/ingest/io/processors_test.go b/ingest/io/processors_test.go similarity index 100% rename from exp/ingest/io/processors_test.go rename to ingest/io/processors_test.go diff --git a/exp/ingest/io/single_ledger_state_reader.go b/ingest/io/single_ledger_state_reader.go similarity index 100% rename from exp/ingest/io/single_ledger_state_reader.go rename to ingest/io/single_ledger_state_reader.go diff --git a/exp/ingest/io/single_ledger_state_reader_test.go b/ingest/io/single_ledger_state_reader_test.go similarity index 100% rename from exp/ingest/io/single_ledger_state_reader_test.go rename to ingest/io/single_ledger_state_reader_test.go diff --git a/exp/ingest/io/stats_change_processor.go b/ingest/io/stats_change_processor.go similarity index 100% rename from exp/ingest/io/stats_change_processor.go rename to ingest/io/stats_change_processor.go diff --git a/exp/ingest/io/stats_change_processor_test.go b/ingest/io/stats_change_processor_test.go similarity index 100% rename from exp/ingest/io/stats_change_processor_test.go rename to ingest/io/stats_change_processor_test.go diff --git a/exp/ingest/io/stats_ledger_transaction_processor.go b/ingest/io/stats_ledger_transaction_processor.go similarity index 100% rename from exp/ingest/io/stats_ledger_transaction_processor.go rename to ingest/io/stats_ledger_transaction_processor.go diff --git a/exp/ingest/io/stats_ledger_transaction_processor_test.go b/ingest/io/stats_ledger_transaction_processor_test.go similarity index 100% rename from exp/ingest/io/stats_ledger_transaction_processor_test.go rename to ingest/io/stats_ledger_transaction_processor_test.go diff --git a/exp/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go similarity index 100% rename from exp/ingest/ledgerbackend/captive_core_backend.go rename to ingest/ledgerbackend/captive_core_backend.go diff --git a/exp/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go similarity index 100% rename from exp/ingest/ledgerbackend/captive_core_backend_test.go rename to ingest/ledgerbackend/captive_core_backend_test.go diff --git a/exp/ingest/ledgerbackend/database_backend.go b/ingest/ledgerbackend/database_backend.go similarity index 100% rename from exp/ingest/ledgerbackend/database_backend.go rename to ingest/ledgerbackend/database_backend.go diff --git a/exp/ingest/ledgerbackend/hash_order_test.go b/ingest/ledgerbackend/hash_order_test.go similarity index 100% rename from exp/ingest/ledgerbackend/hash_order_test.go rename to ingest/ledgerbackend/hash_order_test.go diff --git a/exp/ingest/ledgerbackend/history_archive_backend.go b/ingest/ledgerbackend/history_archive_backend.go similarity index 100% rename from exp/ingest/ledgerbackend/history_archive_backend.go rename to ingest/ledgerbackend/history_archive_backend.go diff --git a/exp/ingest/ledgerbackend/history_archive_backend_test.go b/ingest/ledgerbackend/history_archive_backend_test.go similarity index 100% rename from exp/ingest/ledgerbackend/history_archive_backend_test.go rename to ingest/ledgerbackend/history_archive_backend_test.go diff --git a/exp/ingest/ledgerbackend/ledger_backend.go b/ingest/ledgerbackend/ledger_backend.go similarity index 100% rename from exp/ingest/ledgerbackend/ledger_backend.go rename to ingest/ledgerbackend/ledger_backend.go diff --git a/exp/ingest/ledgerbackend/mock_database_backend.go b/ingest/ledgerbackend/mock_database_backend.go similarity index 100% rename from exp/ingest/ledgerbackend/mock_database_backend.go rename to ingest/ledgerbackend/mock_database_backend.go diff --git a/exp/ingest/ledgerbackend/remote_captive_core.go b/ingest/ledgerbackend/remote_captive_core.go similarity index 100% rename from exp/ingest/ledgerbackend/remote_captive_core.go rename to ingest/ledgerbackend/remote_captive_core.go diff --git a/exp/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go similarity index 100% rename from exp/ingest/ledgerbackend/stellar_core_runner.go rename to ingest/ledgerbackend/stellar_core_runner.go diff --git a/exp/ingest/ledgerbackend/stellar_core_runner_posix.go b/ingest/ledgerbackend/stellar_core_runner_posix.go similarity index 100% rename from exp/ingest/ledgerbackend/stellar_core_runner_posix.go rename to ingest/ledgerbackend/stellar_core_runner_posix.go diff --git a/exp/ingest/ledgerbackend/stellar_core_runner_windows.go b/ingest/ledgerbackend/stellar_core_runner_windows.go similarity index 100% rename from exp/ingest/ledgerbackend/stellar_core_runner_windows.go rename to ingest/ledgerbackend/stellar_core_runner_windows.go diff --git a/exp/ingest/verify/main.go b/ingest/verify/main.go similarity index 98% rename from exp/ingest/verify/main.go rename to ingest/verify/main.go index 1d770d5310..b040823288 100644 --- a/exp/ingest/verify/main.go +++ b/ingest/verify/main.go @@ -7,8 +7,8 @@ import ( "encoding/base64" stdio "io" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) diff --git a/exp/ingest/verify/main_test.go b/ingest/verify/main_test.go similarity index 98% rename from exp/ingest/verify/main_test.go rename to ingest/verify/main_test.go index 5ec00e4d6f..eea220dc38 100644 --- a/exp/ingest/verify/main_test.go +++ b/ingest/verify/main_test.go @@ -5,8 +5,8 @@ import ( stdio "io" "testing" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" "github.com/stretchr/testify/assert" diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 5abfd4b78b..b1d5f7981f 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).x +## Unreleased + +* The `service` field emitted in ingestion logs has been changed from `expingest` to `ingest`. + ## v1.10.0 **After upgrading Horizon will rebuild its state. During this process (which can take several minutes) it will not ingest new ledgers.** diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index a677ea609f..dd874fc3e6 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -13,7 +13,7 @@ import ( horizon "github.com/stellar/go/services/horizon/internal" "github.com/stellar/go/services/horizon/internal/db2/schema" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" support "github.com/stellar/go/support/config" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" @@ -218,7 +218,7 @@ var dbReingestRangeCmd = &cobra.Command{ log.Fatalf("cannot open Horizon DB: %v", err) } - ingestConfig := expingest.Config{ + ingestConfig := ingest.Config{ NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], @@ -241,7 +241,7 @@ var dbReingestRangeCmd = &cobra.Command{ } if parallelWorkers < 2 { - system, systemErr := expingest.NewSystem(ingestConfig) + system, systemErr := ingest.NewSystem(ingestConfig) if systemErr != nil { log.Fatal(systemErr) } @@ -252,7 +252,7 @@ var dbReingestRangeCmd = &cobra.Command{ reingestForce, ) } else { - system, systemErr := expingest.NewParallelSystems(ingestConfig, parallelWorkers) + system, systemErr := ingest.NewParallelSystems(ingestConfig, parallelWorkers) if systemErr != nil { log.Fatal(systemErr) } @@ -269,7 +269,7 @@ var dbReingestRangeCmd = &cobra.Command{ return } - if errors.Cause(err) == expingest.ErrReingestRangeConflict { + if errors.Cause(err) == ingest.ErrReingestRangeConflict { message := ` The range you have provided overlaps with Horizon's most recently ingested ledger. It is not possible to run the reingest command on this range in parallel with diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index b57a3b4169..d31265ec81 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -11,7 +11,7 @@ import ( "github.com/stellar/go/historyarchive" horizon "github.com/stellar/go/services/horizon/internal" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" support "github.com/stellar/go/support/config" "github.com/stellar/go/support/db" "github.com/stellar/go/support/log" @@ -98,7 +98,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ log.Fatal("`--to` must be a checkpoint ledger when `--verify-state` is set.") } - ingestConfig := expingest.Config{ + ingestConfig := ingest.Config{ NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], @@ -119,7 +119,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ ingestConfig.CoreSession = coreSession } - system, err := expingest.NewSystem(ingestConfig) + system, err := ingest.NewSystem(ingestConfig) if err != nil { log.Fatal(err) } @@ -183,7 +183,7 @@ var ingestStressTestCmd = &cobra.Command{ log.Fatal("`--changes` must be positive") } - ingestConfig := expingest.Config{ + ingestConfig := ingest.Config{ NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], @@ -204,7 +204,7 @@ var ingestStressTestCmd = &cobra.Command{ ingestConfig.CoreSession = coreSession } - system, err := expingest.NewSystem(ingestConfig) + system, err := ingest.NewSystem(ingestConfig) if err != nil { log.Fatal(err) } @@ -264,7 +264,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{ log.Fatalf("cannot run on non-empty DB") } - ingestConfig := expingest.Config{ + ingestConfig := ingest.Config{ NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], @@ -284,7 +284,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{ ingestConfig.CoreSession = coreSession } - system, err := expingest.NewSystem(ingestConfig) + system, err := ingest.NewSystem(ingestConfig) if err != nil { log.Fatal(err) } diff --git a/services/horizon/internal/action_offers_test.go b/services/horizon/internal/action_offers_test.go index 75f81e4797..c4a1d3ae06 100644 --- a/services/horizon/internal/action_offers_test.go +++ b/services/horizon/internal/action_offers_test.go @@ -7,7 +7,7 @@ import ( "github.com/stellar/go/protocols/horizon" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/xdr" ) @@ -18,7 +18,7 @@ func TestOfferActions_Show(t *testing.T) { err := q.UpdateLastLedgerExpIngest(100) ht.Assert.NoError(err) - err = q.UpdateExpIngestVersion(expingest.CurrentVersion) + err = q.UpdateExpIngestVersion(ingest.CurrentVersion) ht.Assert.NoError(err) ledgerCloseTime := time.Now().Unix() diff --git a/services/horizon/internal/actions_account_test.go b/services/horizon/internal/actions_account_test.go index d60eafdb58..728364b910 100644 --- a/services/horizon/internal/actions_account_test.go +++ b/services/horizon/internal/actions_account_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/xdr" ) @@ -16,7 +16,7 @@ func TestAccountActions_InvalidID(t *testing.T) { q := history.Q{ht.HorizonSession()} err := q.UpdateLastLedgerExpIngest(100) ht.Assert.NoError(err) - err = q.UpdateExpIngestVersion(expingest.CurrentVersion) + err = q.UpdateExpIngestVersion(ingest.CurrentVersion) ht.Assert.NoError(err) _, err = q.InsertLedger(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ diff --git a/services/horizon/internal/actions_data_test.go b/services/horizon/internal/actions_data_test.go index 87eee29544..56daaca20b 100644 --- a/services/horizon/internal/actions_data_test.go +++ b/services/horizon/internal/actions_data_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/xdr" ) @@ -55,7 +55,7 @@ func TestDataActions_Show(t *testing.T) { // Makes StateMiddleware happy err := q.UpdateLastLedgerExpIngest(100) ht.Assert.NoError(err) - err = q.UpdateExpIngestVersion(expingest.CurrentVersion) + err = q.UpdateExpIngestVersion(ingest.CurrentVersion) ht.Assert.NoError(err) _, err = q.InsertLedger(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ diff --git a/services/horizon/internal/actions_effects_test.go b/services/horizon/internal/actions_effects_test.go index 8d487e16d3..d8510530a2 100644 --- a/services/horizon/internal/actions_effects_test.go +++ b/services/horizon/internal/actions_effects_test.go @@ -5,7 +5,7 @@ import ( "github.com/stellar/go/protocols/horizon/effects" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/test" ) @@ -45,7 +45,7 @@ func TestEffectActions_Index(t *testing.T) { q := history.Q{ht.HorizonSession()} err := q.UpdateLastLedgerExpIngest(3) ht.Assert.NoError(err) - err = q.UpdateExpIngestVersion(expingest.CurrentVersion) + err = q.UpdateExpIngestVersion(ingest.CurrentVersion) ht.Assert.NoError(err) // checks if empty param returns 404 instead of all payments diff --git a/services/horizon/internal/actions_payment_test.go b/services/horizon/internal/actions_payment_test.go index 04297f9f33..7b03fc1ed1 100644 --- a/services/horizon/internal/actions_payment_test.go +++ b/services/horizon/internal/actions_payment_test.go @@ -8,7 +8,7 @@ import ( "github.com/stellar/go/protocols/horizon" "github.com/stellar/go/protocols/horizon/operations" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" ) // Moved to TestGetOperationsOnlyPayments @@ -37,7 +37,7 @@ func TestPaymentActions(t *testing.T) { q := history.Q{ht.HorizonSession()} err := q.UpdateLastLedgerExpIngest(3) ht.Assert.NoError(err) - err = q.UpdateExpIngestVersion(expingest.CurrentVersion) + err = q.UpdateExpIngestVersion(ingest.CurrentVersion) ht.Assert.NoError(err) } initializeStateMiddleware() diff --git a/services/horizon/internal/actions_transaction_test.go b/services/horizon/internal/actions_transaction_test.go index 913e0ee6c3..ecdebeed16 100644 --- a/services/horizon/internal/actions_transaction_test.go +++ b/services/horizon/internal/actions_transaction_test.go @@ -9,7 +9,7 @@ import ( "github.com/stellar/go/protocols/horizon" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/xdr" ) @@ -195,7 +195,7 @@ func TestTransactionActions_Index(t *testing.T) { q := history.Q{ht.HorizonSession()} err := q.UpdateLastLedgerExpIngest(100) ht.Assert.NoError(err) - err = q.UpdateExpIngestVersion(expingest.CurrentVersion) + err = q.UpdateExpIngestVersion(ingest.CurrentVersion) ht.Assert.NoError(err) // checks if empty param returns 404 instead of all payments diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index a422afbf20..7c38960173 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -16,8 +16,8 @@ import ( proto "github.com/stellar/go/protocols/stellarcore" "github.com/stellar/go/services/horizon/internal/actions" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" "github.com/stellar/go/services/horizon/internal/httpx" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/ledger" "github.com/stellar/go/services/horizon/internal/logmetrics" "github.com/stellar/go/services/horizon/internal/operationfeestats" @@ -59,10 +59,10 @@ type App struct { cancel func() horizonVersion string coreSettings coreSettingsStore - orderBookStream *expingest.OrderBookStream + orderBookStream *ingest.OrderBookStream submitter *txsub.System paths paths.Finder - expingester expingest.System + ingester ingest.System reaper *reap.System ticks *time.Ticker @@ -116,10 +116,10 @@ func (a *App) Serve() { // all services gracefully shutdown. var wg sync.WaitGroup - if a.expingester != nil { + if a.ingester != nil { wg.Add(1) go func() { - a.expingester.Run() + a.ingester.Run() wg.Done() }() } @@ -155,8 +155,8 @@ func (a *App) waitForDone() { defer cancel() a.webServer.Shutdown(webShutdownCtx) a.cancel() - if a.expingester != nil { - a.expingester.Shutdown() + if a.ingester != nil { + a.ingester.Shutdown() } a.ticks.Stop() } @@ -175,8 +175,8 @@ func (a *App) HistoryQ() *history.Q { } // Ingestion returns the ingestion system associated with this Horizon instance -func (a *App) Ingestion() expingest.System { - return a.expingester +func (a *App) Ingestion() ingest.System { + return a.ingester } // HorizonSession returns a new session that loads data from the horizon @@ -428,7 +428,7 @@ func (a *App) init() error { mustInitHorizonDB(a) if a.config.Ingest { - // expingester + // ingester initExpIngester(a) } initPathFinder(a) diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go index dfc34d071e..d69de71348 100644 --- a/services/horizon/internal/db2/history/fee_bump_scenario.go +++ b/services/horizon/internal/db2/history/fee_bump_scenario.go @@ -8,7 +8,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/services/horizon/internal/toid" diff --git a/services/horizon/internal/db2/history/key_value.go b/services/horizon/internal/db2/history/key_value.go index bea4855d2f..7d40fe5d68 100644 --- a/services/horizon/internal/db2/history/key_value.go +++ b/services/horizon/internal/db2/history/key_value.go @@ -40,7 +40,7 @@ func (q *Q) GetLastLedgerExpIngestNonBlocking() (uint32, error) { } } -// GetLastLedgerExpIngest returns the last ledger ingested by expingest system +// GetLastLedgerExpIngest returns the last ledger ingested by ingest system // in Horizon. Returns ErrKeyNotFound error if no value has been previously set. // This is using `SELECT ... FOR UPDATE` what means it's blocking the row for all other // transactions.This behaviour is critical in distributed ingestion so do not change @@ -66,7 +66,7 @@ func (q *Q) GetLastLedgerExpIngest() (uint32, error) { } } -// UpdateLastLedgerExpIngest updates the last ledger ingested by expingest system. +// UpdateLastLedgerExpIngest updates the last ledger ingested by ingest system. // Can be read using GetLastLedgerExpIngest. func (q *Q) UpdateLastLedgerExpIngest(ledgerSequence uint32) error { return q.updateValueInStore( diff --git a/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go index 8e07444ba6..975b95dbfc 100644 --- a/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go @@ -1,7 +1,7 @@ package history import ( - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stretchr/testify/mock" ) diff --git a/services/horizon/internal/db2/history/transaction_batch_insert_builder.go b/services/horizon/internal/db2/history/transaction_batch_insert_builder.go index 5efd4c4c5f..81403b5477 100644 --- a/services/horizon/internal/db2/history/transaction_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/transaction_batch_insert_builder.go @@ -12,7 +12,7 @@ import ( "github.com/guregu/null" "github.com/lib/pq" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/services/horizon/internal/utf8" "github.com/stellar/go/support/db" diff --git a/services/horizon/internal/db2/history/transaction_batch_insert_builder_test.go b/services/horizon/internal/db2/history/transaction_batch_insert_builder_test.go index a3c06caa5f..287297f057 100644 --- a/services/horizon/internal/db2/history/transaction_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/transaction_batch_insert_builder_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/xdr" ) diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go index c9423b134c..a32a124c93 100644 --- a/services/horizon/internal/db2/history/transaction_test.go +++ b/services/horizon/internal/db2/history/transaction_test.go @@ -8,7 +8,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/services/horizon/internal/toid" ) diff --git a/services/horizon/internal/docs/admin.md b/services/horizon/internal/docs/admin.md index 8c301e7de0..5543f07baf 100644 --- a/services/horizon/internal/docs/admin.md +++ b/services/horizon/internal/docs/admin.md @@ -165,40 +165,40 @@ If you were running the new system in the past during experimental stage (`ENABL ## Reading the logs -In order to check the progress and the status of experimental ingestion you should check the logs. All logs connected to experimental ingestion are tagged with `service=expingest`. +In order to check the progress and the status of experimental ingestion you should check the logs. All logs connected to experimental ingestion are tagged with `service=ingest`. It starts with informing you about state ingestion: ``` -INFO[2019-08-29T13:04:13.473+02:00] Starting ingestion system from empty state... pid=5965 service=expingest temp_set="*io.MemoryTempSet" -INFO[2019-08-29T13:04:15.263+02:00] Reading from History Archive Snapshot ledger=25565887 pid=5965 service=expingest +INFO[2019-08-29T13:04:13.473+02:00] Starting ingestion system from empty state... pid=5965 service=ingest temp_set="*io.MemoryTempSet" +INFO[2019-08-29T13:04:15.263+02:00] Reading from History Archive Snapshot ledger=25565887 pid=5965 service=ingest ``` During state ingestion, Horizon will log number of processed entries every 100,000 entries (there are currently around 7M entries in the public network): ``` -INFO[2019-08-29T13:04:34.652+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=100000 pid=5965 service=expingest -INFO[2019-08-29T13:04:38.487+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=200000 pid=5965 service=expingest -INFO[2019-08-29T13:04:41.322+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=300000 pid=5965 service=expingest -INFO[2019-08-29T13:04:48.429+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=400000 pid=5965 service=expingest -INFO[2019-08-29T13:05:00.306+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=500000 pid=5965 service=expingest +INFO[2019-08-29T13:04:34.652+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=100000 pid=5965 service=ingest +INFO[2019-08-29T13:04:38.487+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=200000 pid=5965 service=ingest +INFO[2019-08-29T13:04:41.322+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=300000 pid=5965 service=ingest +INFO[2019-08-29T13:04:48.429+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=400000 pid=5965 service=ingest +INFO[2019-08-29T13:05:00.306+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=500000 pid=5965 service=ingest ``` When state ingestion is finished it will proceed to ledger ingestion starting from the next ledger after checkpoint ledger (25565887+1 in this example) to update the state using transaction meta: ``` -INFO[2019-08-29T13:39:41.590+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=5300000 pid=5965 service=expingest -INFO[2019-08-29T13:39:44.518+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=5400000 pid=5965 service=expingest -INFO[2019-08-29T13:39:47.488+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=5500000 pid=5965 service=expingest -INFO[2019-08-29T13:40:00.670+02:00] Processed ledger ledger=25565887 pid=5965 service=expingest type=state_pipeline -INFO[2019-08-29T13:40:00.670+02:00] Finished processing History Archive Snapshot duration=2145.337575904 ledger=25565887 numEntries=5529931 pid=5965 service=expingest shutdown=false -INFO[2019-08-29T13:40:00.693+02:00] Reading new ledger ledger=25565888 pid=5965 service=expingest -INFO[2019-08-29T13:40:00.694+02:00] Processing ledger ledger=25565888 pid=5965 service=expingest type=ledger_pipeline updating_database=true -INFO[2019-08-29T13:40:00.779+02:00] Processed ledger ledger=25565888 pid=5965 service=expingest type=ledger_pipeline -INFO[2019-08-29T13:40:00.779+02:00] Finished processing ledger duration=0.086024492 ledger=25565888 pid=5965 service=expingest shutdown=false transactions=14 -INFO[2019-08-29T13:40:00.815+02:00] Reading new ledger ledger=25565889 pid=5965 service=expingest -INFO[2019-08-29T13:40:00.816+02:00] Processing ledger ledger=25565889 pid=5965 service=expingest type=ledger_pipeline updating_database=true -INFO[2019-08-29T13:40:00.881+02:00] Processed ledger ledger=25565889 pid=5965 service=expingest type=ledger_pipeline -INFO[2019-08-29T13:40:00.881+02:00] Finished processing ledger duration=0.06619956 ledger=25565889 pid=5965 service=expingest shutdown=false transactions=29 -INFO[2019-08-29T13:40:00.901+02:00] Reading new ledger ledger=25565890 pid=5965 service=expingest -INFO[2019-08-29T13:40:00.902+02:00] Processing ledger ledger=25565890 pid=5965 service=expingest type=ledger_pipeline updating_database=true -INFO[2019-08-29T13:40:00.972+02:00] Processed ledger ledger=25565890 pid=5965 service=expingest type=ledger_pipeline -INFO[2019-08-29T13:40:00.972+02:00] Finished processing ledger duration=0.071039012 ledger=25565890 pid=5965 service=expingest shutdown=false transactions=20 +INFO[2019-08-29T13:39:41.590+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=5300000 pid=5965 service=ingest +INFO[2019-08-29T13:39:44.518+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=5400000 pid=5965 service=ingest +INFO[2019-08-29T13:39:47.488+02:00] Processing entries from History Archive Snapshot ledger=25565887 numEntries=5500000 pid=5965 service=ingest +INFO[2019-08-29T13:40:00.670+02:00] Processed ledger ledger=25565887 pid=5965 service=ingest type=state_pipeline +INFO[2019-08-29T13:40:00.670+02:00] Finished processing History Archive Snapshot duration=2145.337575904 ledger=25565887 numEntries=5529931 pid=5965 service=ingest shutdown=false +INFO[2019-08-29T13:40:00.693+02:00] Reading new ledger ledger=25565888 pid=5965 service=ingest +INFO[2019-08-29T13:40:00.694+02:00] Processing ledger ledger=25565888 pid=5965 service=ingest type=ledger_pipeline updating_database=true +INFO[2019-08-29T13:40:00.779+02:00] Processed ledger ledger=25565888 pid=5965 service=ingest type=ledger_pipeline +INFO[2019-08-29T13:40:00.779+02:00] Finished processing ledger duration=0.086024492 ledger=25565888 pid=5965 service=ingest shutdown=false transactions=14 +INFO[2019-08-29T13:40:00.815+02:00] Reading new ledger ledger=25565889 pid=5965 service=ingest +INFO[2019-08-29T13:40:00.816+02:00] Processing ledger ledger=25565889 pid=5965 service=ingest type=ledger_pipeline updating_database=true +INFO[2019-08-29T13:40:00.881+02:00] Processed ledger ledger=25565889 pid=5965 service=ingest type=ledger_pipeline +INFO[2019-08-29T13:40:00.881+02:00] Finished processing ledger duration=0.06619956 ledger=25565889 pid=5965 service=ingest shutdown=false transactions=29 +INFO[2019-08-29T13:40:00.901+02:00] Reading new ledger ledger=25565890 pid=5965 service=ingest +INFO[2019-08-29T13:40:00.902+02:00] Processing ledger ledger=25565890 pid=5965 service=ingest type=ledger_pipeline updating_database=true +INFO[2019-08-29T13:40:00.972+02:00] Processed ledger ledger=25565890 pid=5965 service=ingest type=ledger_pipeline +INFO[2019-08-29T13:40:00.972+02:00] Finished processing ledger duration=0.071039012 ledger=25565890 pid=5965 service=ingest shutdown=false transactions=20 ``` diff --git a/services/horizon/internal/httpx/middleware.go b/services/horizon/internal/httpx/middleware.go index e880db0f8c..e853b1e2b1 100644 --- a/services/horizon/internal/httpx/middleware.go +++ b/services/horizon/internal/httpx/middleware.go @@ -16,8 +16,8 @@ import ( horizonContext "github.com/stellar/go/services/horizon/internal/context" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/errors" - "github.com/stellar/go/services/horizon/internal/expingest" "github.com/stellar/go/services/horizon/internal/hchi" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/ledger" "github.com/stellar/go/services/horizon/internal/render" hProblem "github.com/stellar/go/services/horizon/internal/render/problem" @@ -281,7 +281,7 @@ func ingestionStatus(q *history.Q) (uint32, bool, error) { return 0, false, supportErrors.Wrap(err, "Error running LatestLedger") } - ready := version == expingest.CurrentVersion && + ready := version == ingest.CurrentVersion && lastIngestedLedger > 0 && lastIngestedLedger == lastHistoryLedger diff --git a/services/horizon/internal/expingest/TESTING.md b/services/horizon/internal/ingest/TESTING.md similarity index 100% rename from services/horizon/internal/expingest/TESTING.md rename to services/horizon/internal/ingest/TESTING.md diff --git a/services/horizon/internal/expingest/build_state_test.go b/services/horizon/internal/ingest/build_state_test.go similarity index 98% rename from services/horizon/internal/expingest/build_state_test.go rename to services/horizon/internal/ingest/build_state_test.go index 50bddbff8e..56fc1be43b 100644 --- a/services/horizon/internal/expingest/build_state_test.go +++ b/services/horizon/internal/ingest/build_state_test.go @@ -1,13 +1,13 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "context" "testing" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/errors" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" @@ -281,7 +281,7 @@ func (s *BuildStateTestSuite) TestUpdateExpIngestVersionIngestReturnsError() { next, err := buildState{checkpointLedger: s.checkpointLedger}.run(s.system) s.Assert().Error(err) - s.Assert().EqualError(err, "Error updating expingest version: my error") + s.Assert().EqualError(err, "Error updating ingest version: my error") s.Assert().Equal(transition{node: startState{}, sleepDuration: defaultSleep}, next) } diff --git a/services/horizon/internal/expingest/database_backend_test.go b/services/horizon/internal/ingest/database_backend_test.go similarity index 93% rename from services/horizon/internal/expingest/database_backend_test.go rename to services/horizon/internal/ingest/database_backend_test.go index d368ec4e19..cdd3ed5b2f 100644 --- a/services/horizon/internal/expingest/database_backend_test.go +++ b/services/horizon/internal/ingest/database_backend_test.go @@ -1,9 +1,9 @@ -package expingest +package ingest import ( "testing" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/test" ) diff --git a/services/horizon/internal/expingest/db_integration_test.go b/services/horizon/internal/ingest/db_integration_test.go similarity index 94% rename from services/horizon/internal/expingest/db_integration_test.go rename to services/horizon/internal/ingest/db_integration_test.go index 0b3808927c..b8657cf775 100644 --- a/services/horizon/internal/expingest/db_integration_test.go +++ b/services/horizon/internal/ingest/db_integration_test.go @@ -1,5 +1,5 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( stdio "io" @@ -7,9 +7,9 @@ import ( "path/filepath" "testing" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/xdr" "github.com/stretchr/testify/suite" @@ -71,7 +71,7 @@ func (s *DBTestSuite) SetupTest() { // sample-changes.xdr is generated by sample_changes_test.go and is checked into // the testdata directory. To regenerate the file run: - // go test -v -timeout 5m --tags=update github.com/stellar/go/services/horizon/internal/expingest -run "^(TestUpdateSampleChanges)$" + // go test -v -timeout 5m --tags=update github.com/stellar/go/services/horizon/internal/ingest -run "^(TestUpdateSampleChanges)$" // and commit the new file to the git repo. s.sampleFile = filepath.Join("testdata", "sample-changes.xdr") diff --git a/services/horizon/internal/expingest/fake_ledger_backend.go b/services/horizon/internal/ingest/fake_ledger_backend.go similarity index 98% rename from services/horizon/internal/expingest/fake_ledger_backend.go rename to services/horizon/internal/ingest/fake_ledger_backend.go index 2b42f9852b..30104b80e9 100644 --- a/services/horizon/internal/expingest/fake_ledger_backend.go +++ b/services/horizon/internal/ingest/fake_ledger_backend.go @@ -1,7 +1,7 @@ -package expingest +package ingest import ( - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/keypair" logpkg "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/fsm.go b/services/horizon/internal/ingest/fsm.go similarity index 98% rename from services/horizon/internal/expingest/fsm.go rename to services/horizon/internal/ingest/fsm.go index 3b0821d8fb..102663a938 100644 --- a/services/horizon/internal/expingest/fsm.go +++ b/services/horizon/internal/ingest/fsm.go @@ -1,12 +1,12 @@ -package expingest +package ingest import ( "fmt" "time" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/ledgerbackend" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/support/errors" logpkg "github.com/stellar/go/support/log" @@ -201,7 +201,7 @@ func (state startState) run(s *system) (transition, error) { case lastHistoryLedger > lastIngestedLedger: // Expingest was running at some point the past but was turned off. // Now it's on by default but the latest history ledger is greater - // than the latest expingest ledger. We reset the exp ledger sequence + // than the latest ingest ledger. We reset the exp ledger sequence // so init state will rebuild the state correctly. err = s.historyQ.UpdateLastLedgerExpIngest(0) if err != nil { @@ -219,7 +219,7 @@ func (state startState) run(s *system) (transition, error) { case lastHistoryLedger != 0 && lastHistoryLedger < lastIngestedLedger: // Expingest was running at some point the past but was turned off. // Now it's on by default but the latest history ledger is less - // than the latest expingest ledger. We catchup history. + // than the latest ingest ledger. We catchup history. return historyRange(lastHistoryLedger+1, lastIngestedLedger), nil default: // lastHistoryLedger == lastIngestedLedger // The other node already ingested a state (just now or in the past) @@ -324,7 +324,7 @@ func (b buildState) run(s *system) (transition, error) { } if err = s.historyQ.UpdateExpIngestVersion(CurrentVersion); err != nil { - return nextFailState, errors.Wrap(err, "Error updating expingest version") + return nextFailState, errors.Wrap(err, "Error updating ingest version") } if err = s.completeIngestion(b.checkpointLedger); err != nil { diff --git a/services/horizon/internal/expingest/group_processors.go b/services/horizon/internal/ingest/group_processors.go similarity index 95% rename from services/horizon/internal/expingest/group_processors.go rename to services/horizon/internal/ingest/group_processors.go index 4e6139cf74..e84402ab8f 100644 --- a/services/horizon/internal/expingest/group_processors.go +++ b/services/horizon/internal/ingest/group_processors.go @@ -1,7 +1,7 @@ -package expingest +package ingest import ( - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/support/errors" ) diff --git a/services/horizon/internal/expingest/group_processors_test.go b/services/horizon/internal/ingest/group_processors_test.go similarity index 90% rename from services/horizon/internal/expingest/group_processors_test.go rename to services/horizon/internal/ingest/group_processors_test.go index a333497ed8..6df698498e 100644 --- a/services/horizon/internal/expingest/group_processors_test.go +++ b/services/horizon/internal/ingest/group_processors_test.go @@ -1,11 +1,11 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "errors" "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -75,7 +75,7 @@ func (s *GroupChangeProcessorsTestSuiteLedger) TestProcessChangeFails() { err := s.processors.ProcessChange(change) s.Assert().Error(err) - s.Assert().EqualError(err, "error in *expingest.mockHorizonChangeProcessor.ProcessChange: transient error") + s.Assert().EqualError(err, "error in *ingest.mockHorizonChangeProcessor.ProcessChange: transient error") } func (s *GroupChangeProcessorsTestSuiteLedger) TestProcessChangeSucceeds() { @@ -98,7 +98,7 @@ func (s *GroupChangeProcessorsTestSuiteLedger) TestCommitFails() { err := s.processors.Commit() s.Assert().Error(err) - s.Assert().EqualError(err, "error in *expingest.mockHorizonChangeProcessor.Commit: transient error") + s.Assert().EqualError(err, "error in *ingest.mockHorizonChangeProcessor.Commit: transient error") } func (s *GroupChangeProcessorsTestSuiteLedger) TestCommitSucceeds() { @@ -146,7 +146,7 @@ func (s *GroupTransactionProcessorsTestSuiteLedger) TestProcessTransactionFails( err := s.processors.ProcessTransaction(transaction) s.Assert().Error(err) - s.Assert().EqualError(err, "error in *expingest.mockHorizonTransactionProcessor.ProcessTransaction: transient error") + s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.ProcessTransaction: transient error") } func (s *GroupTransactionProcessorsTestSuiteLedger) TestProcessTransactionSucceeds() { @@ -169,7 +169,7 @@ func (s *GroupTransactionProcessorsTestSuiteLedger) TestCommitFails() { err := s.processors.Commit() s.Assert().Error(err) - s.Assert().EqualError(err, "error in *expingest.mockHorizonTransactionProcessor.Commit: transient error") + s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.Commit: transient error") } func (s *GroupTransactionProcessorsTestSuiteLedger) TestCommitSucceeds() { diff --git a/services/horizon/internal/expingest/ingest_history_range_state_test.go b/services/horizon/internal/ingest/ingest_history_range_state_test.go similarity index 99% rename from services/horizon/internal/expingest/ingest_history_range_state_test.go rename to services/horizon/internal/ingest/ingest_history_range_state_test.go index a023056496..a8c391ac2c 100644 --- a/services/horizon/internal/expingest/ingest_history_range_state_test.go +++ b/services/horizon/internal/ingest/ingest_history_range_state_test.go @@ -1,14 +1,14 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "context" "testing" "github.com/jmoiron/sqlx" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/support/errors" "github.com/stretchr/testify/suite" diff --git a/services/horizon/internal/expingest/init_state_test.go b/services/horizon/internal/ingest/init_state_test.go similarity index 95% rename from services/horizon/internal/expingest/init_state_test.go rename to services/horizon/internal/ingest/init_state_test.go index baef0a341c..c8f5fc52fc 100644 --- a/services/horizon/internal/expingest/init_state_test.go +++ b/services/horizon/internal/ingest/init_state_test.go @@ -1,11 +1,11 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "context" "testing" - "github.com/stellar/go/exp/ingest/adapters" + "github.com/stellar/go/ingest/adapters" "github.com/stellar/go/support/errors" "github.com/stretchr/testify/suite" ) @@ -125,7 +125,7 @@ func (s *InitStateTestSuite) TestBuildStateEmptyDatabaseFromSuggestedCheckpoint( } // TestBuildStateWait is testing the case when: -// * the ingest system version has been incremented or no expingest ledger, +// * the ingest system version has been incremented or no ingest ledger, // * the old system is in front of the latest checkpoint. func (s *InitStateTestSuite) TestBuildStateWait() { s.historyQ.On("Begin").Return(nil).Once() @@ -144,7 +144,7 @@ func (s *InitStateTestSuite) TestBuildStateWait() { } // TestBuildStateCatchup is testing the case when: -// * the ingest system version has been incremented or no expingest ledger, +// * the ingest system version has been incremented or no ingest ledger, // * the old system is behind the latest checkpoint. func (s *InitStateTestSuite) TestBuildStateCatchup() { s.historyQ.On("Begin").Return(nil).Once() @@ -166,7 +166,7 @@ func (s *InitStateTestSuite) TestBuildStateCatchup() { } // TestBuildStateOldHistory is testing the case when: -// * the ingest system version has been incremented or no expingest ledger, +// * the ingest system version has been incremented or no ingest ledger, // * the old system latest ledger is equal to the latest checkpoint. func (s *InitStateTestSuite) TestBuildStateOldHistory() { s.historyQ.On("Begin").Return(nil).Once() @@ -189,7 +189,7 @@ func (s *InitStateTestSuite) TestBuildStateOldHistory() { // TestResumeStateInFront is testing the case when: // * state doesn't need to be rebuilt, -// * history is in front of expingest. +// * history is in front of ingest. func (s *InitStateTestSuite) TestResumeStateInFront() { s.historyQ.On("Begin").Return(nil).Once() s.historyQ.On("GetLastLedgerExpIngest").Return(uint32(100), nil).Once() @@ -206,7 +206,7 @@ func (s *InitStateTestSuite) TestResumeStateInFront() { // TestResumeStateBehind is testing the case when: // * state doesn't need to be rebuilt, -// * history is behind of expingest. +// * history is behind of ingest. func (s *InitStateTestSuite) TestResumeStateBehind() { s.historyQ.On("Begin").Return(nil).Once() s.historyQ.On("GetLastLedgerExpIngest").Return(uint32(130), nil).Once() @@ -247,7 +247,7 @@ func (s *InitStateTestSuite) TestResumeStateBehindHistory0() { // TestResumeStateBehind is testing the case when: // * state doesn't need to be rebuilt, -// * history is in sync with expingest. +// * history is in sync with ingest. func (s *InitStateTestSuite) TestResumeStateSync() { s.historyQ.On("Begin").Return(nil).Once() s.historyQ.On("GetLastLedgerExpIngest").Return(uint32(130), nil).Once() diff --git a/services/horizon/internal/expingest/logger_change_reader_test.go b/services/horizon/internal/ingest/logger_change_reader_test.go similarity index 96% rename from services/horizon/internal/expingest/logger_change_reader_test.go rename to services/horizon/internal/ingest/logger_change_reader_test.go index 644cce8af1..c6d529421a 100644 --- a/services/horizon/internal/expingest/logger_change_reader_test.go +++ b/services/horizon/internal/ingest/logger_change_reader_test.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "errors" @@ -6,7 +6,7 @@ import ( stdio "io" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/xdr" "github.com/stretchr/testify/assert" ) diff --git a/services/horizon/internal/expingest/logging_change_reader.go b/services/horizon/internal/ingest/logging_change_reader.go similarity index 95% rename from services/horizon/internal/expingest/logging_change_reader.go rename to services/horizon/internal/ingest/logging_change_reader.go index 352dc188d0..1247b27b6b 100644 --- a/services/horizon/internal/expingest/logging_change_reader.go +++ b/services/horizon/internal/ingest/logging_change_reader.go @@ -1,9 +1,9 @@ -package expingest +package ingest import ( "runtime" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" ) func bToMb(b uint64) uint64 { diff --git a/services/horizon/internal/expingest/main.go b/services/horizon/internal/ingest/main.go similarity index 98% rename from services/horizon/internal/expingest/main.go rename to services/horizon/internal/ingest/main.go index 3f210bc9ec..7957c40d09 100644 --- a/services/horizon/internal/expingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -1,7 +1,7 @@ -// Package expingest contains the new ingestion system for horizon. +// Package ingest contains the new ingestion system for horizon. // It currently runs completely independent of the old one, that means // that the new system can be ledgers behind/ahead the old system. -package expingest +package ingest import ( "context" @@ -11,10 +11,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/clients/stellarcore" - "github.com/stellar/go/exp/ingest/adapters" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/ledgerbackend" "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/adapters" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" @@ -60,7 +60,7 @@ const ( stateVerificationErrorThreshold = 3 ) -var log = logpkg.DefaultLogger.WithField("service", "expingest") +var log = logpkg.DefaultLogger.WithField("service", "ingest") type Config struct { CoreSession *db.Session diff --git a/services/horizon/internal/expingest/main_test.go b/services/horizon/internal/ingest/main_test.go similarity index 98% rename from services/horizon/internal/expingest/main_test.go rename to services/horizon/internal/ingest/main_test.go index 725b8f3fdc..a67b01d299 100644 --- a/services/horizon/internal/expingest/main_test.go +++ b/services/horizon/internal/ingest/main_test.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "bytes" @@ -10,9 +10,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/expingest/mock_orderbook_graph.go b/services/horizon/internal/ingest/mock_orderbook_graph.go similarity index 98% rename from services/horizon/internal/expingest/mock_orderbook_graph.go rename to services/horizon/internal/ingest/mock_orderbook_graph.go index fc5c54fc9b..6957044a25 100644 --- a/services/horizon/internal/expingest/mock_orderbook_graph.go +++ b/services/horizon/internal/ingest/mock_orderbook_graph.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "github.com/stellar/go/exp/orderbook" diff --git a/services/horizon/internal/expingest/orderbook.go b/services/horizon/internal/ingest/orderbook.go similarity index 99% rename from services/horizon/internal/expingest/orderbook.go rename to services/horizon/internal/ingest/orderbook.go index 3db0bc2536..9b9a7b0741 100644 --- a/services/horizon/internal/expingest/orderbook.go +++ b/services/horizon/internal/ingest/orderbook.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "context" diff --git a/services/horizon/internal/expingest/orderbook_test.go b/services/horizon/internal/ingest/orderbook_test.go similarity index 99% rename from services/horizon/internal/expingest/orderbook_test.go rename to services/horizon/internal/ingest/orderbook_test.go index 276258a05b..14a22469e3 100644 --- a/services/horizon/internal/expingest/orderbook_test.go +++ b/services/horizon/internal/ingest/orderbook_test.go @@ -1,5 +1,5 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "fmt" diff --git a/services/horizon/internal/expingest/parallel.go b/services/horizon/internal/ingest/parallel.go similarity index 99% rename from services/horizon/internal/expingest/parallel.go rename to services/horizon/internal/ingest/parallel.go index 2e48d3e8f2..af518d4b91 100644 --- a/services/horizon/internal/expingest/parallel.go +++ b/services/horizon/internal/ingest/parallel.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "fmt" diff --git a/services/horizon/internal/expingest/parallel_test.go b/services/horizon/internal/ingest/parallel_test.go similarity index 99% rename from services/horizon/internal/expingest/parallel_test.go rename to services/horizon/internal/ingest/parallel_test.go index 83b0a340ad..7df5e09aed 100644 --- a/services/horizon/internal/expingest/parallel_test.go +++ b/services/horizon/internal/ingest/parallel_test.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "math/rand" diff --git a/services/horizon/internal/expingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go similarity index 97% rename from services/horizon/internal/expingest/processor_runner.go rename to services/horizon/internal/ingest/processor_runner.go index 7f72615431..60f3e10ee8 100644 --- a/services/horizon/internal/expingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -1,15 +1,15 @@ -package expingest +package ingest import ( "bytes" "context" "fmt" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest/processors" + "github.com/stellar/go/services/horizon/internal/ingest/processors" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) diff --git a/services/horizon/internal/expingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go similarity index 98% rename from services/horizon/internal/expingest/processor_runner_test.go rename to services/horizon/internal/ingest/processor_runner_test.go index 4fd2a1665d..2c531dac4e 100644 --- a/services/horizon/internal/expingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "context" @@ -6,12 +6,12 @@ import ( "testing" "github.com/guregu/null" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest/processors" + "github.com/stellar/go/services/horizon/internal/ingest/processors" "github.com/stellar/go/xdr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/services/horizon/internal/expingest/processors/account_data_processor.go b/services/horizon/internal/ingest/processors/account_data_processor.go similarity index 96% rename from services/horizon/internal/expingest/processors/account_data_processor.go rename to services/horizon/internal/ingest/processors/account_data_processor.go index 0872a4d5a1..2258bf59f7 100644 --- a/services/horizon/internal/expingest/processors/account_data_processor.go +++ b/services/horizon/internal/ingest/processors/account_data_processor.go @@ -1,8 +1,8 @@ package processors import ( - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/accounts_data_processor_test.go b/services/horizon/internal/ingest/processors/accounts_data_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/accounts_data_processor_test.go rename to services/horizon/internal/ingest/processors/accounts_data_processor_test.go index b9512dc2bd..1d40b4e801 100644 --- a/services/horizon/internal/expingest/processors/accounts_data_processor_test.go +++ b/services/horizon/internal/ingest/processors/accounts_data_processor_test.go @@ -4,7 +4,7 @@ package processors import ( "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/xdr" "github.com/stretchr/testify/suite" diff --git a/services/horizon/internal/expingest/processors/accounts_processor.go b/services/horizon/internal/ingest/processors/accounts_processor.go similarity index 95% rename from services/horizon/internal/expingest/processors/accounts_processor.go rename to services/horizon/internal/ingest/processors/accounts_processor.go index 8cfc953d77..eed0f10024 100644 --- a/services/horizon/internal/expingest/processors/accounts_processor.go +++ b/services/horizon/internal/ingest/processors/accounts_processor.go @@ -1,8 +1,8 @@ package processors import ( - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/accounts_processor_test.go b/services/horizon/internal/ingest/processors/accounts_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/accounts_processor_test.go rename to services/horizon/internal/ingest/processors/accounts_processor_test.go index 9664870f36..3a17916b6d 100644 --- a/services/horizon/internal/expingest/processors/accounts_processor_test.go +++ b/services/horizon/internal/ingest/processors/accounts_processor_test.go @@ -5,7 +5,7 @@ import ( "context" "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/xdr" "github.com/stretchr/testify/suite" diff --git a/services/horizon/internal/expingest/processors/asset_stats_processor.go b/services/horizon/internal/ingest/processors/asset_stats_processor.go similarity index 98% rename from services/horizon/internal/expingest/processors/asset_stats_processor.go rename to services/horizon/internal/ingest/processors/asset_stats_processor.go index a9383f3d1b..e41532485b 100644 --- a/services/horizon/internal/expingest/processors/asset_stats_processor.go +++ b/services/horizon/internal/ingest/processors/asset_stats_processor.go @@ -4,8 +4,8 @@ import ( "database/sql" "math/big" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/asset_stats_processor_test.go b/services/horizon/internal/ingest/processors/asset_stats_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/asset_stats_processor_test.go rename to services/horizon/internal/ingest/processors/asset_stats_processor_test.go index 9e4f0400a0..1c5d66f840 100644 --- a/services/horizon/internal/expingest/processors/asset_stats_processor_test.go +++ b/services/horizon/internal/ingest/processors/asset_stats_processor_test.go @@ -5,7 +5,7 @@ import ( "database/sql" "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/xdr" "github.com/stretchr/testify/suite" diff --git a/services/horizon/internal/expingest/processors/asset_stats_set.go b/services/horizon/internal/ingest/processors/asset_stats_set.go similarity index 100% rename from services/horizon/internal/expingest/processors/asset_stats_set.go rename to services/horizon/internal/ingest/processors/asset_stats_set.go diff --git a/services/horizon/internal/expingest/processors/asset_stats_set_test.go b/services/horizon/internal/ingest/processors/asset_stats_set_test.go similarity index 100% rename from services/horizon/internal/expingest/processors/asset_stats_set_test.go rename to services/horizon/internal/ingest/processors/asset_stats_set_test.go diff --git a/services/horizon/internal/expingest/processors/claimable_balances_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_processor.go similarity index 96% rename from services/horizon/internal/expingest/processors/claimable_balances_processor.go rename to services/horizon/internal/ingest/processors/claimable_balances_processor.go index 2a464f9b4e..c5a662d60f 100644 --- a/services/horizon/internal/expingest/processors/claimable_balances_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_processor.go @@ -1,8 +1,8 @@ package processors import ( - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/claimable_balances_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/claimable_balances_processor_test.go rename to services/horizon/internal/ingest/processors/claimable_balances_processor_test.go index ea1d211d21..032d38502a 100644 --- a/services/horizon/internal/expingest/processors/claimable_balances_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_processor_test.go @@ -4,7 +4,7 @@ package processors import ( "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/xdr" "github.com/stretchr/testify/suite" diff --git a/services/horizon/internal/expingest/processors/effects_processor.go b/services/horizon/internal/ingest/processors/effects_processor.go similarity index 99% rename from services/horizon/internal/expingest/processors/effects_processor.go rename to services/horizon/internal/ingest/processors/effects_processor.go index d28c15536d..a1c56cecd0 100644 --- a/services/horizon/internal/expingest/processors/effects_processor.go +++ b/services/horizon/internal/ingest/processors/effects_processor.go @@ -8,7 +8,7 @@ import ( "sort" "github.com/stellar/go/amount" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/expingest/processors/effects_processor_test.go b/services/horizon/internal/ingest/processors/effects_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/effects_processor_test.go rename to services/horizon/internal/ingest/processors/effects_processor_test.go index 9f3fa040ce..400f2eced7 100644 --- a/services/horizon/internal/expingest/processors/effects_processor_test.go +++ b/services/horizon/internal/ingest/processors/effects_processor_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" . "github.com/stellar/go/services/horizon/internal/test/transactions" "github.com/stellar/go/services/horizon/internal/toid" diff --git a/services/horizon/internal/expingest/processors/ledgers_processor.go b/services/horizon/internal/ingest/processors/ledgers_processor.go similarity index 97% rename from services/horizon/internal/expingest/processors/ledgers_processor.go rename to services/horizon/internal/ingest/processors/ledgers_processor.go index 7b907d2af7..69c1a05dee 100644 --- a/services/horizon/internal/expingest/processors/ledgers_processor.go +++ b/services/horizon/internal/ingest/processors/ledgers_processor.go @@ -1,7 +1,7 @@ package processors import ( - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/ledgers_processor_test.go b/services/horizon/internal/ingest/processors/ledgers_processor_test.go similarity index 98% rename from services/horizon/internal/expingest/processors/ledgers_processor_test.go rename to services/horizon/internal/ingest/processors/ledgers_processor_test.go index 573611615b..4df06d35cd 100644 --- a/services/horizon/internal/expingest/processors/ledgers_processor_test.go +++ b/services/horizon/internal/ingest/processors/ledgers_processor_test.go @@ -4,7 +4,7 @@ package processors import ( "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/main.go b/services/horizon/internal/ingest/processors/main.go similarity index 85% rename from services/horizon/internal/expingest/processors/main.go rename to services/horizon/internal/ingest/processors/main.go index 97cb804c41..5088dd97aa 100644 --- a/services/horizon/internal/expingest/processors/main.go +++ b/services/horizon/internal/ingest/processors/main.go @@ -6,7 +6,7 @@ import ( "github.com/stellar/go/xdr" ) -var log = logpkg.DefaultLogger.WithField("service", "expingest") +var log = logpkg.DefaultLogger.WithField("service", "ingest") const maxBatchSize = 100000 diff --git a/services/horizon/internal/expingest/processors/offers_processor.go b/services/horizon/internal/ingest/processors/offers_processor.go similarity index 97% rename from services/horizon/internal/expingest/processors/offers_processor.go rename to services/horizon/internal/ingest/processors/offers_processor.go index 51beccd27f..f244090c15 100644 --- a/services/horizon/internal/expingest/processors/offers_processor.go +++ b/services/horizon/internal/ingest/processors/offers_processor.go @@ -1,8 +1,8 @@ package processors import ( - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/offers_processor_test.go b/services/horizon/internal/ingest/processors/offers_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/offers_processor_test.go rename to services/horizon/internal/ingest/processors/offers_processor_test.go index b337f242b9..24156109b4 100644 --- a/services/horizon/internal/expingest/processors/offers_processor_test.go +++ b/services/horizon/internal/ingest/processors/offers_processor_test.go @@ -4,8 +4,8 @@ package processors import ( "testing" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/operations_processor.go b/services/horizon/internal/ingest/processors/operations_processor.go similarity index 99% rename from services/horizon/internal/expingest/processors/operations_processor.go rename to services/horizon/internal/ingest/processors/operations_processor.go index c9b1fa6800..5c42e0cfa1 100644 --- a/services/horizon/internal/expingest/processors/operations_processor.go +++ b/services/horizon/internal/ingest/processors/operations_processor.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/stellar/go/amount" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/expingest/processors/operations_processor_test.go b/services/horizon/internal/ingest/processors/operations_processor_test.go similarity index 98% rename from services/horizon/internal/expingest/processors/operations_processor_test.go rename to services/horizon/internal/ingest/processors/operations_processor_test.go index 8ff263d873..2b2bb06040 100644 --- a/services/horizon/internal/expingest/processors/operations_processor_test.go +++ b/services/horizon/internal/ingest/processors/operations_processor_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/participants_processor.go b/services/horizon/internal/ingest/processors/participants_processor.go similarity index 99% rename from services/horizon/internal/expingest/processors/participants_processor.go rename to services/horizon/internal/ingest/processors/participants_processor.go index fef5a0ca25..5c42871a38 100644 --- a/services/horizon/internal/expingest/processors/participants_processor.go +++ b/services/horizon/internal/ingest/processors/participants_processor.go @@ -2,7 +2,7 @@ package processors import ( - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/expingest/processors/participants_processor_test.go b/services/horizon/internal/ingest/processors/participants_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/participants_processor_test.go rename to services/horizon/internal/ingest/processors/participants_processor_test.go index 39cc5b4932..13fec68c43 100644 --- a/services/horizon/internal/expingest/processors/participants_processor_test.go +++ b/services/horizon/internal/ingest/processors/participants_processor_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/expingest/processors/participants_test.go b/services/horizon/internal/ingest/processors/participants_test.go similarity index 98% rename from services/horizon/internal/expingest/processors/participants_test.go rename to services/horizon/internal/ingest/processors/participants_test.go index 9a012e814e..71383ad9ee 100644 --- a/services/horizon/internal/expingest/processors/participants_test.go +++ b/services/horizon/internal/ingest/processors/participants_test.go @@ -2,7 +2,7 @@ package processors import "testing" -import "github.com/stellar/go/exp/ingest/io" +import "github.com/stellar/go/ingest/io" import "github.com/stretchr/testify/assert" diff --git a/services/horizon/internal/expingest/processors/signer_processor_test.go b/services/horizon/internal/ingest/processors/signer_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/signer_processor_test.go rename to services/horizon/internal/ingest/processors/signer_processor_test.go index 1688182425..6dc7813c75 100644 --- a/services/horizon/internal/expingest/processors/signer_processor_test.go +++ b/services/horizon/internal/ingest/processors/signer_processor_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/guregu/null" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/signers_processor.go b/services/horizon/internal/ingest/processors/signers_processor.go similarity index 97% rename from services/horizon/internal/expingest/processors/signers_processor.go rename to services/horizon/internal/ingest/processors/signers_processor.go index 626f8754c1..aac8281274 100644 --- a/services/horizon/internal/expingest/processors/signers_processor.go +++ b/services/horizon/internal/ingest/processors/signers_processor.go @@ -2,8 +2,8 @@ package processors import ( "github.com/guregu/null" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/trades_processor.go b/services/horizon/internal/ingest/processors/trades_processor.go similarity index 99% rename from services/horizon/internal/expingest/processors/trades_processor.go rename to services/horizon/internal/ingest/processors/trades_processor.go index 63acaf5609..c69038c962 100644 --- a/services/horizon/internal/expingest/processors/trades_processor.go +++ b/services/horizon/internal/ingest/processors/trades_processor.go @@ -3,7 +3,7 @@ package processors import ( "time" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/expingest/processors/trades_processor_test.go b/services/horizon/internal/ingest/processors/trades_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/trades_processor_test.go rename to services/horizon/internal/ingest/processors/trades_processor_test.go index 0c4d220ded..b33751ae7d 100644 --- a/services/horizon/internal/expingest/processors/trades_processor_test.go +++ b/services/horizon/internal/ingest/processors/trades_processor_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/toid" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/transaction_operation_wrapper_test.go b/services/horizon/internal/ingest/processors/transaction_operation_wrapper_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/transaction_operation_wrapper_test.go rename to services/horizon/internal/ingest/processors/transaction_operation_wrapper_test.go index eb5ebc2a6c..97da0557df 100644 --- a/services/horizon/internal/expingest/processors/transaction_operation_wrapper_test.go +++ b/services/horizon/internal/ingest/processors/transaction_operation_wrapper_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" . "github.com/stellar/go/services/horizon/internal/test/transactions" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/transactions_processor.go b/services/horizon/internal/ingest/processors/transactions_processor.go similarity index 96% rename from services/horizon/internal/expingest/processors/transactions_processor.go rename to services/horizon/internal/ingest/processors/transactions_processor.go index f8c2c83088..51c6734225 100644 --- a/services/horizon/internal/expingest/processors/transactions_processor.go +++ b/services/horizon/internal/ingest/processors/transactions_processor.go @@ -1,7 +1,7 @@ package processors import ( - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" ) diff --git a/services/horizon/internal/expingest/processors/transactions_processor_test.go b/services/horizon/internal/ingest/processors/transactions_processor_test.go similarity index 100% rename from services/horizon/internal/expingest/processors/transactions_processor_test.go rename to services/horizon/internal/ingest/processors/transactions_processor_test.go diff --git a/services/horizon/internal/expingest/processors/trust_lines_processor.go b/services/horizon/internal/ingest/processors/trust_lines_processor.go similarity index 96% rename from services/horizon/internal/expingest/processors/trust_lines_processor.go rename to services/horizon/internal/ingest/processors/trust_lines_processor.go index 12feca6a17..23b7e0d322 100644 --- a/services/horizon/internal/expingest/processors/trust_lines_processor.go +++ b/services/horizon/internal/ingest/processors/trust_lines_processor.go @@ -1,8 +1,8 @@ package processors import ( - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/processors/trust_lines_processor_test.go b/services/horizon/internal/ingest/processors/trust_lines_processor_test.go similarity index 99% rename from services/horizon/internal/expingest/processors/trust_lines_processor_test.go rename to services/horizon/internal/ingest/processors/trust_lines_processor_test.go index 968a7bed84..9c9f5ad144 100644 --- a/services/horizon/internal/expingest/processors/trust_lines_processor_test.go +++ b/services/horizon/internal/ingest/processors/trust_lines_processor_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/io" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/resume_state_test.go b/services/horizon/internal/ingest/resume_state_test.go similarity index 98% rename from services/horizon/internal/expingest/resume_state_test.go rename to services/horizon/internal/ingest/resume_state_test.go index fd81a2a3b9..a896f05270 100644 --- a/services/horizon/internal/expingest/resume_state_test.go +++ b/services/horizon/internal/ingest/resume_state_test.go @@ -1,13 +1,13 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "context" "testing" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" - "github.com/stellar/go/exp/ingest/ledgerbackend" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" "github.com/stretchr/testify/mock" diff --git a/services/horizon/internal/expingest/sample_changes_test.go b/services/horizon/internal/ingest/sample_changes_test.go similarity index 98% rename from services/horizon/internal/expingest/sample_changes_test.go rename to services/horizon/internal/ingest/sample_changes_test.go index 021c9a9636..09426b677e 100644 --- a/services/horizon/internal/expingest/sample_changes_test.go +++ b/services/horizon/internal/ingest/sample_changes_test.go @@ -1,6 +1,6 @@ // +build update -package expingest +package ingest import ( "context" @@ -9,9 +9,9 @@ import ( "path/filepath" "testing" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" logpkg "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" ) diff --git a/services/horizon/internal/expingest/stress_test.go b/services/horizon/internal/ingest/stress_test.go similarity index 97% rename from services/horizon/internal/expingest/stress_test.go rename to services/horizon/internal/ingest/stress_test.go index 24862b7b50..5cfd28f799 100644 --- a/services/horizon/internal/expingest/stress_test.go +++ b/services/horizon/internal/ingest/stress_test.go @@ -1,11 +1,11 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "testing" - "github.com/stellar/go/exp/ingest/adapters" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/adapters" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/support/errors" "github.com/stretchr/testify/suite" ) diff --git a/services/horizon/internal/expingest/testdata/sample-changes.xdr b/services/horizon/internal/ingest/testdata/sample-changes.xdr similarity index 100% rename from services/horizon/internal/expingest/testdata/sample-changes.xdr rename to services/horizon/internal/ingest/testdata/sample-changes.xdr diff --git a/services/horizon/internal/expingest/verify.go b/services/horizon/internal/ingest/verify.go similarity index 99% rename from services/horizon/internal/expingest/verify.go rename to services/horizon/internal/ingest/verify.go index dd1c3804e4..56b274f706 100644 --- a/services/horizon/internal/expingest/verify.go +++ b/services/horizon/internal/ingest/verify.go @@ -1,4 +1,4 @@ -package expingest +package ingest import ( "context" @@ -7,12 +7,12 @@ import ( "time" "github.com/guregu/null" - ingesterrors "github.com/stellar/go/exp/ingest/errors" - "github.com/stellar/go/exp/ingest/verify" "github.com/stellar/go/historyarchive" + ingesterrors "github.com/stellar/go/ingest/errors" + "github.com/stellar/go/ingest/verify" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest/processors" + "github.com/stellar/go/services/horizon/internal/ingest/processors" "github.com/stellar/go/support/errors" logpkg "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" diff --git a/services/horizon/internal/expingest/verify_range_state_test.go b/services/horizon/internal/ingest/verify_range_state_test.go similarity index 99% rename from services/horizon/internal/expingest/verify_range_state_test.go rename to services/horizon/internal/ingest/verify_range_state_test.go index c349c32a58..51204d13a5 100644 --- a/services/horizon/internal/expingest/verify_range_state_test.go +++ b/services/horizon/internal/ingest/verify_range_state_test.go @@ -1,5 +1,5 @@ //lint:file-ignore U1001 Ignore all unused code, staticcheck doesn't understand testify/suite -package expingest +package ingest import ( "context" @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/stellar/go/exp/ingest/adapters" - ingestio "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/adapters" + ingestio "github.com/stellar/go/ingest/io" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index fbc9e12001..1c12a72a6d 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -9,7 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/exp/orderbook" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/ledger" "github.com/stellar/go/services/horizon/internal/simplepath" "github.com/stellar/go/services/horizon/internal/txsub" @@ -33,13 +33,13 @@ func mustInitHorizonDB(app *App) { maxIdle := app.config.HorizonDBMaxIdleConnections maxOpen := app.config.HorizonDBMaxOpenConnections if app.config.Ingest { - maxIdle -= expingest.MaxDBConnections - maxOpen -= expingest.MaxDBConnections + maxIdle -= ingest.MaxDBConnections + maxOpen -= ingest.MaxDBConnections if maxIdle <= 0 { - log.Fatalf("max idle connections to horizon db must be greater than %d", expingest.MaxDBConnections) + log.Fatalf("max idle connections to horizon db must be greater than %d", ingest.MaxDBConnections) } if maxOpen <= 0 { - log.Fatalf("max open connections to horizon db must be greater than %d", expingest.MaxDBConnections) + log.Fatalf("max open connections to horizon db must be greater than %d", ingest.MaxDBConnections) } } @@ -52,12 +52,12 @@ func mustInitHorizonDB(app *App) { func initExpIngester(app *App) { var err error - app.expingester, err = expingest.NewSystem(expingest.Config{ + app.ingester, err = ingest.NewSystem(ingest.Config{ CoreSession: mustNewDBSession( - app.config.StellarCoreDatabaseURL, expingest.MaxDBConnections, expingest.MaxDBConnections, + app.config.StellarCoreDatabaseURL, ingest.MaxDBConnections, ingest.MaxDBConnections, ), HistorySession: mustNewDBSession( - app.config.DatabaseURL, expingest.MaxDBConnections, expingest.MaxDBConnections, + app.config.DatabaseURL, ingest.MaxDBConnections, ingest.MaxDBConnections, ), NetworkPassphrase: app.config.NetworkPassphrase, // TODO: @@ -79,7 +79,7 @@ func initExpIngester(app *App) { func initPathFinder(app *App) { orderBookGraph := orderbook.NewOrderBookGraph() - app.orderBookStream = expingest.NewOrderBookStream( + app.orderBookStream = ingest.NewOrderBookStream( &history.Q{app.HorizonSession(app.ctx)}, orderBookGraph, ) @@ -233,14 +233,14 @@ func initProcessMetrics(app *App) { // initIngestMetrics registers the metrics for the ingestion into the provided // app's metrics registry. func initIngestMetrics(app *App) { - if app.expingester == nil { + if app.ingester == nil { return } app.ingestingGauge.Inc() - app.prometheusRegistry.MustRegister(app.expingester.Metrics().LedgerIngestionDuration) - app.prometheusRegistry.MustRegister(app.expingester.Metrics().StateVerifyDuration) - app.prometheusRegistry.MustRegister(app.expingester.Metrics().StateInvalidGauge) + app.prometheusRegistry.MustRegister(app.ingester.Metrics().LedgerIngestionDuration) + app.prometheusRegistry.MustRegister(app.ingester.Metrics().StateVerifyDuration) + app.prometheusRegistry.MustRegister(app.ingester.Metrics().StateInvalidGauge) } func initTxSubMetrics(app *App) { diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go index af7adcc58b..6381be8a10 100644 --- a/services/horizon/internal/middleware_test.go +++ b/services/horizon/internal/middleware_test.go @@ -14,8 +14,8 @@ import ( "github.com/stellar/go/services/horizon/internal/actions" horizonContext "github.com/stellar/go/services/horizon/internal/context" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/services/horizon/internal/expingest" "github.com/stellar/go/services/horizon/internal/httpx" + "github.com/stellar/go/services/horizon/internal/ingest" "github.com/stellar/go/services/horizon/internal/ledger" hProblem "github.com/stellar/go/services/horizon/internal/render/problem" "github.com/stellar/go/services/horizon/internal/test" @@ -183,7 +183,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: true, latestHistoryLedger: 2, lastIngestedLedger: 2, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: http.StatusInternalServerError, expectTransaction: false, @@ -193,7 +193,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: false, latestHistoryLedger: 1, lastIngestedLedger: 0, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: hProblem.StillIngesting.Status, expectTransaction: false, @@ -203,7 +203,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: false, latestHistoryLedger: 3, lastIngestedLedger: 2, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: hProblem.StillIngesting.Status, expectTransaction: false, @@ -213,17 +213,17 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: false, latestHistoryLedger: 4, lastIngestedLedger: 5, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: hProblem.StillIngesting.Status, expectTransaction: false, }, { - name: "responds with still ingesting if version != expingest.CurrentVersion", + name: "responds with still ingesting if version != ingest.CurrentVersion", stateInvalid: false, latestHistoryLedger: 5, lastIngestedLedger: 5, - ingestionVersion: expingest.CurrentVersion - 1, + ingestionVersion: ingest.CurrentVersion - 1, sseRequest: false, expectedStatus: hProblem.StillIngesting.Status, expectTransaction: false, @@ -233,7 +233,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: false, latestHistoryLedger: 6, lastIngestedLedger: 6, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: http.StatusOK, expectTransaction: true, @@ -243,7 +243,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: false, latestHistoryLedger: 7, lastIngestedLedger: 7, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: true, expectedStatus: http.StatusOK, expectTransaction: false, @@ -254,7 +254,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: false, latestHistoryLedger: 8, lastIngestedLedger: 8, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: http.StatusOK, expectTransaction: true, @@ -265,7 +265,7 @@ func TestStateMiddleware(t *testing.T) { stateInvalid: true, latestHistoryLedger: 9, lastIngestedLedger: 9, - ingestionVersion: expingest.CurrentVersion, + ingestionVersion: ingest.CurrentVersion, sseRequest: false, expectedStatus: http.StatusOK, expectTransaction: true, diff --git a/services/horizon/internal/test/transactions/main.go b/services/horizon/internal/test/transactions/main.go index 855de552f7..a0943bfbb9 100644 --- a/services/horizon/internal/test/transactions/main.go +++ b/services/horizon/internal/test/transactions/main.go @@ -5,7 +5,7 @@ import ( "encoding/hex" "testing" - "github.com/stellar/go/exp/ingest/io" + "github.com/stellar/go/ingest/io" "github.com/stellar/go/xdr" "github.com/stretchr/testify/assert" ) diff --git a/tools/archive-reader/archive_reader.go b/tools/archive-reader/archive_reader.go index 2550c6ce98..516bf500f4 100644 --- a/tools/archive-reader/archive_reader.go +++ b/tools/archive-reader/archive_reader.go @@ -7,8 +7,8 @@ import ( stdio "io" "log" - "github.com/stellar/go/exp/ingest/adapters" "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/adapters" ) func main() {