#4911: updated changelog/readme

.github/workflows/ledgerexporter.yml

@@ -13,9 +13,10 @@ jobs:
       CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
       LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
       LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
-      # this pins to a version of quickstart:testing that has the same version as LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
+      # this pins to a version of quickstart:testing that has the same version of core
+      # as specified on LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5c8186f53cc98571749054dd782dce33b0aca2d1a622a7610362f7c15b79b1bf
       LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
     steps:
       - name: Install captive core

services/horizon/CHANGELOG.md

@@ -3,6 +3,17 @@
 All notable changes to this project will be documented in this
 file. This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Pending
+
+### Added
+
+- Reingest from pre-computed tx meta on remote cloud storage. ([4911](https://github.com/stellar/go/issues/4911)), ([5374](https://github.com/stellar/go/pull/5374))
+  - Configure horizon reingestion to obtain ledger tx meta in pre-computed files from a Google Cloud Storage(GCS) location. 
+  - Using this option will no longer require a captive core binary be present and it no longer runs a captive core sub-process, instead obtaining the tx meta from the GCS backend.
+  - Horizon supports this new feature with two new parameters `ledgerbackend` and `datastore-config` on the `reingest` command. Refer to [Reingestion README](./internal/ingest/README.md#reingestion).
+
+
+
 ## 2.31.0
 
 ### Breaking Changes

services/horizon/cmd/db_test.go

@@ -52,7 +52,7 @@ func (s *DBCommandsTestSuite) TestDefaultParallelJobSizeForBufferedBackend() {
 		"--network", "testnet",
 		"--parallel-workers", "2",
 		"--ledgerbackend", "datastore",
-		"--datastore-config", "../config.storagebackend.toml",
+		"--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml",
 		"2",
 		"10"})
 
@@ -99,7 +99,7 @@ func (s *DBCommandsTestSuite) TestUsesParallelJobSizeWhenSetForBuffered() {
 		"--parallel-workers", "2",
 		"--parallel-job-size", "5",
 		"--ledgerbackend", "datastore",
-		"--datastore-config", "../config.storagebackend.toml",
+		"--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml",
 		"2",
 		"10"})
 
@@ -154,7 +154,7 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() {
 			args: []string{
 				"1", "100",
 				"--ledgerbackend", "datastore",
-				"--datastore-config", "../config.storagebackend.toml",
+				"--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml",
 				"--network-passphrase", "passphrase",
 				"--history-archive-urls", "[]",
 			},
@@ -165,7 +165,7 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() {
 			args: []string{
 				"1", "100",
 				"--ledgerbackend", "datastore",
-				"--datastore-config", "../config.storagebackend.toml",
+				"--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml",
 			},
 			expectError:  true,
 			errorMessage: "network-passphrase must be set",
@@ -175,7 +175,7 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() {
 			args: []string{
 				"1", "100",
 				"--ledgerbackend", "datastore",
-				"--datastore-config", "../config.storagebackend.toml",
+				"--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml",
 				"--network-passphrase", "passphrase",
 			},
 			expectError:  true,
@@ -217,7 +217,7 @@ func (s *DBCommandsTestSuite) TestDbReingestAndFillGapsCmds() {
 				"1", "100",
 				"--network", "testnet",
 				"--ledgerbackend", "datastore",
-				"--datastore-config", "../config.storagebackend.toml",
+				"--datastore-config", "../internal/ingest/testdata/config.storagebackend.toml",
 			},
 			expectError: false,
 		},

services/horizon/internal/ingest/README.md

@@ -140,8 +140,45 @@ This pauses the state machine for 10 seconds then tries again, in hopes that a n
 **Next state**: [`start`](#start-state)
 
 
-# Ingestion
-TODO
+# Reingestion
+Horizon supports running reingestion by executing a sub command `db reingest range <from_ledger> <to_ledger>` which will execute as an o/s process and will be synchronous, exiting the process only after the complete reingestion range is finished or an error is encountered.
+
+By default this sub-command will attempt to use captive core configuration in the form of stellar core binary(`--stellar-core-binary-path`) and stellar core config(`--captive-core-config-path`) to obtain ledger tx meta from a stellar network to be ingested. 
+
+The `db reingest range` sub-command can optionally be configured to consume pre-computed ledger tx meta files from a Google Cloud Storage(GCS) location instead of running captive core on host machine. 
+Pre-requirements: 
+  - Have a GCS account.
+  - Run the [ledgerexporter] to publish ledger tx meta files to your GCS bucket location.
+Run the `db reingest` sub-command, configured to import tx meta from your GCS bucket:
+  ```$ DATABASE_URL=<your db host dsn> \
+     NETWORK=testnet \
+     stellar-horizon db reingest range \
+		   --parallel-workers, 2 \
+		   --ledgerbackend, "datastore" \
+		   --datastore-config, "config.storagebackend.toml",
+  ```  
+Notice, even though we no longer need to provide stellar-core related config for binary or config file, we do still need to provide network related config, using convenience parameter `NETWORK=testnet|pubnet` or directly with `NETWORK_PASSPHRASE` and `HISTORY_ARCHIVE_URLS`
+
+The `--datastore-config` must point to a new toml config file that will provide the necessary parameters for ingestion to work with remote GCS storage.
+
+example config toml:
+```
+# Datastore Configuration
+[datastore_config]
+# Specifies the type of datastore. 
+# Currently, only Google Cloud Storage (GCS) is supported.
+type = "GCS"
+
+[datastore_config.params]
+# The Google Cloud Storage bucket path for storing data, with optional subpaths for organization.
+destination_bucket_path = "path/to/my/bucket"
+
+[datastore_config.schema]
+# Configuration for data organization of the remote files
+ledgers_per_file = 1          # Number of ledgers stored in each file.
+files_per_partition = 64000   # Number of files per partition/directory.
+
+```
 
 # Range Preparation
 TODO: See `maybePrepareRange`

services/horizon/internal/integration/db_test.go

@@ -614,7 +614,7 @@ func TestReingestDatastore(t *testing.T) {
 		"--network", "testnet",
 		"--parallel-workers", "1",
 		"--ledgerbackend", "datastore",
-		"--datastore-config", "../../config.storagebackend.toml",
+		"--datastore-config", "../ingest/testdata/config.storagebackend.toml",
 		"997",
 		"999"})