exp/services/ledgerexporter: create go integration tests for sub commands

.github/workflows/horizon.yml

@@ -148,4 +148,3 @@ jobs:
       - if: github.ref == 'refs/heads/master'
         name: Push to DockerHub
         run: docker push stellar/horizon-verify-range:latest
-

.github/workflows/ledgerexporter-release.yml

@@ -10,17 +10,40 @@ jobs:
     name: Test and push the Ledger Exporter images
     runs-on: ubuntu-latest
     env:
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      # this pins to a version of quickstart:testing that has the same version as STELLAR_CORE_VERSION
+      # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
       STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
-      VERSION: ${GITHUB_REF_NAME#ledgerexporter-v}
+      VERSION: ${GITHUB_REF_NAME#ledgerexporter-v} 
     steps:
       - uses: actions/checkout@v3
         with:
           ref: github.sha
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
+      - name: Pull Quickstart image
+        shell: bash
+        run: |
+          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"    
+      - name: Install captive core
+        run: |
+          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+          # message about why the installation fails)
+          sudo apt list --installed | grep libc++
+          sudo apt-get remove -y libc++1-* libc++abi1-* || true
+
+          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+          sudo apt-get update && sudo apt-get install -y stellar-core="$STELLAR_CORE_VERSION"
+          echo "Using stellar core version $(stellar-core version)"
 
       - name: Run Ledger Exporter test
-        run: make -C exp/services/ledgerexporter docker-test
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 
+
+      - name: Build Ledger Exporter docker
+        run: make -C exp/services/ledgerexporter docker-build
 
       # Push images
       - name: Login to DockerHub

.github/workflows/ledgerexporter.yml

@@ -7,18 +7,39 @@ on:
 
 jobs:
   ledger-exporter:
-    name: Build and test Ledger Exporter image
+    name: Test Ledger Exporter
     runs-on: ubuntu-latest
     env:
-      STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      # this pins to a version of quickstart:testing that has the same version as LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
+      # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
     steps:
+      - name: Install captive core
+        run: |
+          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+          # message about why the installation fails)
+          sudo apt list --installed | grep libc++
+          sudo apt-get remove -y libc++1-* libc++abi1-* || true
+
+          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+          sudo apt-get update && sudo apt-get install -y stellar-core="$CAPTIVE_CORE_DEBIAN_PKG_VERSION"
+          echo "Using stellar core version $(stellar-core version)"
+
+      - name: Pull Quickstart image
+        shell: bash
+        run: |
+          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+
       - uses: actions/checkout@v3
         with:
           # For pull requests, build and test the PR head not a merge of the PR with the destination.
           ref:  ${{ github.event.pull_request.head.sha || github.ref }}
-      - name: Build Ledger Exporter docker
-        run: make -C exp/services/ledgerexporter docker-build
-
+
       - name: Run Ledger Exporter test
-        run: make -C exp/services/ledgerexporter docker-test
-
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/... 

exp/services/ledgerexporter/DEVELOPER_GUIDE.md

@@ -32,14 +32,31 @@ To achieve its goals, the ledger exporter uses the following architecture, which
 - An example implementation of `DataStore` for GCS, Google Cloud Storage. This plugin is located in the [support](https://github.com/stellar/go/tree/master/support/datastore) package. 
 - The ledger exporter currently implements the interface only for Google Cloud Storage (GCS). The [GCS plugin](https://github.com/stellar/go/blob/master/support/datastore/gcs_datastore.go) uses GCS-specific behaviors like conditional puts, automatic retry, metadata, and CRC checksum.
 
-## Build, Run and Test using Docker
+## Build and Run using Docker
 The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) required to run the ledger exporter. 
 
 - Build: To build the Docker container, use the provided [Makefile](./Makefile). Simply run make `make docker-build` to build a new container after making any changes.
 
 - Run: For instructions on running the Docker container, refer to the [Installation Guide](./README.md).
 
-- Test: To test the Docker container, refer to the [docker-test](./Makefile) command for an example of how to use the [GCS emulator](https://github.com/fsouza/fake-gcs-server) for local testing.
+- Run ledgerexporter with a local, fake GCS backend: Requires `make docker-build` first, then run `make docker-test-fake-gcs`. This will run the ledger exporter against `testnet` and export to the 'fake' GCS instance started in the container.
+
+## Running Integration Tests:
+from top directory of stellar/go repo, run go test to launch ledger exporter integration
+tests.
+
+`LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true` is required environment variable to allow
+tests to run.
+
+Optional, tests will try to run `stellar-core` from o/s PATH for captive core, if not resolvable, then set `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN=/path/to/stellar-core` 
+
+Optional, can override the version of quickstart used to run standalone stellar network, `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE=docker.io/stellar/quickstart:<tag>`. By default it will try to docker pull `stellar/quickstart:testing` image to local host's docker image store. Set `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL=false` to skip the pull, if you know host has up to date image.
+
+Note, the version of stellar core in `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE` and `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN` needs to be on the same major rev or the captive core process may not be able to join or parse ledger meta from the `local` network created by `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE`
+
+```
+$ LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/...
+```
 
 ## Adding support for a new storage type
 Support for different data storage types are encapsulated as 'plugins', which are implementation of `DataStore` interface in a go package. To add a data storage plugin based on a new storage type (e.g. AWS S3), follow these steps:

exp/services/ledgerexporter/Makefile

@@ -20,7 +20,7 @@ docker-clean:
 	$(SUDO) rm -rf ${PWD}/storage || true
 	$(SUDO) docker network rm test-network || true
 
-docker-test: docker-clean
+docker-test-fake-gcs: docker-clean
 	# Create temp storage dir
 	$(SUDO) mkdir -p ${PWD}/storage/exporter-test 
 

exp/services/ledgerexporter/README.md

@@ -126,5 +126,4 @@ docker run --platform linux/amd64 -d \
 Arguments:
 - `--start <start_ledger>` (required): The starting ledger sequence number in the range to export.
 - `--end <end_ledger>` (required): The ending ledger sequence number in the range.
-- `--config-file <config_file_path>` (optional): The path to your configuration file, containing details like GCS bucket information. If not provided, the exporter will look for config.toml in the directory where you run the command.
-
+- `--config-file <config_file_path>` (optional): The path to your configuration file, containing details like GCS bucket information. If not provided, the exporter will look for config.toml in the directory where you run the command.

exp/services/ledgerexporter/internal/app.go

@@ -186,7 +186,7 @@ func newAdminServer(adminPort int, prometheusRegistry *prometheus.Registry) *htt
 }
 
 func (a *App) Run(runtimeSettings RuntimeSettings) error {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(runtimeSettings.Ctx)
 	defer cancel()
 
 	if err := a.init(ctx, runtimeSettings); err != nil {

exp/services/ledgerexporter/internal/config.go

@@ -48,6 +48,7 @@ type RuntimeSettings struct {
 	EndLedger      uint32
 	ConfigFilePath string
 	Mode           Mode
+	Ctx            context.Context
 }
 
 type StellarCoreConfig struct {
@@ -56,6 +57,8 @@ type StellarCoreConfig struct {
 	HistoryArchiveUrls    []string `toml:"history_archive_urls"`
 	StellarCoreBinaryPath string   `toml:"stellar_core_binary_path"`
 	CaptiveCoreTomlPath   string   `toml:"captive_core_toml_path"`
+	CheckpointFrequency   uint32   `toml:"checkpoint_frequency"`
+	StoragePath           string   `toml:"storage_path"`
 }
 
 type Config struct {
@@ -98,8 +101,8 @@ func NewConfig(settings RuntimeSettings, getCoreVersionFn ledgerbackend.CoreBuil
 	}
 	logger.Infof("Network Config Archive URLs: %v", config.StellarCoreConfig.HistoryArchiveUrls)
 	logger.Infof("Network Config Archive Passphrase: %v", config.StellarCoreConfig.NetworkPassphrase)
-	logger.Infof("Network Config Archive Stellar Core Binary Path: %v", config.StellarCoreConfig.StellarCoreBinaryPath)
-	logger.Infof("Network Config Archive Stellar Core Toml Config: %v", string(config.SerializedCaptiveCoreToml))
+	logger.Infof("Network Config Stellar Core Binary Path: %v", config.StellarCoreConfig.StellarCoreBinaryPath)
+	logger.Infof("Network Config Stellar Core Toml Config: %v", string(config.SerializedCaptiveCoreToml))
 
 	return config, nil
 }
@@ -185,15 +188,20 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb
 		return ledgerbackend.CaptiveCoreConfig{}, errors.Wrap(err, "Failed to create captive-core toml")
 	}
 
+	checkpointFrequency := historyarchive.DefaultCheckpointFrequency
+	if config.StellarCoreConfig.CheckpointFrequency > 0 {
+		checkpointFrequency = config.StellarCoreConfig.CheckpointFrequency
+	}
 	return ledgerbackend.CaptiveCoreConfig{
 		BinaryPath:          config.StellarCoreConfig.StellarCoreBinaryPath,
 		NetworkPassphrase:   params.NetworkPassphrase,
 		HistoryArchiveURLs:  params.HistoryArchiveURLs,
-		CheckpointFrequency: historyarchive.DefaultCheckpointFrequency,
+		CheckpointFrequency: checkpointFrequency,
 		Log:                 logger.WithField("subservice", "stellar-core"),
 		Toml:                captiveCoreToml,
 		UserAgent:           "ledger-exporter",
 		UseDB:               true,
+		StoragePath:         config.StellarCoreConfig.StoragePath,
 	}, nil
 }