Revert ledgerexport relocation

.github/workflows/galexie-release.yml → .github/workflows/ledgerexporter-release.yml

@@ -1,34 +1,34 @@
-name: Galexie Release 
+name: Ledger Exporter release
 
 on:
   push:
-    tags: ['galexie-v*']
+    tags: ['ledgerexporter-v*']
 
 jobs:
 
   publish-docker:
-    name: Test and push docker image
+    name: Test and push the Ledger Exporter images
     runs-on: ubuntu-latest
     env:
-      GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
-      GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as STELLAR_CORE_VERSION
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
-      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
       STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
     steps:
       - name: Set VERSION
         run: |
-          echo "VERSION=${GITHUB_REF_NAME#galexie-v}" >> $GITHUB_ENV
+          echo "VERSION=${GITHUB_REF_NAME#ledgerexporter-v}" >> $GITHUB_ENV
 
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.sha }}
       - name: Pull Quickstart image
         shell: bash
         run: |
-          docker pull "$GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
       - name: Install captive core
         run: |
           # Workaround for https://github.com/actions/virtual-environments/issues/5245,
@@ -42,11 +42,11 @@ jobs:
           sudo apt-get update && sudo apt-get install -y stellar-core="$STELLAR_CORE_VERSION"
           echo "Using stellar core version $(stellar-core version)"
 
-      - name: Run tests 
-        run: go test -v -race -run TestGalexieTestSuite ./exp/services/galexie/...
+      - name: Run Ledger Exporter test
+        run: go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/...
 
-      - name: Build docker
-        run: make -C exp/services/galexie docker-build
+      - name: Build Ledger Exporter docker
+        run: make -C exp/services/ledgerexporter docker-build
 
       # Push images
       - name: Login to DockerHub
@@ -56,4 +56,4 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Push to DockerHub
-        run: make -C exp/services/galexie docker-push
+        run: make -C exp/services/ledgerexporter docker-push

.github/workflows/galexie.yml → .github/workflows/ledgerexporter.yml

@@ -1,22 +1,23 @@
-name: Galexie
+name: LedgerExporter
 
 on:
   push:
     branches: [master]
   pull_request:
 
 jobs:
-  galexie:
-    name: Test
+  ledger-exporter:
+    name: Test Ledger Exporter
     runs-on: ubuntu-latest
     env:
       CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
-      GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
-      GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
-      # this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
+      LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED: "true"
+      LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
+      # this pins to a version of quickstart:testing that has the same version of core
+      # as specified on LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
-      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5c8186f53cc98571749054dd782dce33b0aca2d1a622a7610362f7c15b79b1bf
-      GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:5c8186f53cc98571749054dd782dce33b0aca2d1a622a7610362f7c15b79b1bf
+      LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
     steps:
       - name: Install captive core
         run: |
@@ -34,12 +35,12 @@ jobs:
       - name: Pull Quickstart image
         shell: bash
         run: |
-          docker pull "$GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE"
+          docker pull "$LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE"
 
       - uses: actions/checkout@v3
         with:
           # For pull requests, build and test the PR head not a merge of the PR with the destination.
           ref:  ${{ github.event.pull_request.head.sha || github.ref }}
 
-      - name: Run test
-        run: go test -v -race -run TestGalexieTestSuite ./services/galexie/... 
+      - name: Run Ledger Exporter test
+        run: go test -v -race -run TestLedgerExporterTestSuite ./services/ledgerexporter/... 

Makefile

@@ -34,8 +34,8 @@ friendbot:
 horizon:
 	$(MAKE) -C services/horizon/ binary-build
 
-galexie:
-	$(MAKE) -C services/galexie/ docker-build
+ledger-exporter:
+	$(MAKE) -C services/ledgerexporter/ docker-build
 
 webauth:
 	$(MAKE) -C exp/services/webauth/ docker-build

services/ledgerexporter/DEVELOPER_GUIDE.md

@@ -0,0 +1,82 @@
+
+# Ledger Exporter Developer Guide
+The ledger exporter is a tool to export Stellar network transaction data to cloud storage in a way that is easy to access.
+
+## Prerequisites
+This document assumes that you have installed and can run the ledger exporter, and that you have familiarity with its CLI and configuration. If not, please refer to the [Installation Guide](./README.md).
+
+## Goal
+The goal of the ledger exporter is to build an easy-to-use tool to export Stellar network ledger data to a configurable remote data store, such as cloud blob storage.
+ - Use cloud storage optimally
+ - Minimize network usage to export
+ - Make it easy and fast to search for a specific ledger or ledger range
+
+## Architecture
+To achieve its goals, the ledger exporter uses the following architecture, which consists of the 3 main components:
+- Captive-core to extract raw transaction metadata from the Stellar Network.
+- Export manager to bundles and organizes the ledgers to get them ready for export.
+- The cloud storage plugin writes to the cloud storage. This is specific to the type of cloud storage, GCS in this case.
+
+
+![ledgerexporter-architecture](./architecture.png)
+
+
+## Data Format
+- Ledger exporter uses a compact and efficient data format called [XDR](https://developers.stellar.org/docs/learn/encyclopedia/data-format/xdr) (External Data Representation), which is a compact binary format. A Stellar Captive Core instance emits data in this format and the data structure is referred to as `LedgerCloseMeta`. The exporter bundles multiple `LedgerCloseMeta`'s into a single object using a custom XDR structure called `LedgerCloseMetaBatch` which is defined in [Stellar-exporter.x](https://github.com/stellar/go/blob/master/xdr/Stellar-exporter.x).
+
+- The metadata for the same batch is also stored alongside each exported object. Supported metadata is defined in [metadata.go](https://github.com/stellar/go/blob/master/support/datastore/metadata.go). 
+
+- Objects are compressed before uploading using the [zstd](http://facebook.github.io/zstd/) (zstandard) compression algorithm to optimize network usage and storage needs.
+
+## Data Storage
+- An example implementation of `DataStore` for GCS, Google Cloud Storage. This plugin is located in the [support](https://github.com/stellar/go/tree/master/support/datastore) package. 
+- The ledger exporter currently implements the interface only for Google Cloud Storage (GCS). The [GCS plugin](https://github.com/stellar/go/blob/master/support/datastore/gcs_datastore.go) uses GCS-specific behaviors like conditional puts, automatic retry, metadata, and CRC checksum.
+
+## Build and Run using Docker
+The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) required to run the ledger exporter. 
+
+- Build: To build the Docker container, use the provided [Makefile](./Makefile). Simply run make `make docker-build` to build a new container after making any changes.
+
+- Run: For instructions on running the Docker container, refer to the [Installation Guide](./README.md).
+
+- Run ledgerexporter with a local, fake GCS backend: Requires `make docker-build` first, then run `make docker-test-fake-gcs`. This will run the ledger exporter against `testnet` and export to the 'fake' GCS instance started in the container.
+
+## Running Integration Tests:
+from top directory of stellar/go repo, run go test to launch ledger exporter integration
+tests.
+
+`LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true` is required environment variable to allow
+tests to run.
+
+Optional, tests will try to run `stellar-core` from o/s PATH for captive core, if not resolvable, then set `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN=/path/to/stellar-core` 
+
+Optional, can override the version of quickstart used to run standalone stellar network, `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE=docker.io/stellar/quickstart:<tag>`. By default it will try to docker pull `stellar/quickstart:testing` image to local host's docker image store. Set `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL=false` to skip the pull, if you know host has up to date image.
+
+Note, the version of stellar core in `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE` and `LEDGEREXPORTER_INTEGRATION_TESTS_CAPTIVE_CORE_BIN` needs to be on the same major rev or the captive core process may not be able to join or parse ledger meta from the `local` network created by `LEDGEREXPORTER_INTEGRATION_TESTS_QUICKSTART_IMAGE`
+
+```
+$ LEDGEREXPORTER_INTEGRATION_TESTS_ENABLED=true go test -v -race -run TestLedgerExporterTestSuite ./exp/services/ledgerexporter/...
+```
+
+## Adding support for a new storage type
+Support for different data storage types are encapsulated as 'plugins', which are implementation of `DataStore` interface in a go package. To add a data storage plugin based on a new storage type (e.g. AWS S3), follow these steps:
+
+- A data storage plugin must implement the [DataStore](https://github.com/stellar/go/blob/master/support/datastore/datastore.go) interface.
+- Add support for new datastore-specific features. Implement any datastore-specific custom logic. Different datastores have different ways of handling 
+  - race conditions
+  - automatic retries
+  - metadata storage, etc.
+- Add the new datastore to the factory function [NewDataStore](https://github.com/stellar/go/blob/master/support/datastore/datastore.go).
+- Add a [config](./config.example.toml) section for the new storage type. This may include configurations like destination, authentication information etc.
+- An emulator such as a GCS emulator [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) can be used for testing without connecting to real cloud storage.
+
+### Design DOs and DONTs
+- Multiple exporters should be able to run in parallel without the need for explicit locking or synchronization.
+- Exporters when restarted do not have any memory of prior operation and rely on the already exported data as much as possible to decide where to resume.
+
+## Using exported data
+The exported data in storage can be used in the ETL pipeline to gather analytics and reporting. To write a tool that consumes exported data you can use Stellar ingestion library's `ledgerbackend` package. This package includes a ledger backend called [BufferedStorageBackend](https://github.com/stellar/go/blob/master/ingest/ledgerbackend/buffered_storage_backend.go),
+which imports data from the storage and validates it. For more details, refer to the ledgerbackend [documentation](https://github.com/stellar/go/tree/master/ingest/ledgerbackend).
+
+## Contributing
+For information on how to contribute, please refer to our [Contribution Guidelines](https://github.com/stellar/go/blob/master/CONTRIBUTING.md).

services/ledgerexporter/Makefile

@@ -0,0 +1,45 @@
+SUDO := $(shell docker version >/dev/null 2>&1 || echo "sudo")
+
+# https://github.com/opencontainers/image-spec/blob/master/annotations.md
+BUILD_DATE := $(shell date -u +%FT%TZ)
+VERSION ?= $(shell git rev-parse --short HEAD)
+DOCKER_IMAGE := stellar/ledger-exporter
+
+docker-build:
+	cd ../../ && \
+	$(SUDO) docker build --platform linux/amd64 --pull --label org.opencontainers.image.created="$(BUILD_DATE)" \
+    --build-arg GOFLAGS="-ldflags=-X=github.com/stellar/go/services/ledgerexporter/internal.version=$(VERSION)" \
+$(if $(STELLAR_CORE_VERSION), --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION)) \
+	-f services/ledgerexporter/docker/Dockerfile \
+	-t $(DOCKER_IMAGE):$(VERSION) \
+	-t $(DOCKER_IMAGE):latest .
+
+docker-clean:
+	$(SUDO) docker stop fake-gcs-server || true
+	$(SUDO) docker rm fake-gcs-server || true
+	$(SUDO) rm -rf ${PWD}/storage || true
+	$(SUDO) docker network rm test-network || true
+
+docker-test-fake-gcs: docker-clean
+	# Create temp storage dir
+	$(SUDO) mkdir -p ${PWD}/storage/exporter-test 
+
+	# Create test network for docker
+	$(SUDO) docker network create test-network
+
+	# Run the fake GCS server
+	$(SUDO) docker run -d --name fake-gcs-server -p 4443:4443 \
+		 -v ${PWD}/storage:/data --network test-network fsouza/fake-gcs-server -scheme http
+
+	# Run the ledger-exporter
+	$(SUDO) docker run --platform linux/amd64 -t --network test-network\
+		 -v ${PWD}/services/ledgerexporter/docker/config.test.toml:/config.toml \
+		 -e STORAGE_EMULATOR_HOST=http://fake-gcs-server:4443 \
+		 $(DOCKER_IMAGE):$(VERSION) \
+		 scan-and-fill --start 1000 --end 2000
+
+	$(MAKE) docker-clean
+
+docker-push:
+	$(SUDO) docker push $(DOCKER_IMAGE):$(VERSION)
+	$(SUDO) docker push $(DOCKER_IMAGE):latest