From c75d9939913bd5c3851cfc235ec950ae12ca8041 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Wed, 3 Jul 2024 13:04:18 -0700 Subject: [PATCH] #hubble-469: bring the fake gcs make target back in, review feedback --- .../ledgerexporter/DEVELOPER_GUIDE.md | 2 ++ exp/services/ledgerexporter/Makefile | 26 +++++++++++++++++++ .../ledgerexporter/docker/config.test.toml | 13 ++++++++++ 3 files changed, 41 insertions(+) create mode 100644 exp/services/ledgerexporter/docker/config.test.toml diff --git a/exp/services/ledgerexporter/DEVELOPER_GUIDE.md b/exp/services/ledgerexporter/DEVELOPER_GUIDE.md index 3c06dc4875..4cef8419bb 100644 --- a/exp/services/ledgerexporter/DEVELOPER_GUIDE.md +++ b/exp/services/ledgerexporter/DEVELOPER_GUIDE.md @@ -39,6 +39,8 @@ The Dockerfile contains all the necessary dependencies (e.g., Stellar-core) requ - Run: For instructions on running the Docker container, refer to the [Installation Guide](./README.md). +- Run ledgerexporter with a local, fake GCS backend: Requires `make docker-build` first, then run `make docker-test-fake-gcs`. This will run the ledger exporter against `testnet` and export to the 'fake' GCS instance started in the container. + ## Running Integration Tests: from top directory of stellar/go repo, run go test to launch ledger exporter integration tests. diff --git a/exp/services/ledgerexporter/Makefile b/exp/services/ledgerexporter/Makefile index b8b45aff89..6561c4f24c 100644 --- a/exp/services/ledgerexporter/Makefile +++ b/exp/services/ledgerexporter/Makefile @@ -14,6 +14,32 @@ $(if $(STELLAR_CORE_VERSION), --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VE -t $(DOCKER_IMAGE):$(VERSION) \ -t $(DOCKER_IMAGE):latest . +docker-clean: + $(SUDO) docker stop fake-gcs-server || true + $(SUDO) docker rm fake-gcs-server || true + $(SUDO) rm -rf ${PWD}/storage || true + $(SUDO) docker network rm test-network || true + +docker-test-fake-gcs: docker-clean + # Create temp storage dir + $(SUDO) mkdir -p ${PWD}/storage/exporter-test + + # Create test network for docker + $(SUDO) docker network create test-network + + # Run the fake GCS server + $(SUDO) docker run -d --name fake-gcs-server -p 4443:4443 \ + -v ${PWD}/storage:/data --network test-network fsouza/fake-gcs-server -scheme http + + # Run the ledger-exporter + $(SUDO) docker run --platform linux/amd64 -t --network test-network\ + -v ${PWD}/exp/services/ledgerexporter/docker/config.test.toml:/config.toml \ + -e STORAGE_EMULATOR_HOST=http://fake-gcs-server:4443 \ + $(DOCKER_IMAGE):$(VERSION) \ + scan-and-fill --start 1000 --end 2000 + + $(MAKE) docker-clean + docker-push: $(SUDO) docker push $(DOCKER_IMAGE):$(VERSION) $(SUDO) docker push $(DOCKER_IMAGE):latest diff --git a/exp/services/ledgerexporter/docker/config.test.toml b/exp/services/ledgerexporter/docker/config.test.toml new file mode 100644 index 0000000000..c5c4519f0b --- /dev/null +++ b/exp/services/ledgerexporter/docker/config.test.toml @@ -0,0 +1,13 @@ +[datastore_config] +type = "GCS" + +[datastore_config.params] +destination_bucket_path = "exporter-test/ledgers/testnet" + +[datastore_config.schema] +ledgers_per_file = 1 +files_per_partition = 64000 + +[stellar_core_config] + network = "testnet" +