services/horizon/docker/ledgerexporter: deploy ledgerexporter image as service

exp/services/ledgerexporter/main.go

@@ -29,6 +29,7 @@ func main() {
 	continueFromLatestLedger := flag.Bool("continue", false, "start export from the last exported ledger (as indicated in the target's /latest path)")
 	endingLedger := flag.Uint("end-ledger", 0, "ledger at which to stop the export (must be a closed ledger), 0 means no ending")
 	writeLatestPath := flag.Bool("write-latest-path", true, "update the value of the /latest path on the target")
+	captiveCoreUseDb := flag.Bool("captive-core-use-db", true, "configure captive core to store database on disk in working directory rather than in memory")
 	flag.Parse()
 
 	logger.SetLevel(supportlog.InfoLevel)
@@ -51,6 +52,7 @@ func main() {
 		CheckpointFrequency: 64,
 		Log:                 logger.WithField("subservice", "stellar-core"),
 		Toml:                captiveCoreToml,
+		UseDB:               *captiveCoreUseDb,
 	}
 	core, err := ledgerbackend.NewCaptive(captiveConfig)
 	logFatalIf(err, "Could not create captive core instance")
@@ -91,7 +93,7 @@ func main() {
 	err = core.PrepareRange(context.Background(), ledgerRange)
 	logFatalIf(err, "could not prepare range")
 
-	for nextLedger := startLedger; nextLedger <= endLedger; {
+	for nextLedger := startLedger; endLedger < 1 || nextLedger <= endLedger; {
 		ledger, err := core.GetLedger(context.Background(), nextLedger)
 		if err != nil {
 			logger.WithError(err).Warnf("could not fetch ledger %v, retrying", nextLedger)

historyarchive/s3_archive.go

@@ -121,7 +121,6 @@ func (b *S3ArchiveBackend) PutFile(pth string, in io.ReadCloser) error {
 	params := &s3.PutObjectInput{
 		Bucket: aws.String(b.bucket),
 		Key:    aws.String(key),
-		ACL:    aws.String(s3.ObjectCannedACLPublicRead),
 		Body:   bytes.NewReader(buf.Bytes()),
 	}
 	req, _ := b.svc.PutObjectRequest(params)

services/horizon/docker/ledgerexporter/captive-core-testnet.cfg

@@ -0,0 +1,30 @@
+PEER_PORT=11725
+DATABASE = "sqlite3:///cc/stellar.db"
+
+UNSAFE_QUORUM=true
+FAILURE_SAFETY=1
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="HIGH"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="core-testnet1.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_2"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP"
+ADDRESS="core-testnet2.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_3"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z"
+ADDRESS="core-testnet3.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}"

services/horizon/docker/ledgerexporter/ledgerexporter.yml

@@ -0,0 +1,92 @@
+# this file contains the ledgerexporter deployment and it's config artifacts.
+# when importing the manifest with kubectl, will only create, skips any that already exist.
+#
+# make sure to include namespace destination, the manifest does not specify, 
+# otherwise it'll go in your current kubectl context.
+#
+# if defining the secrets for first time, substitue <base64 encoded value here> placeholders.
+#
+# $ kubectl create -f ledgerexporter.yml -n horizon-dev
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  annotations:
+    fluxcd.io/ignore: "true"
+  labels:
+    app: ledgerexporter
+  name: ledgerexporter-pubnet-env
+data:
+  START: "2"
+  END: "0"
+  # can only have CONTINUE or START set, not both.
+  #CONTINUE: "true"
+  WRITE_LATEST_PATH: "true"
+  CAPTIVE_CORE_USE_DB: "true"
+  HISTORY_ARCHIVE_URLS: "https://history.stellar.org/prd/core-live/core_live_001,https://history.stellar.org/prd/core-live/core_live_002,https://history.stellar.org/prd/core-live/core_live_003"
+  NETWORK_PASSPHRASE: "Public Global Stellar Network ; September 2015"
+  ARCHIVE_TARGET: "s3://horizon-ledgermeta-pubnet"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app: ledgerexporter
+  name: ledgerexporter-pubnet-secret
+type: Opaque
+data:
+  AWS_REGION: <base64 encoded value here>
+  AWS_ACCESS_KEY_ID: <base64 encoded value here>
+  AWS_SECRET_ACCESS_KEY: <base64 encoded value here>  
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    fluxcd.io/ignore: "true"
+    deployment.kubernetes.io/revision: "3"
+  labels:
+    app: ledgerexporter
+  name: ledgerexporter-deployment
+spec:
+  selector:
+    matchLabels:
+      app: ledgerexporter
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        fluxcd.io/ignore: "true"
+        # if we expect to add metrics at some point to ledgerexporter
+        # this just needs to be set to true
+        prometheus.io/port: "6060"
+        prometheus.io/scrape: "false"
+      labels:
+        app: ledgerexporter
+    spec:
+      containers:
+      - envFrom:
+        - secretRef:
+            name: ledgerexporter-pubnet-secret
+        - configMapRef:
+            name: ledgerexporter-pubnet-env
+        image: stellar/horizon-ledgerexporter:latest
+        imagePullPolicy: Always
+        name: ledgerexporter
+        resources:  
+          limits:
+            cpu:     1
+            memory:  4Gi
+          requests:
+            cpu:      250m
+            memory:   500m
+        volumeMounts:
+        - mountPath: /cc
+          name: tempfs-volume
+      dnsPolicy: ClusterFirst
+      volumes:
+      - name: tempfs-volume
+        emptyDir: 
+          medium: Memory
+
+
+

services/horizon/docker/ledgerexporter/start

@@ -1,17 +1,25 @@
 #! /usr/bin/env bash
 set -e
 
-START="${START:=0}"
+START="${START:=2}"
 END="${END:=0}"
 CONTINUE="${CONTINUE:=false}"
 # Writing to /latest is disabled by default to avoid race conditions between parallel container runs
 WRITE_LATEST_PATH="${WRITE_LATEST_PATH:=false}"
+NETWORK_PASSPHRASE="${NETWORK_PASSPHRASE:=Public Global Stellar Network ; September 2015}"
+HISTORY_ARCHIVE_URLS="${HISTORY_ARCHIVE_URLS:=https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001}"
+CAPTIVE_CORE_CONFIG="${CAPTIVE_CORE_CONFIG:=/captive-core-pubnet.cfg}"
+CAPTIVE_CORE_USE_DB="${CAPTIVE_CORE_USE_DB:=true}"
 
 if [ -z "$ARCHIVE_TARGET" ]; then
   echo "error: undefined ARCHIVE_TARGET env variable"
   exit 1
 fi
 
+if [ "$NETWORK_PASSPHRASE" = "Test SDF Network ; September 2015" ]; then
+  CAPTIVE_CORE_CONFIG="/captive-core-testnet.cfg"
+fi
+
 # Calculate params for AWS Batch
 if [ ! -z "$AWS_BATCH_JOB_ARRAY_INDEX" ]; then
 	# The batch should have three env variables:
@@ -39,9 +47,10 @@ fi
 echo "START: $START END: $END"
 
 export TRACY_NO_INVARIANT_CHECK=1
-/ledgerexporter --target $ARCHIVE_TARGET \
-                --captive-core-toml-path /captive-core-pubnet.cfg \
-                --history-archive-urls 'https://history.stellar.org/prd/core-live/core_live_001' --network-passphrase 'Public Global Stellar Network ; September 2015' \
-                --continue="$CONTINUE" --write-latest-path="$WRITE_LATEST_PATH" --start-ledger "$START" --end-ledger "$END"
+/ledgerexporter --target "$ARCHIVE_TARGET" \
+                --captive-core-toml-path "$CAPTIVE_CORE_CONFIG" \
+                --history-archive-urls "$HISTORY_ARCHIVE_URLS" --network-passphrase "$NETWORK_PASSPHRASE" \
+                --continue="$CONTINUE" --write-latest-path="$WRITE_LATEST_PATH" \
+				--start-ledger "$START" --end-ledger "$END" --captive-core-use-db "$CAPTIVE_CORE_USE_DB"
 
 echo "OK"