services/horizon/docker/ledgerexporter: deploy ledgerexporter image as service

exp/services/ledgerexporter/main.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/stellar/go/historyarchive"
 	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/network"
@@ -30,6 +31,7 @@ func main() {
 	continueFromLatestLedger := flag.Bool("continue", false, "start export from the last exported ledger (as indicated in the target's /latest path)")
 	endingLedger := flag.Uint("end-ledger", 0, "ledger at which to stop the export (must be a closed ledger), 0 means no ending")
 	writeLatestPath := flag.Bool("write-latest-path", true, "update the value of the /latest path on the target")
+	captiveCoreUseDb := flag.Bool("captive-core-use-db", true, "configure captive core to store database on disk in working directory rather than in memory")
 	flag.Parse()
 
 	logger.SetLevel(supportlog.InfoLevel)
@@ -52,14 +54,16 @@ func main() {
 		CheckpointFrequency: 64,
 		Log:                 logger.WithField("subservice", "stellar-core"),
 		Toml:                captiveCoreToml,
+		UseDB:               *captiveCoreUseDb,
 	}
 	core, err := ledgerbackend.NewCaptive(captiveConfig)
 	logFatalIf(err, "Could not create captive core instance")
 
 	target, err := historyarchive.ConnectBackend(
 		*targetUrl,
 		storage.ConnectOptions{
-			Context: context.Background(),
+			Context:    context.Background(),
+			S3WriteACL: s3.ObjectCannedACLBucketOwnerFullControl,
 		},
 	)
 	logFatalIf(err, "Could not connect to target")
@@ -68,18 +72,23 @@ func main() {
 	// Build the appropriate range for the given backend state.
 	startLedger := uint32(*startingLedger)
 	endLedger := uint32(*endingLedger)
-	if startLedger < 2 {
-		logger.Fatalf("-start-ledger must be >= 2")
-	}
-	if endLedger != 0 && endLedger < startLedger {
-		logger.Fatalf("-end-ledger must be >= -start-ledger")
-	}
+
+	logger.Infof("processing requested range of -start-ledger=%v, -end-ledger=%v", startLedger, endLedger)
 	if *continueFromLatestLedger {
 		if startLedger != 0 {
 			logger.Fatalf("-start-ledger and -continue cannot both be set")
 		}
 		startLedger = readLatestLedger(target)
+		logger.Infof("continue flag was enabled, next ledger found was %v", startLedger)
 	}
+
+	if startLedger < 2 {
+		logger.Fatalf("-start-ledger must be >= 2")
+	}
+	if endLedger != 0 && endLedger < startLedger {
+		logger.Fatalf("-end-ledger must be >= -start-ledger")
+	}
+
 	var ledgerRange ledgerbackend.Range
 	if endLedger == 0 {
 		ledgerRange = ledgerbackend.UnboundedRange(startLedger)
@@ -91,7 +100,7 @@ func main() {
 	err = core.PrepareRange(context.Background(), ledgerRange)
 	logFatalIf(err, "could not prepare range")
 
-	for nextLedger := startLedger; nextLedger <= endLedger; {
+	for nextLedger := startLedger; endLedger < 1 || nextLedger <= endLedger; {
 		ledger, err := core.GetLedger(context.Background(), nextLedger)
 		if err != nil {
 			logger.WithError(err).Warnf("could not fetch ledger %v, retrying", nextLedger)

services/horizon/docker/ledgerexporter/Dockerfile

@@ -27,6 +27,7 @@ RUN apt-get update && apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 RUN apt-get clean
 
 ADD captive-core-pubnet.cfg /
+ADD captive-core-testnet.cfg /
 
 ADD start /
 RUN ["chmod", "+x", "start"]

services/horizon/docker/ledgerexporter/captive-core-testnet.cfg

@@ -0,0 +1,30 @@
+PEER_PORT=11725
+DATABASE = "sqlite3:///cc/stellar.db"
+
+UNSAFE_QUORUM=true
+FAILURE_SAFETY=1
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="HIGH"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="core-testnet1.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_2"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP"
+ADDRESS="core-testnet2.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_3"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z"
+ADDRESS="core-testnet3.stellar.org"
+HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}"

services/horizon/docker/ledgerexporter/ledgerexporter.yml

@@ -0,0 +1,125 @@
+# this file contains the ledgerexporter deployment and it's config artifacts.
+#
+# when applying the manifest on a cluster, make sure to include namespace destination, 
+# as the manifest does not specify namespace, otherwise it'll go in your current kubectl context.
+#
+# make sure to set the secrets values, substitue <base64 encoded value here> placeholders.
+#
+# $ kubectl apply -f ledgerexporter.yml -n horizon-dev
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  annotations:
+    fluxcd.io/ignore: "true"
+  labels:
+    app: ledgerexporter
+  name: ledgerexporter-pubnet-env
+data:
+  # when using core 'on disk', the earliest ledger to get streamed out after catchup to 2, is 3
+  # whereas on in-memory it streas out 2, adjusted here, otherwise horizon ingest will abort
+  # and stop process with error that ledger 3 is not <= expected ledger of 2.    
+  START: "0"
+  END: "0"
+
+  # can only have CONTINUE or START set, not both.
+  CONTINUE: "true"
+  WRITE_LATEST_PATH: "true"
+  CAPTIVE_CORE_USE_DB: "true"
+
+  # configure the network to export
+  HISTORY_ARCHIVE_URLS: "https://history.stellar.org/prd/core-live/core_live_001,https://history.stellar.org/prd/core-live/core_live_002,https://history.stellar.org/prd/core-live/core_live_003"
+  NETWORK_PASSPHRASE: "Public Global Stellar Network ; September 2015"
+  # can refer to canned cfg's for pubnet and testnet which are included on the image 
+  # `/captive-core-pubnet.cfg` or `/captive-core-testnet.cfg`. 
+  # If exporting a standalone network, then mount a volume to the pod container with your standalone core's .cfg,
+  # and set full path to that volume here
+  CAPTIVE_CORE_CONFIG: "/captive-core-pubnet.cfg"
+
+  # example of testnet network config. 
+  # HISTORY_ARCHIVE_URLS: "https://history.stellar.org/prd/core-testnet/core_testnet_001,https://history.stellar.org/prd/core-testnet/core_testnet_002"
+  # NETWORK_PASSPHRASE: "Test SDF Network ; September 2015"
+  # CAPTIVE_CORE_CONFIG: "/captive-core-testnet.cfg"
+
+  # provide the url for the external s3 bucket to be populated
+  # update the ledgerexporter-pubnet-secret to have correct aws key/secret for access to the bucket
+  ARCHIVE_TARGET: "s3://horizon-ledgermeta-prodnet-test"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app: ledgerexporter
+  name: ledgerexporter-pubnet-secret
+type: Opaque
+data:
+  AWS_REGION: <base64 encoded value here>
+  AWS_ACCESS_KEY_ID: <base64 encoded value here>
+  AWS_SECRET_ACCESS_KEY: <base64 encoded value here> 
+---
+# running captive core with on-disk mode limits RAM to around 2G usage, but 
+# requires some dedicated disk storage space that has at least 3k IOPS for read/write.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ledgerexporter-pubnet-core-storage
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 500Gi
+  storageClassName: default
+  volumeMode: Filesystem
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    fluxcd.io/ignore: "true"
+    deployment.kubernetes.io/revision: "3"
+  labels:
+    app: ledgerexporter-pubnet
+  name: ledgerexporter-pubnet-deployment
+spec:
+  selector:
+    matchLabels:
+      app: ledgerexporter-pubnet
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        fluxcd.io/ignore: "true"
+        # if we expect to add metrics at some point to ledgerexporter
+        # this just needs to be set to true
+        prometheus.io/port: "6060"
+        prometheus.io/scrape: "false"
+      labels:
+        app: ledgerexporter-pubnet
+    spec:
+      containers:
+      - envFrom:
+        - secretRef:
+            name: ledgerexporter-pubnet-secret
+        - configMapRef:
+            name: ledgerexporter-pubnet-env
+        image: stellar/horizon-ledgerexporter:latest
+        imagePullPolicy: Always
+        name: ledgerexporter-pubnet
+        resources:  
+          limits:
+            cpu:     3
+            memory:  8Gi
+          requests:
+            cpu:      500m
+            memory:   2Gi
+        volumeMounts:
+        - mountPath: /cc
+          name: core-storage
+      dnsPolicy: ClusterFirst
+      volumes:
+      - name: core-storage
+        persistentVolumeClaim:
+          claimName: ledgerexporter-pubnet-core-storage 
+
+
+

services/horizon/docker/ledgerexporter/start

@@ -1,12 +1,19 @@
 #! /usr/bin/env bash
 set -e
 
-START="${START:=0}"
+START="${START:=2}"
 END="${END:=0}"
 CONTINUE="${CONTINUE:=false}"
 # Writing to /latest is disabled by default to avoid race conditions between parallel container runs
 WRITE_LATEST_PATH="${WRITE_LATEST_PATH:=false}"
 
+# config defaults to pubnet core, any other network requires setting all 3 of these in container env
+NETWORK_PASSPHRASE="${NETWORK_PASSPHRASE:=Public Global Stellar Network ; September 2015}"
+HISTORY_ARCHIVE_URLS="${HISTORY_ARCHIVE_URLS:=https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001}"
+CAPTIVE_CORE_CONFIG="${CAPTIVE_CORE_CONFIG:=/captive-core-pubnet.cfg}"
+
+CAPTIVE_CORE_USE_DB="${CAPTIVE_CORE_USE_DB:=true}"
+
 if [ -z "$ARCHIVE_TARGET" ]; then
   echo "error: undefined ARCHIVE_TARGET env variable"
   exit 1
@@ -39,9 +46,10 @@ fi
 echo "START: $START END: $END"
 
 export TRACY_NO_INVARIANT_CHECK=1
-/ledgerexporter --target $ARCHIVE_TARGET \
-                --captive-core-toml-path /captive-core-pubnet.cfg \
-                --history-archive-urls 'https://history.stellar.org/prd/core-live/core_live_001' --network-passphrase 'Public Global Stellar Network ; September 2015' \
-                --continue="$CONTINUE" --write-latest-path="$WRITE_LATEST_PATH" --start-ledger "$START" --end-ledger "$END"
+/ledgerexporter --target "$ARCHIVE_TARGET" \
+                --captive-core-toml-path "$CAPTIVE_CORE_CONFIG" \
+                --history-archive-urls "$HISTORY_ARCHIVE_URLS" --network-passphrase "$NETWORK_PASSPHRASE" \
+                --continue="$CONTINUE" --write-latest-path="$WRITE_LATEST_PATH" \
+				--start-ledger "$START" --end-ledger "$END" --captive-core-use-db="$CAPTIVE_CORE_USE_DB"
 
 echo "OK"

support/storage/main.go

@@ -31,6 +31,9 @@ type ConnectOptions struct {
 
 	// Wrap the Storage after connection. For example, to add a caching or introspection layer.
 	Wrap func(Storage) (Storage, error)
+
+	// When putting file object to s3 bucket, specify the ACL for the object.
+	S3WriteACL string
 }
 
 func ConnectBackend(u string, opts ConnectOptions) (Storage, error) {
@@ -60,6 +63,7 @@ func ConnectBackend(u string, opts ConnectOptions) (Storage, error) {
 			opts.S3Region,
 			opts.S3Endpoint,
 			opts.UnsignedRequests,
+			opts.S3WriteACL,
 		)
 
 	case "gcs":

support/storage/s3.go

@@ -12,15 +12,17 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/stellar/go/support/errors"
 )
 
 type S3Storage struct {
 	ctx              context.Context
-	svc              *s3.S3
+	svc              s3iface.S3API
 	bucket           string
 	prefix           string
 	unsignedRequests bool
+	writeACLrule     string
 }
 
 func NewS3Storage(
@@ -30,6 +32,7 @@ func NewS3Storage(
 	region string,
 	endpoint string,
 	unsignedRequests bool,
+	writeACLrule string,
 ) (Storage, error) {
 	log.WithFields(log.Fields{"bucket": bucket,
 		"prefix":   prefix,
@@ -52,6 +55,7 @@ func NewS3Storage(
 		bucket:           bucket,
 		prefix:           prefix,
 		unsignedRequests: unsignedRequests,
+		writeACLrule:     writeACLrule,
 	}
 	return &backend, nil
 }
@@ -139,6 +143,13 @@ func (b *S3Storage) Size(pth string) (int64, error) {
 	}
 }
 
+func (b *S3Storage) GetACLWriteRule() string {
+	if b.writeACLrule == "" {
+		return s3.ObjectCannedACLPublicRead
+	}
+	return b.writeACLrule
+}
+
 func (b *S3Storage) PutFile(pth string, in io.ReadCloser) error {
 	var buf bytes.Buffer
 	_, err := buf.ReadFrom(in)
@@ -150,7 +161,7 @@ func (b *S3Storage) PutFile(pth string, in io.ReadCloser) error {
 	params := &s3.PutObjectInput{
 		Bucket: aws.String(b.bucket),
 		Key:    aws.String(key),
-		ACL:    aws.String(s3.ObjectCannedACLPublicRead),
+		ACL:    aws.String(b.GetACLWriteRule()),
 		Body:   bytes.NewReader(buf.Bytes()),
 	}
 	req, _ := b.svc.PutObjectRequest(params)