update verify-range for captive core use-disk (#4444)

.github/workflows/horizon.yml

@@ -96,6 +96,9 @@ jobs:
   verify-range:
     name: Test (and push) verify-range image
     runs-on: ubuntu-latest
+    env:
+      STELLAR_CORE_VERSION: 19.2.0-966.d18d54aa3.focal 
+      CAPTIVE_CORE_STORAGE_PATH: /tmp
     steps:
     - uses: actions/checkout@v3
       with:

services/horizon/docker/verify-range/Dockerfile

@@ -12,6 +12,9 @@ RUN ["chmod", "+x", "dependencies"]
 RUN /dependencies
 
 ADD stellar-core.cfg /
+ADD captive-core-pubnet.cfg /
+RUN mkdir -p /cc
+RUN mkdir -p /data
 
 ADD start /
 RUN ["chmod", "+x", "start"]

services/horizon/docker/verify-range/captive-core-pubnet.cfg

@@ -0,0 +1,193 @@
+PEER_PORT=11725
+DATABASE = "sqlite3:///cc/stellar.db"
+
+FAILURE_SAFETY=1
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="stellar.org"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="satoshipay.io"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="lobstr.co"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="www.coinqvest.com"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="publicnode.org"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="stellar.blockdaemon.com"
+QUALITY="HIGH"
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="wirexapp.com"
+QUALITY="HIGH"
+
+[[VALIDATORS]]
+NAME="sdf_1"
+HOME_DOMAIN="stellar.org"
+PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH"
+ADDRESS="core-live-a.stellar.org:11625"
+HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_2"
+HOME_DOMAIN="stellar.org"
+PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK"
+ADDRESS="core-live-b.stellar.org:11625"
+HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="sdf_3"
+HOME_DOMAIN="stellar.org"
+PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ"
+ADDRESS="core-live-c.stellar.org:11625"
+HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="satoshipay_singapore"
+HOME_DOMAIN="satoshipay.io"
+PUBLIC_KEY="GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT"
+ADDRESS="stellar-sg-sin.satoshipay.io:11625"
+HISTORY="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="satoshipay_iowa"
+HOME_DOMAIN="satoshipay.io"
+PUBLIC_KEY="GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY"
+ADDRESS="stellar-us-iowa.satoshipay.io:11625"
+HISTORY="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="satoshipay_frankfurt"
+HOME_DOMAIN="satoshipay.io"
+PUBLIC_KEY="GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE"
+ADDRESS="stellar-de-fra.satoshipay.io:11625"
+HISTORY="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_1_europe"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7"
+ADDRESS="v1.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-1-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_2_europe"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GDXQB3OMMQ6MGG43PWFBZWBFKBBDUZIVSUDAZZTRAWQZKES2CDSE5HKJ"
+ADDRESS="v2.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-2-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_3_north_america"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63"
+ADDRESS="v3.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-3-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_4_asia"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J"
+ADDRESS="v4.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-4-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lobstr_5_australia"
+HOME_DOMAIN="lobstr.co"
+PUBLIC_KEY="GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7"
+ADDRESS="v5.stellar.lobstr.co:11625"
+HISTORY="curl -sf https://stellar-archive-5-lobstr.s3.amazonaws.com/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="coinqvest_hong_kong"
+HOME_DOMAIN="www.coinqvest.com"
+PUBLIC_KEY="GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z"
+ADDRESS="hongkong.stellar.coinqvest.com:11625"
+HISTORY="curl -sf https://hongkong.stellar.coinqvest.com/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="coinqvest_germany"
+HOME_DOMAIN="www.coinqvest.com"
+PUBLIC_KEY="GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN"
+ADDRESS="germany.stellar.coinqvest.com:11625"
+HISTORY="curl -sf https://germany.stellar.coinqvest.com/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="coinqvest_finland"
+HOME_DOMAIN="www.coinqvest.com"
+PUBLIC_KEY="GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T"
+ADDRESS="finland.stellar.coinqvest.com:11625"
+HISTORY="curl -sf https://finland.stellar.coinqvest.com/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="bootes"
+HOME_DOMAIN="publicnode.org"
+PUBLIC_KEY="GCVJ4Z6TI6Z2SOGENSPXDQ2U4RKH3CNQKYUHNSSPYFPNWTLGS6EBH7I2"
+ADDRESS="bootes.publicnode.org"
+HISTORY="curl -sf https://bootes-history.publicnode.org/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="hercules"
+HOME_DOMAIN="publicnode.org"
+PUBLIC_KEY="GBLJNN3AVZZPG2FYAYTYQKECNWTQYYUUY2KVFN2OUKZKBULXIXBZ4FCT"
+ADDRESS="hercules.publicnode.org"
+HISTORY="curl -sf https://hercules-history.publicnode.org/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="lyra"
+HOME_DOMAIN="publicnode.org"
+PUBLIC_KEY="GCIXVKNFPKWVMKJKVK2V4NK7D4TC6W3BUMXSIJ365QUAXWBRPPJXIR2Z"
+ADDRESS="lyra.publicnode.org"
+HISTORY="curl -sf https://lyra-history.publicnode.org/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="Blockdaemon_Validator_1"
+HOME_DOMAIN="stellar.blockdaemon.com"
+PUBLIC_KEY="GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU"
+ADDRESS="stellar-full-validator1.bdnodes.net"
+HISTORY="curl -sf https://stellar-full-history1.bdnodes.net/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="Blockdaemon_Validator_2"
+HOME_DOMAIN="stellar.blockdaemon.com"
+PUBLIC_KEY="GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C"
+ADDRESS="stellar-full-validator2.bdnodes.net"
+HISTORY="curl -sf https://stellar-full-history2.bdnodes.net/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="Blockdaemon_Validator_3"
+HOME_DOMAIN="stellar.blockdaemon.com"
+PUBLIC_KEY="GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4"
+ADDRESS="stellar-full-validator3.bdnodes.net"
+HISTORY="curl -sf https://stellar-full-history3.bdnodes.net/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="wirexUS"
+ADDRESS="us.stellar.wirexapp.com"
+HOME_DOMAIN="wirexapp.com"
+PUBLIC_KEY="GDXUKFGG76WJC7ACEH3JUPLKM5N5S76QSMNDBONREUXPCZYVPOLFWXUS"
+HISTORY="curl -sf http://wxhorizonusstga1.blob.core.windows.net/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="wirexUK"
+ADDRESS="uk.stellar.wirexapp.com"
+HOME_DOMAIN="wirexapp.com"
+PUBLIC_KEY="GBBQQT3EIUSXRJC6TGUCGVA3FVPXVZLGG3OJYACWBEWYBHU46WJLWXEU"
+HISTORY="curl -sf http://wxhorizonukstga1.blob.core.windows.net/history/{0} -o {1}"
+
+[[VALIDATORS]]
+NAME="wirexSG"
+ADDRESS="sg.stellar.wirexapp.com"
+HOME_DOMAIN="wirexapp.com"
+PUBLIC_KEY="GAB3GZIE6XAYWXGZUDM4GMFFLJBFMLE2JDPUCWUZXMOMT3NHXDHEWXAS"
+HISTORY="curl -sf http://wxhorizonasiastga1.blob.core.windows.net/history/{0} -o {1}"

services/horizon/docker/verify-range/dependencies

@@ -14,12 +14,6 @@ apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 wget -q https://dl.google.com/go/go1.17.linux-amd64.tar.gz
 tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
 
-# configure postgres
-service postgresql start
-sudo -u postgres createdb horizon
-
-sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
-
 git clone https://github.com/stellar/go.git stellar-go
 cd stellar-go
 # By default "git fetch" only fetches refs/<branchname>

services/horizon/docker/verify-range/start

@@ -1,7 +1,17 @@
 #! /usr/bin/env bash
 set -e
 
-service postgresql start
+# configure postgres
+if [ -z "${PGDATA}" ]; then
+    export PGDATA="/data"
+fi
+rm -rf "$PGDATA"/* 
+sudo chown -R postgres "$PGDATA"
+sudo chmod -R 775 "$PGDATA"
+sudo -u postgres --preserve-env=PGDATA /usr/lib/postgresql/12/bin/initdb
+sudo -u postgres --preserve-env=PGDATA /usr/lib/postgresql/12/bin/pg_ctl start
+sudo -u postgres createdb horizon
+sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
 
 # Calculate params for AWS Batch
 if [ ! -z "$AWS_BATCH_JOB_ARRAY_INDEX" ]; then
@@ -61,6 +71,19 @@ dump_horizon_db() {
 export NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
 export HISTORY_ARCHIVE_URLS="https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001"
 export DATABASE_URL="postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable"
+export CAPTIVE_CORE_USE_DB="true"
+export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" 
+export STELLAR_CORE_BINARY_PATH="/usr/bin/stellar-core"
+export ENABLE_CAPTIVE_CORE_INGESTION="true"
+
+# CAPTIVE_CORE_STORAGE_PATH will store the archive files that core downloads
+# /cc path in this vm will have the sqlite3://stellar.db file which captive core will use for on-disk 
+# do a docker volume mount on /cc if the main disk storage devivce won't have at least 3k IOPS and 256GB
+# which is required by core for the i/o it will do against sqlite3:///cc/stellardb  
+if [ -z "${CAPTIVE_CORE_STORAGE_PATH}" ]; then
+    export CAPTIVE_CORE_STORAGE_PATH="/cc"
+fi	
+export CAPTIVE_CORE_REUSE_STORAGE_PATH="false"
 
 cd stellar-go
 git pull origin
@@ -93,13 +116,14 @@ function alter_tables_unlogged() {
 }
 
 /usr/local/go/bin/go build -v ./services/horizon
-./horizon db migrate up
+cd $CAPTIVE_CORE_STORAGE_PATH
+/stellar-go/horizon db migrate up
 alter_tables_unlogged
-./horizon ingest verify-range --from $FROM --to $TO --verify-state
+/stellar-go/horizon ingest verify-range --from $FROM --to $TO --verify-state
 
 function compare() {
-  local expected="old_$1"
-  local actual="new_$1"
+  local expected="/data/compare/old_$1"
+  local actual="/data/compare/new_$1"
 
   # Files can be very large, leading to `diff` running out of memory.
   # As a workaround, since files are expected to be identical,
@@ -114,24 +138,27 @@ function compare() {
 
 BASE_BRANCH=${BASE_BRANCH:-horizon-v2.0.0}
 
+rm -rf /data/compare
+mkdir /data/compare
 if [ ! -z "$VERIFY_HISTORY" ]; then
-	dump_horizon_db "new_history"
+	dump_horizon_db "/data/compare/new_history"
 
 	echo "Done dump_horizon_db new_history"
 
 	psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -c "DROP SCHEMA public CASCADE;"
 	psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -c "CREATE SCHEMA public;"
 
+    cd /stellar-go
 	git checkout "$BASE_BRANCH"
-
 	/usr/local/go/bin/go build -v ./services/horizon
-
-	./horizon db migrate up
+
+	cd $CAPTIVE_CORE_STORAGE_PATH
+	/stellar-go/horizon db migrate up
 	alter_tables_unlogged
 	REINGEST_FROM=$((FROM + 1)) # verify-range does not ingest starting ledger
-	./horizon db reingest range $REINGEST_FROM $TO
+	/stellar-go/horizon db reingest range $REINGEST_FROM $TO
 
-	dump_horizon_db "old_history"
+	dump_horizon_db "/data/compare/old_history"
 	echo "Done dump_horizon_db old_history"
 
 	compare history_effects

services/horizon/docker/verify-range/stellar-core.cfg

@@ -1,10 +1,6 @@
 AUTOMATIC_MAINTENANCE_PERIOD=0
-
-HTTP_PORT=11626
 PUBLIC_HTTP_PORT=true
 LOG_FILE_PATH=""
-
-DATABASE="postgresql://dbname=core host=localhost user=postgres password=postgres"
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
 CATCHUP_RECENT=100