diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index ecd2074f21..7c3e94d5b1 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -96,6 +96,9 @@ jobs: verify-range: name: Test (and push) verify-range image runs-on: ubuntu-latest + env: + STELLAR_CORE_VERSION: 19.2.0-966.d18d54aa3.focal + CAPTIVE_CORE_STORAGE_PATH: /tmp steps: - uses: actions/checkout@v3 with: diff --git a/services/horizon/docker/verify-range/Dockerfile b/services/horizon/docker/verify-range/Dockerfile index 4cc335b664..499f86881e 100644 --- a/services/horizon/docker/verify-range/Dockerfile +++ b/services/horizon/docker/verify-range/Dockerfile @@ -12,6 +12,9 @@ RUN ["chmod", "+x", "dependencies"] RUN /dependencies ADD stellar-core.cfg / +ADD captive-core-pubnet.cfg / +RUN mkdir -p /cc +RUN mkdir -p /data ADD start / RUN ["chmod", "+x", "start"] diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg new file mode 100644 index 0000000000..537325d4e9 --- /dev/null +++ b/services/horizon/docker/verify-range/captive-core-pubnet.cfg @@ -0,0 +1,193 @@ +PEER_PORT=11725 +DATABASE = "sqlite3:///cc/stellar.db" + +FAILURE_SAFETY=1 + +[[HOME_DOMAINS]] +HOME_DOMAIN="stellar.org" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="satoshipay.io" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="lobstr.co" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="www.coinqvest.com" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="publicnode.org" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="stellar.blockdaemon.com" +QUALITY="HIGH" + +[[HOME_DOMAINS]] +HOME_DOMAIN="wirexapp.com" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="sdf_1" +HOME_DOMAIN="stellar.org" +PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH" +ADDRESS="core-live-a.stellar.org:11625" +HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdf_2" +HOME_DOMAIN="stellar.org" +PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK" +ADDRESS="core-live-b.stellar.org:11625" +HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdf_3" +HOME_DOMAIN="stellar.org" +PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ" +ADDRESS="core-live-c.stellar.org:11625" +HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" + +[[VALIDATORS]] +NAME="satoshipay_singapore" +HOME_DOMAIN="satoshipay.io" +PUBLIC_KEY="GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT" +ADDRESS="stellar-sg-sin.satoshipay.io:11625" +HISTORY="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="satoshipay_iowa" +HOME_DOMAIN="satoshipay.io" +PUBLIC_KEY="GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY" +ADDRESS="stellar-us-iowa.satoshipay.io:11625" +HISTORY="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="satoshipay_frankfurt" +HOME_DOMAIN="satoshipay.io" +PUBLIC_KEY="GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE" +ADDRESS="stellar-de-fra.satoshipay.io:11625" +HISTORY="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_1_europe" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7" +ADDRESS="v1.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-1-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_2_europe" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GDXQB3OMMQ6MGG43PWFBZWBFKBBDUZIVSUDAZZTRAWQZKES2CDSE5HKJ" +ADDRESS="v2.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-2-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_3_north_america" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63" +ADDRESS="v3.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-3-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_4_asia" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J" +ADDRESS="v4.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-4-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="lobstr_5_australia" +HOME_DOMAIN="lobstr.co" +PUBLIC_KEY="GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7" +ADDRESS="v5.stellar.lobstr.co:11625" +HISTORY="curl -sf https://stellar-archive-5-lobstr.s3.amazonaws.com/{0} -o {1}" + +[[VALIDATORS]] +NAME="coinqvest_hong_kong" +HOME_DOMAIN="www.coinqvest.com" +PUBLIC_KEY="GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z" +ADDRESS="hongkong.stellar.coinqvest.com:11625" +HISTORY="curl -sf https://hongkong.stellar.coinqvest.com/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="coinqvest_germany" +HOME_DOMAIN="www.coinqvest.com" +PUBLIC_KEY="GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN" +ADDRESS="germany.stellar.coinqvest.com:11625" +HISTORY="curl -sf https://germany.stellar.coinqvest.com/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="coinqvest_finland" +HOME_DOMAIN="www.coinqvest.com" +PUBLIC_KEY="GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T" +ADDRESS="finland.stellar.coinqvest.com:11625" +HISTORY="curl -sf https://finland.stellar.coinqvest.com/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="bootes" +HOME_DOMAIN="publicnode.org" +PUBLIC_KEY="GCVJ4Z6TI6Z2SOGENSPXDQ2U4RKH3CNQKYUHNSSPYFPNWTLGS6EBH7I2" +ADDRESS="bootes.publicnode.org" +HISTORY="curl -sf https://bootes-history.publicnode.org/{0} -o {1}" + +[[VALIDATORS]] +NAME="hercules" +HOME_DOMAIN="publicnode.org" +PUBLIC_KEY="GBLJNN3AVZZPG2FYAYTYQKECNWTQYYUUY2KVFN2OUKZKBULXIXBZ4FCT" +ADDRESS="hercules.publicnode.org" +HISTORY="curl -sf https://hercules-history.publicnode.org/{0} -o {1}" + +[[VALIDATORS]] +NAME="lyra" +HOME_DOMAIN="publicnode.org" +PUBLIC_KEY="GCIXVKNFPKWVMKJKVK2V4NK7D4TC6W3BUMXSIJ365QUAXWBRPPJXIR2Z" +ADDRESS="lyra.publicnode.org" +HISTORY="curl -sf https://lyra-history.publicnode.org/{0} -o {1}" + +[[VALIDATORS]] +NAME="Blockdaemon_Validator_1" +HOME_DOMAIN="stellar.blockdaemon.com" +PUBLIC_KEY="GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU" +ADDRESS="stellar-full-validator1.bdnodes.net" +HISTORY="curl -sf https://stellar-full-history1.bdnodes.net/{0} -o {1}" + +[[VALIDATORS]] +NAME="Blockdaemon_Validator_2" +HOME_DOMAIN="stellar.blockdaemon.com" +PUBLIC_KEY="GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C" +ADDRESS="stellar-full-validator2.bdnodes.net" +HISTORY="curl -sf https://stellar-full-history2.bdnodes.net/{0} -o {1}" + +[[VALIDATORS]] +NAME="Blockdaemon_Validator_3" +HOME_DOMAIN="stellar.blockdaemon.com" +PUBLIC_KEY="GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4" +ADDRESS="stellar-full-validator3.bdnodes.net" +HISTORY="curl -sf https://stellar-full-history3.bdnodes.net/{0} -o {1}" + +[[VALIDATORS]] +NAME="wirexUS" +ADDRESS="us.stellar.wirexapp.com" +HOME_DOMAIN="wirexapp.com" +PUBLIC_KEY="GDXUKFGG76WJC7ACEH3JUPLKM5N5S76QSMNDBONREUXPCZYVPOLFWXUS" +HISTORY="curl -sf http://wxhorizonusstga1.blob.core.windows.net/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="wirexUK" +ADDRESS="uk.stellar.wirexapp.com" +HOME_DOMAIN="wirexapp.com" +PUBLIC_KEY="GBBQQT3EIUSXRJC6TGUCGVA3FVPXVZLGG3OJYACWBEWYBHU46WJLWXEU" +HISTORY="curl -sf http://wxhorizonukstga1.blob.core.windows.net/history/{0} -o {1}" + +[[VALIDATORS]] +NAME="wirexSG" +ADDRESS="sg.stellar.wirexapp.com" +HOME_DOMAIN="wirexapp.com" +PUBLIC_KEY="GAB3GZIE6XAYWXGZUDM4GMFFLJBFMLE2JDPUCWUZXMOMT3NHXDHEWXAS" +HISTORY="curl -sf http://wxhorizonasiastga1.blob.core.windows.net/history/{0} -o {1}" \ No newline at end of file diff --git a/services/horizon/docker/verify-range/dependencies b/services/horizon/docker/verify-range/dependencies index 45b9e60e08..f963595ccb 100644 --- a/services/horizon/docker/verify-range/dependencies +++ b/services/horizon/docker/verify-range/dependencies @@ -14,12 +14,6 @@ apt-get install -y stellar-core=${STELLAR_CORE_VERSION} wget -q https://dl.google.com/go/go1.17.linux-amd64.tar.gz tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz -# configure postgres -service postgresql start -sudo -u postgres createdb horizon - -sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" - git clone https://github.com/stellar/go.git stellar-go cd stellar-go # By default "git fetch" only fetches refs/ diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index 0ebc868873..a0c5f0df25 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -1,7 +1,17 @@ #! /usr/bin/env bash set -e -service postgresql start +# configure postgres +if [ -z "${PGDATA}" ]; then + export PGDATA="/data" +fi +rm -rf "$PGDATA"/* +sudo chown -R postgres "$PGDATA" +sudo chmod -R 775 "$PGDATA" +sudo -u postgres --preserve-env=PGDATA /usr/lib/postgresql/12/bin/initdb +sudo -u postgres --preserve-env=PGDATA /usr/lib/postgresql/12/bin/pg_ctl start +sudo -u postgres createdb horizon +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" # Calculate params for AWS Batch if [ ! -z "$AWS_BATCH_JOB_ARRAY_INDEX" ]; then @@ -61,6 +71,19 @@ dump_horizon_db() { export NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" export HISTORY_ARCHIVE_URLS="https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001" export DATABASE_URL="postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" +export CAPTIVE_CORE_USE_DB="true" +export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" +export STELLAR_CORE_BINARY_PATH="/usr/bin/stellar-core" +export ENABLE_CAPTIVE_CORE_INGESTION="true" + +# CAPTIVE_CORE_STORAGE_PATH will store the archive files that core downloads +# /cc path in this vm will have the sqlite3://stellar.db file which captive core will use for on-disk +# do a docker volume mount on /cc if the main disk storage devivce won't have at least 3k IOPS and 256GB +# which is required by core for the i/o it will do against sqlite3:///cc/stellardb +if [ -z "${CAPTIVE_CORE_STORAGE_PATH}" ]; then + export CAPTIVE_CORE_STORAGE_PATH="/cc" +fi +export CAPTIVE_CORE_REUSE_STORAGE_PATH="false" cd stellar-go git pull origin @@ -93,13 +116,14 @@ function alter_tables_unlogged() { } /usr/local/go/bin/go build -v ./services/horizon -./horizon db migrate up +cd $CAPTIVE_CORE_STORAGE_PATH +/stellar-go/horizon db migrate up alter_tables_unlogged -./horizon ingest verify-range --from $FROM --to $TO --verify-state +/stellar-go/horizon ingest verify-range --from $FROM --to $TO --verify-state function compare() { - local expected="old_$1" - local actual="new_$1" + local expected="/data/compare/old_$1" + local actual="/data/compare/new_$1" # Files can be very large, leading to `diff` running out of memory. # As a workaround, since files are expected to be identical, @@ -114,24 +138,27 @@ function compare() { BASE_BRANCH=${BASE_BRANCH:-horizon-v2.0.0} +rm -rf /data/compare +mkdir /data/compare if [ ! -z "$VERIFY_HISTORY" ]; then - dump_horizon_db "new_history" + dump_horizon_db "/data/compare/new_history" echo "Done dump_horizon_db new_history" psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -c "DROP SCHEMA public CASCADE;" psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -c "CREATE SCHEMA public;" + cd /stellar-go git checkout "$BASE_BRANCH" - /usr/local/go/bin/go build -v ./services/horizon - - ./horizon db migrate up + + cd $CAPTIVE_CORE_STORAGE_PATH + /stellar-go/horizon db migrate up alter_tables_unlogged REINGEST_FROM=$((FROM + 1)) # verify-range does not ingest starting ledger - ./horizon db reingest range $REINGEST_FROM $TO + /stellar-go/horizon db reingest range $REINGEST_FROM $TO - dump_horizon_db "old_history" + dump_horizon_db "/data/compare/old_history" echo "Done dump_horizon_db old_history" compare history_effects diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg index 3dc6c70fb0..6139f9f528 100644 --- a/services/horizon/docker/verify-range/stellar-core.cfg +++ b/services/horizon/docker/verify-range/stellar-core.cfg @@ -1,10 +1,6 @@ AUTOMATIC_MAINTENANCE_PERIOD=0 - -HTTP_PORT=11626 PUBLIC_HTTP_PORT=true LOG_FILE_PATH="" - -DATABASE="postgresql://dbname=core host=localhost user=postgres password=postgres" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100