Merge branch 'lighthorizon' into lighthorizon_refactorArchive

.github/workflows/horizon.yml

@@ -134,14 +134,27 @@ jobs:
         # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
         run: |
           chmod 755 ./exp/lighthorizon/build/build.sh
-          ./exp/lighthorizon/build/build.sh ledgerexporter stellar latest false
-          docker run -e ARCHIVE_TARGET=file:///ledgerexport-test\
+          mkdir $PWD/ledgerexport
+          mkdir $PWD/index
+
+          ./exp/lighthorizon/build/build.sh all stellar latest false
+          docker run -e ARCHIVE_TARGET=file:///ledgerexport\
                      -e START=5\
-                     -e END=50\
+                     -e END=150\
                      -e NETWORK_PASSPHRASE="Test SDF Network ; September 2015"\
                      -e CAPTIVE_CORE_CONFIG="/captive-core-testnet.cfg"\
                      -e HISTORY_ARCHIVE_URLS="https://history.stellar.org/prd/core-testnet/core_testnet_001,https://history.stellar.org/prd/core-testnet/core_testnet_002"\
+                     -v $PWD/ledgerexport:/ledgerexport\
                      stellar/lighthorizon-ledgerexporter
+          
+          # run map job
+          docker run -e NETWORK_PASSPHRASE='testnet' -e JOB_INDEX_ENV=AWS_BATCH_JOB_ARRAY_INDEX -e AWS_BATCH_JOB_ARRAY_INDEX=0 -e BATCH_SIZE=64 -e FIRST_CHECKPOINT=64 \
+                     -e WORKER_COUNT=1 -e RUN_MODE=map -v $PWD/ledgerexport:/ledgermeta -e TXMETA_SOURCE=file:///ledgermeta -v $PWD/index:/index -e INDEX_TARGET=file:///index stellar/lighthorizon-index-batch
+
+          # run reduce job
+          docker run -e NETWORK_PASSPHRASE='testnet' -e JOB_INDEX_ENV=AWS_BATCH_JOB_ARRAY_INDEX -e AWS_BATCH_JOB_ARRAY_INDEX=0 -e MAP_JOB_COUNT=1 -e REDUCE_JOB_COUNT=1 \
+                     -e WORKER_COUNT=1 -e RUN_MODE=reduce -v $PWD/index:/index -e INDEX_SOURCE_ROOT=file:///index -e INDEX_TARGET=file:///index stellar/lighthorizon-index-batch
+                     
       # Push images
       - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lighthorizon'
         name: Login to DockerHub

exp/lighthorizon/actions/accounts.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"github.com/stellar/go/support/log"
+	"github.com/stellar/go/xdr"
 
 	"github.com/stellar/go/exp/lighthorizon/adapters"
 	"github.com/stellar/go/exp/lighthorizon/services"
@@ -72,9 +73,10 @@ func NewTXByAccountHandler(lightHorizon services.LightHorizon) func(http.Respons
 			return
 		}
 
+		encoder := xdr.NewEncodingBuffer()
 		for _, txn := range txns {
 			var response hProtocol.Transaction
-			response, err = adapters.PopulateTransaction(r, &txn)
+			response, err = adapters.PopulateTransaction(r.URL, &txn, encoder)
 			if err != nil {
 				log.Error(err)
 				sendErrorResponse(r.Context(), w, supportProblem.ServerError)

exp/lighthorizon/adapters/testdata/transactions.json

@@ -0,0 +1,67 @@
+{
+ "_links": {
+  "self": {
+   "href": "https://horizon.stellar.org/accounts/GBFHFINUD6NVGSX33PY25DDRCABN3H2JTDMLUEXAUEJVV22HTXVGLEZD/transactions?cursor=179530990183178241\u0026limit=1\u0026order=desc"
+  },
+  "next": {
+   "href": "https://horizon.stellar.org/accounts/GBFHFINUD6NVGSX33PY25DDRCABN3H2JTDMLUEXAUEJVV22HTXVGLEZD/transactions?cursor=179530990183174144\u0026limit=1\u0026order=desc"
+  },
+  "prev": {
+   "href": "https://horizon.stellar.org/accounts/GBFHFINUD6NVGSX33PY25DDRCABN3H2JTDMLUEXAUEJVV22HTXVGLEZD/transactions?cursor=179530990183174144\u0026limit=1\u0026order=asc"
+  }
+ },
+ "_embedded": {
+  "records": [
+   {
+    "_links": {
+     "self": {
+      "href": "https://horizon.stellar.org/transactions/55d8aa3693489ffc1d70b8ba33b8b5c012ec098f6f104383e3f090048488febd"
+     },
+     "account": {
+      "href": "https://horizon.stellar.org/accounts/GBFHFINUD6NVGSX33PY25DDRCABN3H2JTDMLUEXAUEJVV22HTXVGLEZD"
+     },
+     "ledger": {
+      "href": "https://horizon.stellar.org/ledgers/41800316"
+     },
+     "operations": {
+      "href": "https://horizon.stellar.org/transactions/55d8aa3693489ffc1d70b8ba33b8b5c012ec098f6f104383e3f090048488febd/operations{?cursor,limit,order}",
+      "templated": true
+     },
+     "effects": {
+      "href": "https://horizon.stellar.org/transactions/55d8aa3693489ffc1d70b8ba33b8b5c012ec098f6f104383e3f090048488febd/effects{?cursor,limit,order}",
+      "templated": true
+     },
+     "precedes": {
+      "href": "https://horizon.stellar.org/transactions?order=asc\u0026cursor=179530990183174144"
+     },
+     "succeeds": {
+      "href": "https://horizon.stellar.org/transactions?order=desc\u0026cursor=179530990183174144"
+     },
+     "transaction": {
+      "href": "https://horizon.stellar.org/transactions/55d8aa3693489ffc1d70b8ba33b8b5c012ec098f6f104383e3f090048488febd"
+     }
+    },
+    "id": "55d8aa3693489ffc1d70b8ba33b8b5c012ec098f6f104383e3f090048488febd",
+    "paging_token": "179530990183174144",
+    "successful": true,
+    "hash": "55d8aa3693489ffc1d70b8ba33b8b5c012ec098f6f104383e3f090048488febd",
+    "ledger": 41800316,
+    "created_at": "2022-07-17T13:08:41Z",
+    "source_account": "GBFHFINUD6NVGSX33PY25DDRCABN3H2JTDMLUEXAUEJVV22HTXVGLEZD",
+    "source_account_sequence": "172589382434294350",
+    "fee_account": "GBFHFINUD6NVGSX33PY25DDRCABN3H2JTDMLUEXAUEJVV22HTXVGLEZD",
+    "fee_charged": "100",
+    "max_fee": "100000",
+    "operation_count": 1,
+    "envelope_xdr": "AAAAAgAAAABKcqG0H5tTSvvb8a6McRAC3Z9JmNi6EuChE1rrR53qZQABhqACZSkhAAAKTgAAAAAAAAAAAAAAAQAAAAEAAAAASnKhtB+bU0r72/GujHEQAt2fSZjYuhLgoRNa60ed6mUAAAANAAAAAXlYTE0AAAAAIjbXcP4NPgFSGXXVz3rEhCtwldaxqddo0+mmMumZBr4AAAACVAvkAAAAAABKcqG0H5tTSvvb8a6McRAC3Z9JmNi6EuChE1rrR53qZQAAAAJEUklGVAAAAAAAAAAAAAAAvSOzPqUOGnDIcJOm7T85qDFRM0wfOVoubgkEPk95DZ0AAAEQvqAGdQAAAAEAAAAAAAAAAAAAAAFHneplAAAAQAVm9muIrK31Z+m2ZvhDYhtuoHcc/n+MO0DOaiQjfW+tsUNVCOw7foHiDRVLBdAHBZT+xxa3F+Ek9wQiKzxtQQM=",
+    "result_xdr": "AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAANAAAAAAAAAAIAAAABAAAAAPaTW9sBV2ja6yDUtPcpGpUrnVEHaTHC4I065TklIsguAAAAAD1H0goAAAAAAAAAAlQE8wsAAAABeVhMTQAAAAAiNtdw/g0+AVIZddXPesSEK3CV1rGp12jT6aYy6ZkGvgAAAAJUC+QAAAAAAgRnzllf8Sas0MUQlkxROsBgUzEoIN2XrYP9tlH5SINjAAAAAkRSSUZUAAAAAAAAAAAAAAC9I7M+pQ4acMhwk6btPzmoMVEzTB85Wi5uCQQ+T3kNnQAAARN/56NFAAAAAAAAAAJUBPMLAAAAAEpyobQfm1NK+9vxroxxEALdn0mY2LoS4KETWutHneplAAAAAkRSSUZUAAAAAAAAAAAAAAC9I7M+pQ4acMhwk6btPzmoMVEzTB85Wi5uCQQ+T3kNnQAAARN/56NFAAAAAA==",
+    "result_meta_xdr": "AAAAAgAAAAIAAAADAn3SfAAAAAAAAAAASnKhtB+bU0r72/GujHEQAt2fSZjYuhLgoRNa60ed6mUAAAAAENitnwJlKSEAAApNAAAACgAAAAEAAAAAxHHGQ3BiyVBqiTQuU4oa2kBNL0HPHTolX0Mh98bg4XUAAAAAAAAACWxvYnN0ci5jbwAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAACfdJjAAAAAGLUCUkAAAAAAAAAAQJ90nwAAAAAAAAAAEpyobQfm1NK+9vxroxxEALdn0mY2LoS4KETWutHneplAAAAABDYrZ8CZSkhAAAKTgAAAAoAAAABAAAAAMRxxkNwYslQaok0LlOKGtpATS9Bzx06JV9DIffG4OF1AAAAAAAAAAlsb2JzdHIuY28AAAABAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAn3SfAAAAABi1AnZAAAAAAAAAAEAAAAMAAAAAwJ90nwAAAAAAAAAAPaTW9sBV2ja6yDUtPcpGpUrnVEHaTHC4I065TklIsguAAAAPP5dpSACFip8AC7CtgAAAAcAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAEAAAAc2nqv7AAAADbUPYzLAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAn3SegAAAABi1AnNAAAAAAAAAAECfdJ8AAAAAAAAAAD2k1vbAVdo2usg1LT3KRqVK51RB2kxwuCNOuU5JSLILgAAADqqWLIVAhYqfAAuwrYAAAAHAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAABAAAAHNp6r+wAAAA0gDiZwAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAJ90noAAAAAYtQJzQAAAAAAAAADAn3SUAAAAAUEZ85ZX/EmrNDFEJZMUTrAYFMxKCDdl62D/bZR+UiDYwAAAAAAAAAAAAAAAkRSSUZUAAAAAAAAAAAAAAC9I7M+pQ4acMhwk6btPzmoMVEzTB85Wi5uCQQ+T3kNnQAAAB4AAAIEaTMNuAAA8H8XoYXHAAAUwrrMCE0AAAAAAAAAaAAAAAAAAAABAn3SfAAAAAUEZ85ZX/EmrNDFEJZMUTrAYFMxKCDdl62D/bZR+UiDYwAAAAAAAAAAAAAAAkRSSUZUAAAAAAAAAAAAAAC9I7M+pQ4acMhwk6btPzmoMVEzTB85Wi5uCQQ+T3kNnQAAAB4AAAIGvTgAwwAA72uXueKCAAAUwrrMCE0AAAAAAAAAaAAAAAAAAAADAn3SYwAAAAEAAAAASnKhtB+bU0r72/GujHEQAt2fSZjYuhLgoRNa60ed6mUAAAACRFJJRlQAAAAAAAAAAAAAAL0jsz6lDhpwyHCTpu0/OagxUTNMHzlaLm4JBD5PeQ2dAAAAAAAAAAB//////////wAAAAEAAAAAAAAAAAAAAAECfdJ8AAAAAQAAAABKcqG0H5tTSvvb8a6McRAC3Z9JmNi6EuChE1rrR53qZQAAAAJEUklGVAAAAAAAAAAAAAAAvSOzPqUOGnDIcJOm7T85qDFRM0wfOVoubgkEPk95DZ0AAAETf+ejRX//////////AAAAAQAAAAAAAAAAAAAAAwJ90nwAAAABAAAAAPaTW9sBV2ja6yDUtPcpGpUrnVEHaTHC4I065TklIsguAAAAAXlYTE0AAAAAIjbXcP4NPgFSGXXVz3rEhCtwldaxqddo0+mmMumZBr4AAAAggUA/Y3//////////AAAAAQAAAAEAAAA21OED/gAAABzamxRcAAAAAAAAAAAAAAABAn3SfAAAAAEAAAAA9pNb2wFXaNrrINS09ykalSudUQdpMcLgjTrlOSUiyC4AAAABeVhMTQAAAAAiNtdw/g0+AVIZddXPesSEK3CV1rGp12jT6aYy6ZkGvgAAACLVTCNjf/////////8AAAABAAAAAQAAADSA1R//AAAAHNqbFFwAAAAAAAAAAAAAAAMCfdJ8AAAAAgAAAAD2k1vbAVdo2usg1LT3KRqVK51RB2kxwuCNOuU5JSLILgAAAAA9R9IKAAAAAAAAAAF5WExNAAAAACI213D+DT4BUhl11c96xIQrcJXWsanXaNPppjLpmQa+AAAANtQ9jMt7hXu5e4QLegAAAAAAAAAAAAAAAAAAAAECfdJ8AAAAAgAAAAD2k1vbAVdo2usg1LT3KRqVK51RB2kxwuCNOuU5JSLILgAAAAA9R9IKAAAAAAAAAAF5WExNAAAAACI213D+DT4BUhl11c96xIQrcJXWsanXaNPppjLpmQa+AAAANIA4mcB7hXu5e4QLegAAAAAAAAAAAAAAAAAAAAMCfcCZAAAAAQAAAABKcqG0H5tTSvvb8a6McRAC3Z9JmNi6EuChE1rrR53qZQAAAAF5WExNAAAAACI213D+DT4BUhl11c96xIQrcJXWsanXaNPppjLpmQa+AAAAEqEyDdl//////////wAAAAEAAAAAAAAAAAAAAAECfdJ8AAAAAQAAAABKcqG0H5tTSvvb8a6McRAC3Z9JmNi6EuChE1rrR53qZQAAAAF5WExNAAAAACI213D+DT4BUhl11c96xIQrcJXWsanXaNPppjLpmQa+AAAAEE0mKdl//////////wAAAAEAAAAAAAAAAAAAAAA=",
+    "fee_meta_xdr": "AAAAAgAAAAMCfdJjAAAAAAAAAABKcqG0H5tTSvvb8a6McRAC3Z9JmNi6EuChE1rrR53qZQAAAAAQ2K4DAmUpIQAACk0AAAAKAAAAAQAAAADEccZDcGLJUGqJNC5TihraQE0vQc8dOiVfQyH3xuDhdQAAAAAAAAAJbG9ic3RyLmNvAAAAAQAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAADAAAAAAJ90mMAAAAAYtQJSQAAAAAAAAABAn3SfAAAAAAAAAAASnKhtB+bU0r72/GujHEQAt2fSZjYuhLgoRNa60ed6mUAAAAAENitnwJlKSEAAApNAAAACgAAAAEAAAAAxHHGQ3BiyVBqiTQuU4oa2kBNL0HPHTolX0Mh98bg4XUAAAAAAAAACWxvYnN0ci5jbwAAAAEAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAACfdJjAAAAAGLUCUkAAAAA",
+    "memo_type": "none",
+    "signatures": [
+     "BWb2a4isrfVn6bZm+ENiG26gdxz+f4w7QM5qJCN9b62xQ1UI7Dt+geINFUsF0AcFlP7HFrcX4ST3BCIrPG1BAw=="
+    ]
+   }
+  ]
+ }
+}