chore(ci): daily ci backup of test2 and test3 networks

.github/workflows/txs-exporter.yml

@@ -0,0 +1,42 @@
+name: Backup testnets
+
+on:
+  # allow to run workflow manually
+  workflow_dispatch: {}
+
+  # Triggers the workflow every day at 1am
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  backup:
+    name: "backup ${{ matrix.testnet }}"
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    permissions:
+      contents: write
+
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        testnet:
+          - test2.gno.land
+          - test3.gno.land
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run backup script
+        run: make -C ${{ matrix.testnet }} fetch
+
+      - name: Run stats script
+        run: make -C ${{ matrix.testnet }} stats
+
+      - name: Run git pull
+        run: git pull
+
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore: update ${{ matrix.testnet }} backup"

deps.go

@@ -3,4 +3,4 @@
 
 package deps
 
-import _ "github.com/gnolang/gno/gno.land/cmd/gnotxsync"
+import _ "github.com/gnolang/tx-archive/cmd"

go.mod

@@ -1,8 +1,11 @@
 module github.com/gnolang/txports
 
-go 1.21.0
+go 1.20
 
-require github.com/gnolang/gno v0.0.0-20231002124245-28455d8ffc5b
+require (
+	github.com/gnolang/gno v0.0.0-20231112174927-b1a53c018ea4 // indirect
+	github.com/gnolang/tx-archive v0.1.1
+)
 
 require (
 	github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c // indirect
@@ -22,7 +25,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
@@ -32,15 +35,14 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rs/cors v1.10.1 // indirect
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
+	go.etcd.io/bbolt v1.3.8 // indirect
 	go.opencensus.io v0.22.5 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/term v0.12.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )

rules.mk

@@ -1,35 +1,47 @@
+
 fetch:
-	go run github.com/gnolang/gno/gno.land/cmd/gnotxsync export --remote $(REMOTE)
-	wc -l txexport.log
-	tail -n 1 txexport.log
-	rm -f txexport-*.log
-	split --lines=1000 --additional-suffix=.log txexport.log txexport-
+	@echo "Backup from: $(FROM_BLOCK) to $(TO_BLOCK)"
+	go run github.com/gnolang/tx-archive/cmd backup \
+		--remote $(REMOTE) \
+		--from-block $(FROM_BLOCK) \
+		--to-block   $(TO_BLOCK) \
+		--output-path backup_$(shell printf '%07d' $(FROM_BLOCK))-$(shell printf '%07d' $(TO_BLOCK)).jsonl
+
+    # Update metadata
+	@cat metadata.json | jq -a '.latest_block_height = $(TO_BLOCK)' > /tmp/aa.json
+	@mv /tmp/aa.json metadata.json
+
+
+fetch-all:
+	@for i in `seq $(FROM_BLOCK) $(MAX_INTERVAL) $(LATEST_BLOCK_HEIGHT)`; do \
+		make -C . fetch FROM_BLOCK="$$i"; \
+	done
 
 stats:
 	echo "# $(REMOTE)" > README.md
 	echo >> README.md
 
 	echo "## TXs" >> README.md
 	echo '```' >> README.md
-	cat txexport-*.log | wc -l >> README.md
+	cat backup_*.jsonl | wc -l >> README.md
 	echo '```' >> README.md
 	echo >> README.md
 
 	echo "## addpkgs" >> README.md
 	echo '```' >> README.md
-	cat txexport-*.log | jq '.msg[].package.Path' | sort | uniq -c | sort -n >> README.md
+	cat backup_*.jsonl | jq '.msg[].package.Path' | sort | uniq -c | sort -n >> README.md
 	echo '```' >> README.md
 	echo >> README.md
 
 	echo "## top realm calls" >> README.md
 	echo '```' >> README.md
-	cat txexport-*.log | jq '.msg[].pkg_path' | sort | uniq -c | sort -n >> README.md
+	cat backup_*.jsonl | jq '.msg[].pkg_path' | sort | uniq -c | sort -n >> README.md
 	echo '```' >> README.md
 	echo >> README.md
 
 	echo "## top faucet requesters" >> README.md
 	echo '```' >> README.md
-	cat txexport*.log | jq -r '.msg[] | select(.["@type"]=="/bank.MsgSend") | select(.["from_address"]=="g127jydsh6cms3lrtdenydxsckh23a8d6emqcvfa") | .to_address + " " + .amount' | sed 's/ugnot$$//' | awk 'NR == 1 {next} {a[$$1] += $$2} {b[$$1] += 1} END {for (i in a) {if (a[i] >= 500000000){printf "%-15s\t%s\t%s\n", i, b[i], a[i]}}}' | sort -rnk2 >> README.md
+	cat backup_*.jsonl | jq -r '.msg[] | select(.["@type"]=="/bank.MsgSend") | select(.["from_address"]=="g127jydsh6cms3lrtdenydxsckh23a8d6emqcvfa") | .to_address + " " + .amount' | sed 's/ugnot$$//' | awk 'NR == 1 {next} {a[$$1] += $$2} {b[$$1] += 1} END {for (i in a) {if (a[i] >= 500000000){printf "%-15s\t%s\t%s\n", i, b[i], a[i]}}}' | sort -rnk2 >> README.md
 	echo '```' >> README.md
 	echo >> README.md
 

test1.gno.land/Makefile

@@ -2,4 +2,13 @@ REMOTE = test1.gno.land:36657
 SHORTNAME = test1
 LOOP_DURATION = 50000
 
+
+FROM_BLOCK = $(shell cat metadata.json | jq -r '.latest_block_height')
+LATEST_BLOCK_HEIGHT = $(shell curl -s $(REMOTE)/status | jq -r '.result.sync_info.latest_block_height')
+TO_BLOCK = $(LATEST_BLOCK_HEIGHT)
+
+ifeq ($(shell echo "$(TO_BLOCK) - $(FROM_BLOCK) > $(MAX_INTERVAL)" | bc ), 1)
+	TO_BLOCK = $(shell echo "$(FROM_BLOCK) + $(MAX_INTERVAL)" | bc)
+endif
+
 -include ../rules.mk

test1.gno.land/metadata.json

@@ -0,0 +1 @@
+{ "latest_block_height": 1 }

test2.gno.land/Makefile

@@ -2,4 +2,14 @@ REMOTE = test2.gno.land:36657
 SHORTNAME = test2
 LOOP_DURATION = 50000
 
+MAX_INTERVAL = 10000
+
+FROM_BLOCK = $(shell cat metadata.json | jq -r '.latest_block_height')
+LATEST_BLOCK_HEIGHT = $(shell curl -s $(REMOTE)/status | jq -r '.result.sync_info.latest_block_height')
+TO_BLOCK = $(LATEST_BLOCK_HEIGHT)
+
+ifeq ($(shell echo "$(TO_BLOCK) - $(FROM_BLOCK) > $(MAX_INTERVAL)" | bc ), 1)
+	TO_BLOCK = $(shell echo "$(FROM_BLOCK) + $(MAX_INTERVAL)" | bc)
+endif
+
 -include ../rules.mk

test2.gno.land/README.md

@@ -2,7 +2,7 @@
 
 ## TXs
 ```
-36890
+36968
 ```
 
 ## addpkgs