Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci(portal loop): Create exports from portal loop #21

Merged
merged 10 commits into from
Oct 12, 2024
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions .github/workflows/portal-loop-txs-exporter.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Backup Portal Loop

on:
# allow to run workflow manually
workflow_dispatch: { }

# Triggers the workflow every hour
schedule:
- cron: "0 * * * *"

jobs:
backup:
name: "backup ${{ matrix.testnet }}"
runs-on: ubuntu-latest
timeout-minutes: 360 # very high; but it can take a while.

permissions:
contents: write

strategy:
fail-fast: false
max-parallel: 1
matrix:
testnet:
- portal-loop

steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.22.x"

- name: Run backup script
run: |
cd portal-loop
bash export.sh

- name: Run stats script
run: make -C ${{ matrix.testnet }} stats-legacy

- name: Run extractor
run: make -C ${{ matrix.testnet }} extractor-legacy

- uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: "chore: update ${{ matrix.testnet }} backup"
111 changes: 85 additions & 26 deletions extractor-0.1.1/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/gnolang/gno/gno.land/pkg/sdk/vm"
"github.com/gnolang/gno/tm2/pkg/amino"
"github.com/gnolang/gno/tm2/pkg/std"
"github.com/gnolang/tx-archive/types"
"github.com/peterbourgon/ff/v3/ffcli"
)
Expand All @@ -37,6 +38,8 @@ type extractorCfg struct {
fileType string
sourcePath string
outputDir string

legacyMode bool
}

func main() {
Expand Down Expand Up @@ -88,6 +91,13 @@ func (c *extractorCfg) registerFlags(fs *flag.FlagSet) {
"./extracted",
"the output directory for the extracted Gno source code",
)

fs.BoolVar(
&c.legacyMode,
"legacy-mode",
false,
"flag indicating if the legacy tx sheet mode should be used",
)
}

// execExtract runs the extract service for Gno source code
Expand Down Expand Up @@ -129,40 +139,83 @@ func execExtract(ctx context.Context, cfg *extractorCfg) error {
return errNoSourceFilesFound
}

for _, sourceFile := range sourceFiles {
sourceFile := sourceFile
var (
unwrapFn = func(data types.TxData) []std.Msg {
return data.Tx.Msgs
}

// Extract messages
msgs, processErr := extractAddMessages(sourceFile)
if processErr != nil {
return processErr
heightFn = func(data types.TxData) uint64 {
return data.BlockNum
}

// Process messages
for _, msg := range msgs {
outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/"))
unwrapLegacyFn = func(tx std.Tx) []std.Msg {
return tx.Msgs
}

if st, err := os.Stat(outputDir); err == nil && st.IsDir() {
outputDir += ":" + strconv.FormatUint(msg.Height, 10)
}
heightLegacyFn = func(_ std.Tx) uint64 {
return 0
}
)

// Write dir before writing files
if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil {
return fmt.Errorf("unable to write dir, %w", dirWriteErr)
for _, sourceFile := range sourceFiles {
select {
case <-ctx.Done():
return ctx.Err()
default:
sourceFile := sourceFile

// Extract messages
var (
msgs []AddPackage
processErr error
)

if !cfg.legacyMode {
msgs, processErr = extractAddMessages(
sourceFile,
unwrapFn,
heightFn,
)
} else {
msgs, processErr = extractAddMessages(
sourceFile,
unwrapLegacyFn,
heightLegacyFn,
)
}

// Write the package source code
if writeErr := writePackageFiles(msg, outputDir); writeErr != nil {
return writeErr
if processErr != nil {
return processErr
}

// Write the package metadata
if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil {
return writeErr
// Process messages
for _, msg := range msgs {
outputDir := filepath.Join(cfg.outputDir, strings.TrimLeft(msg.Package.Path, "gno.land/"))

if !cfg.legacyMode {
if st, err := os.Stat(outputDir); err == nil && st.IsDir() {
outputDir += ":" + strconv.FormatUint(msg.Height, 10)
}
}

// Write dir before writing files
if dirWriteErr := os.MkdirAll(outputDir, os.ModePerm); dirWriteErr != nil {
return fmt.Errorf("unable to write dir, %w", dirWriteErr)
}

// Write the package source code
if writeErr := writePackageFiles(msg, outputDir); writeErr != nil {
return writeErr
}

// Write the package metadata
if writeErr := writePackageMetadata(metadataFromMsg(msg), outputDir); writeErr != nil {
return writeErr
}
}
}

}

return nil
}

Expand Down Expand Up @@ -204,7 +257,12 @@ type AddPackage struct {
Height uint64
}

func extractAddMessages(filePath string) ([]AddPackage, error) {
// extractAddMessages extracts the AddPackage messages
func extractAddMessages[T std.Tx | types.TxData](
filePath string,
unwrapFn func(T) []std.Msg,
heightFn func(T) uint64,
) ([]AddPackage, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("unable to open file, %w", err)
Expand All @@ -226,7 +284,8 @@ func extractAddMessages(filePath string) ([]AddPackage, error) {
tempBuf := make([]byte, 0)

for {
var txData types.TxData
var txData T

line, isPrefix, err := reader.ReadLine()

// Exit if no more lines in file
Expand Down Expand Up @@ -262,7 +321,7 @@ func extractAddMessages(filePath string) ([]AddPackage, error) {
tempBuf = nil
}

for _, msg := range txData.Tx.Msgs {
for _, msg := range unwrapFn(txData) {
// Only MsgAddPkg should be parsed
if msg.Type() != "add_package" {
continue
Expand All @@ -279,7 +338,7 @@ func extractAddMessages(filePath string) ([]AddPackage, error) {

msgArr = append(msgArr, AddPackage{
MsgAddPackage: msgAddPkg,
Height: txData.BlockNum,
Height: heightFn(txData),
})
}
}
Expand Down
6 changes: 6 additions & 0 deletions portal-loop/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
EXTRACTOR_DIR = extractor-0.1.1
REMOTE = "https://rpc.gno.land"
SHORTNAME = portal-loop
LOOP_DURATION = 50000

-include ../rules.mk
129 changes: 129 additions & 0 deletions portal-loop/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
# https://rpc.gno.land

## TXs
```
5542
```

## addpkgs
```
```

## top realm calls
```
1920 "gno.land/r/portal/counter"
220 "gno.land/r/gnoland/blog"
108 "gno.land/r/demo/teritori/social_feeds"
106 "gno.land/r/demo/memeland"
68 "gno.land/r/demo/postit/v1"
60 "gno.land/r/demo/users"
48 "gno.land/r/demo/userbook"
37 "gno.land/r/gnome/dao/v1dev1"
36 "gno.land/r/demo/wugnot"
32 "gno.land/r/demo/staging/memeland/v1"
30 "gno.land/r/demo/boards"
24 "gno.land/r/demo/event_emitter"
24 "gno.land/r/gnome/dao/v1dev0"
23 "gno.land/r/gnoland/events"
23 "gno.land/r/gnome/dao/pre1"
19 "gno.land/r/gnome/dao/v1pre2"
16 "gno.land/r/gnome/dao/v1pre3"
15 "gno.land/r/gc24/raffle"
15 "gno.land/r/gnome/dao/v1pre1"
15 "gno.land/r/jeronimoalbi/testpoll"
14 "gno.land/r/g1lyzcpa7duh69lk04nahxup484xrz4k6k2nqdun/gnome/dao/v1"
13 "gno.land/r/demo/staging/memeland/v2"
11 "gno.land/r/mikecito/gh_test_4"
11 "gno.land/r/varmeta/vmt721"
10 "gno.land/r/gnome/dao/v1dev01"
10 "gno.land/r/test321/hello"
9 "gno.land/r/albttx/home"
8 "gno.land/r/demo/postit"
8 "gno.land/r/gnome/dao/v1pre4"
8 "gno.land/r/leon/test/avltestv1"
7 "gno.land/r/leon/v2/raffle"
7 "gno.land/r/malek/todolist1"
6 "gno.land/r/gnome/dao/pre2"
6 "gno.land/r/leon/staging/v3/raffle"
6 "gno.land/r/mikecito/gh_test_3"
6 "gno.land/r/test2/hello"
5 "gno.land/r/demo/foo20"
5 "gno.land/r/demo/restest"
5 "gno.land/r/demo/test_event"
5 "gno.land/r/leon/test/test1/whitelistfactory"
5 "gno.land/r/leon/tokens/leongrc20"
5 "gno.land/r/mikecito/gh_test_2"
5 "gno.land/r/teritori/social_feeds"
4 "gno.land/r/michelle37/testpoll"
4 "gno.land/r/reza/rezablog"
4 "gno.land/r/test/counter"
3 "gno.land/r/boom/boom"
3 "gno.land/r/demo/banktest"
3 "gno.land/r/demo/hello001"
3 "gno.land/r/demo/mikecito/social_feeds"
3 "gno.land/r/demo/nft2"
3 "gno.land/r/gnoland/faucet"
3 "gno.land/r/leon/demo/poll/v2"
3 "gno.land/r/leon/grc/v1/leon200"
3 "gno.land/r/leon/staging/v4/raffle"
3 "gno.land/r/leon/test/avltest"
3 "gno.land/r/leon/testing/counter3"
3 "gno.land/r/leon/v5/memeland"
3 "gno.land/r/mikecito/gh_test_1"
2 "gno.land/r/demo/event"
2 "gno.land/r/demo/hi001"
2 "gno.land/r/demo/morgan/variadic"
2 "gno.land/r/gnome/tutorials/pre1"
2 "gno.land/r/hehe321/hehe321"
2 "gno.land/r/hehe321/hehe3211"
2 "gno.land/r/hehe321/hehe3212"
2 "gno.land/r/jeronimoalbi2/testpoll"
2 "gno.land/r/leon/counter/e/hello"
2 "gno.land/r/leon/demo/poll"
2 "gno.land/r/leon/testing/counter2"
2 "gno.land/r/leon/v1/raffle"
2 "gno.land/r/mikaelvallenet/mvc"
2 "gno.land/r/test3/hello"
2 "gno.land/r/test4/hello"
1 "gno.land/r/deelawn/tsrf"
1 "gno.land/r/demo/abcitest123"
1 "gno.land/r/demo/art/gnoface"
1 "gno.land/r/demo/counter"
1 "gno.land/r/demo/emit_main"
1 "gno.land/r/demo/faucet"
1 "gno.land/r/demo/grc20factory"
1 "gno.land/r/demo/hello"
1 "gno.land/r/demo/microblog"
1 "gno.land/r/demo/tamagotchi"
1 "gno.land/r/demo/test_event2"
1 "gno.land/r/demoyy/counter"
1 "gno.land/r/foo/event"
1 "gno.land/r/gnoland/ghverify"
1 "gno.land/r/gophercon/mood"
1 "gno.land/r/hehe3212/hehe3212"
1 "gno.land/r/hello/event"
1 "gno.land/r/hello/event_emitter"
1 "gno.land/r/iamkevin/kraffle5"
1 "gno.land/r/leon/demo/poll/v1"
1 "gno.land/r/leon/grc/v1/leon20"
1 "gno.land/r/leon/test/realm/hello"
1 "gno.land/r/leon/test/realms"
1 "gno.land/r/leon/test/realms2"
1 "gno.land/r/leon/test/todolist"
1 "gno.land/r/leon/v1/memeland"
1 "gno.land/r/manfred/home"
1 "gno.land/r/mikaelvallenet/path"
1 "gno.land/r/morgan/guestbook"
1 "gno.land/r/nebular24/guessbook"
1 "gno.land/r/stuyk/mood"
1 "gno.land/r/test/hello"
1 "gno.land/r/test10/poll"
1 "gno.land/r/test5/hello"
1 "gno.land/r/varmeta/demo/v1/domain/registrar"
1 "gno.land/r/whisky/goraffle"
```

## top faucet requesters
```
```

Loading