From 1b460d6435dc9e67caedc054577a5853782ff9a3 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 27 Apr 2020 18:14:18 +0800 Subject: [PATCH 1/7] add new utility --- beacon-chain/core/state/BUILD.bazel | 3 +- beacon-chain/state/BUILD.bazel | 1 + beacon-chain/state/stateutil/BUILD.bazel | 1 + tools/pcli/BUILD.bazel | 71 ++++++++++++ tools/pcli/main.go | 142 +++++++++++++++++++++++ 5 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tools/pcli/BUILD.bazel create mode 100644 tools/pcli/main.go diff --git a/beacon-chain/core/state/BUILD.bazel b/beacon-chain/core/state/BUILD.bazel index 4deb3c76a59d..22d18db1ad56 100644 --- a/beacon-chain/core/state/BUILD.bazel +++ b/beacon-chain/core/state/BUILD.bazel @@ -12,11 +12,12 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state", visibility = [ "//beacon-chain:__subpackages__", + "//endtoend:__pkg__", "//shared/interop:__pkg__", "//shared/testutil:__pkg__", "//tools/benchmark-files-gen:__pkg__", "//tools/genesis-state-gen:__pkg__", - "//endtoend:__pkg__", + "//tools/pcli:__pkg__", ], deps = [ "//beacon-chain/cache:go_default_library", diff --git a/beacon-chain/state/BUILD.bazel b/beacon-chain/state/BUILD.bazel index 5a50640d2288..5fa378a0599d 100644 --- a/beacon-chain/state/BUILD.bazel +++ b/beacon-chain/state/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//shared/benchutil:__pkg__", "//shared/testutil:__pkg__", "//tools/benchmark-files-gen:__pkg__", + "//tools/pcli:__pkg__", ], deps = [ "//beacon-chain/core/state/stateutils:go_default_library", diff --git a/beacon-chain/state/stateutil/BUILD.bazel b/beacon-chain/state/stateutil/BUILD.bazel index 544fa1e5728a..6d88b7795426 100644 --- a/beacon-chain/state/stateutil/BUILD.bazel +++ b/beacon-chain/state/stateutil/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "//beacon-chain:__subpackages__", "//proto/testing:__subpackages__", "//shared/testutil:__subpackages__", + "//tools/pcli:__pkg__", ], deps = [ "//proto/beacon/p2p/v1:go_default_library", diff --git a/tools/pcli/BUILD.bazel b/tools/pcli/BUILD.bazel new file mode 100644 index 000000000000..648fc8735c50 --- /dev/null +++ b/tools/pcli/BUILD.bazel @@ -0,0 +1,71 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("@io_bazel_rules_docker//go:image.bzl", "go_image") +load("@io_bazel_rules_docker//container:container.bzl", "container_bundle") +load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/prysmaticlabs/prysm/tools/pcli", + visibility = ["//visibility:private"], + deps = [ + "//beacon-chain/core/state:go_default_library", + "//beacon-chain/state:go_default_library", + "//beacon-chain/state/stateutil:go_default_library", + "//proto/beacon/p2p/v1:go_default_library", + "//shared/version:go_default_library", + "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", + "@com_github_prysmaticlabs_go_ssz//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library", + "@in_gopkg_d4l3k_messagediff_v1//:go_default_library", + "@in_gopkg_urfave_cli_v2//:go_default_library", + ], +) + +go_image( + name = "image", + srcs = ["main.go"], + base = "//tools:go_image", + goarch = "amd64", + goos = "linux", + importpath = "github.com/prysmaticlabs/prysm/tools/pcli", + pure = "on", + race = "off", + tags = ["manual"], + visibility = ["//visibility:private"], + deps = [ + "//beacon-chain/core/state:go_default_library", + "//beacon-chain/state:go_default_library", + "//beacon-chain/state/stateutil:go_default_library", + "//proto/beacon/p2p/v1:go_default_library", + "//shared/version:go_default_library", + "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", + "@com_github_prysmaticlabs_go_ssz//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library", + "@in_gopkg_d4l3k_messagediff_v1//:go_default_library", + "@in_gopkg_urfave_cli_v2//:go_default_library", + ], +) + +go_binary( + name = "pcli", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +container_bundle( + name = "image_bundle", + images = { + "gcr.io/prysmaticlabs/prysm/pcli:latest": ":image", + "gcr.io/prysmaticlabs/prysm/pcli:{DOCKER_TAG}": ":image", + }, + tags = ["manual"], +) + +docker_push( + name = "push_images", + bundle = ":image_bundle", + tags = ["manual"], +) diff --git a/tools/pcli/main.go b/tools/pcli/main.go new file mode 100644 index 000000000000..2cc4d40763d3 --- /dev/null +++ b/tools/pcli/main.go @@ -0,0 +1,142 @@ +package main + +import ( + "bufio" + "context" + "io/ioutil" + "os" + "strings" + + ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" + "github.com/prysmaticlabs/go-ssz" + "github.com/prysmaticlabs/prysm/beacon-chain/core/state" + stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" + "github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil" + pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/version" + log "github.com/sirupsen/logrus" + prefixed "github.com/x-cray/logrus-prefixed-formatter" + "gopkg.in/d4l3k/messagediff.v1" + "gopkg.in/urfave/cli.v2" +) + +func main() { + var blockPath string + var preStatePath string + var expectedPostStatePath string + + customFormatter := new(prefixed.TextFormatter) + customFormatter.TimestampFormat = "2006-01-02 15:04:05" + customFormatter.FullTimestamp = true + log.SetFormatter(customFormatter) + + app := cli.App{} + app.Name = "pcli" + app.Usage = "A command line utility to run eth2 specific actions" + app.Version = version.GetVersion() + app.Commands = []*cli.Command{{ + Name: "state-transition", + Category: "state-transition", + Usage: "Subcommand to run manual state transitions", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "blockPath", + Usage: "Path to block file(ssz)", + Destination: &blockPath, + }, + &cli.StringFlag{ + Name: "preStatePath", + Usage: "Path to pre state file(ssz)", + Destination: &preStatePath, + }, + &cli.StringFlag{ + Name: "expectedPostStatePath", + Usage: "Path to expected post state file(ssz)", + Destination: &expectedPostStatePath, + }, + }, + Action: func(c *cli.Context) error { + if blockPath == "" { + log.Info("Block path not provided for state transition. " + + "Please provide path") + reader := bufio.NewReader(os.Stdin) + text, err := reader.ReadString('\n') + if err != nil { + log.Fatal(err) + } + if text = strings.Replace(text, "\n", "", -1); text == "" { + log.Fatal("Empty block path given") + } + blockPath = text + } + block := ðpb.SignedBeaconBlock{} + if err := dataFetcher(blockPath, block); err != nil { + log.Fatal(err) + } + blkRoot, err := stateutil.BlockRoot(block.Block) + if err != nil { + log.Fatal(err) + } + if preStatePath == "" { + log.Info("Pre State path not provided for state transition. " + + "Please provide path") + reader := bufio.NewReader(os.Stdin) + text, err := reader.ReadString('\n') + if err != nil { + log.Fatal(err) + } + if text = strings.Replace(text, "\n", "", -1); text == "" { + log.Fatal("Empty state path given") + } + preStatePath = text + } + preState := &pb.BeaconState{} + if err := dataFetcher(preStatePath, preState); err != nil { + log.Fatal(err) + } + stateObj, err := stateTrie.InitializeFromProto(preState) + if err != nil { + log.Fatal(err) + } + preStateRoot, err := stateObj.HashTreeRoot(context.Background()) + if err != nil { + log.Fatal(err) + } + log.Infof("Performing State Transition with a block root of %#x and pre state root of %#x", + blkRoot, preStateRoot) + postState, err := state.ExecuteStateTransition(context.Background(), stateObj, block) + if err != nil { + log.Fatal(err) + } + postRoot, err := postState.HashTreeRoot(context.Background()) + log.Infof("Finished Performing State Transition with Post state root of %#x", postRoot) + + // diff the state if a post state is provided + if expectedPostStatePath != "" { + expectedState := &pb.BeaconState{} + if err := dataFetcher(expectedPostStatePath, expectedState); err != nil { + log.Fatal(err) + } + if !ssz.DeepEqual(expectedState, postState.InnerStateUnsafe()) { + diff, _ := messagediff.PrettyDiff(expectedState, postState.InnerStateUnsafe()) + log.Errorf("Derived state differs from provided post state: %s", diff) + } + } + return nil + }, + }, + } + if err := app.Run(os.Args); err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// fetch data and unmarshal to provided data structure from file +func dataFetcher(fPath string, data interface{}) error { + rawFile, err := ioutil.ReadFile(fPath) + if err != nil { + return err + } + return ssz.Unmarshal(rawFile, data) +} From e6376bae1f4fb5f46fd4ed6324eea54c3a704864 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Mon, 27 Apr 2020 18:56:59 +0800 Subject: [PATCH 2/7] Update tools/pcli/main.go Co-Authored-By: Victor Farazdagi --- tools/pcli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pcli/main.go b/tools/pcli/main.go index 2cc4d40763d3..42ddbdfc65c0 100644 --- a/tools/pcli/main.go +++ b/tools/pcli/main.go @@ -111,7 +111,7 @@ func main() { postRoot, err := postState.HashTreeRoot(context.Background()) log.Infof("Finished Performing State Transition with Post state root of %#x", postRoot) - // diff the state if a post state is provided + // Diff the state if a post state is provided. if expectedPostStatePath != "" { expectedState := &pb.BeaconState{} if err := dataFetcher(expectedPostStatePath, expectedState); err != nil { From ec03634e9b294f7374ef06914ba639dbc57a0fc3 Mon Sep 17 00:00:00 2001 From: Nishant Das Date: Mon, 27 Apr 2020 18:57:08 +0800 Subject: [PATCH 3/7] Update tools/pcli/main.go Co-Authored-By: Victor Farazdagi --- tools/pcli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pcli/main.go b/tools/pcli/main.go index 42ddbdfc65c0..46679aeb0a48 100644 --- a/tools/pcli/main.go +++ b/tools/pcli/main.go @@ -132,7 +132,7 @@ func main() { } } -// fetch data and unmarshal to provided data structure from file +// dataFetcher fetches and unmarshals data from file to provided data structure. func dataFetcher(fPath string, data interface{}) error { rawFile, err := ioutil.ReadFile(fPath) if err != nil { From a661df7b3344a4dc661421faf0f6320cf8fc86b1 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 27 Apr 2020 20:12:22 +0800 Subject: [PATCH 4/7] readme and docker fix --- tools/pcli/BUILD.bazel | 3 +-- tools/pcli/README.md | 45 ++++++++++++++++++++++++++++++++++++++++++ tools/pcli/main.go | 2 +- 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 tools/pcli/README.md diff --git a/tools/pcli/BUILD.bazel b/tools/pcli/BUILD.bazel index 648fc8735c50..702103f31642 100644 --- a/tools/pcli/BUILD.bazel +++ b/tools/pcli/BUILD.bazel @@ -26,11 +26,10 @@ go_library( go_image( name = "image", srcs = ["main.go"], - base = "//tools:go_image", + base = "//tools:cc_image", goarch = "amd64", goos = "linux", importpath = "github.com/prysmaticlabs/prysm/tools/pcli", - pure = "on", race = "off", tags = ["manual"], visibility = ["//visibility:private"], diff --git a/tools/pcli/README.md b/tools/pcli/README.md new file mode 100644 index 000000000000..55c58eea38cf --- /dev/null +++ b/tools/pcli/README.md @@ -0,0 +1,45 @@ +## Pcli (Prysm CLI) + +This is a utility to help users perform eth2 specific actions. + +### Usage + +*Name:* + **pcli** - A command line utility to run eth2 specific actions + +*Usage:* + pcli [global options] command [command options] [arguments...] + +*Commands:* + help, h Shows a list of commands or help for one command + state-transition: + state-transition Subcommand to run manual state transitions + + +*Flags:* + --help, -h show help (default: false) + --version, -v print the version (default: false) + +*State Transition Subcommand:* + pcli state-transition - Subcommand to run manual state transitions + +*State Transition Usage:*: + pcli state-transition [command options] [arguments...] + + +*State Transition Flags:* + --blockPath value Path to block file(ssz) + --preStatePath value Path to pre state file(ssz) + --expectedPostStatePath value Path to expected post state file(ssz) + --help, -h show help (default: false) + + + +### Example + +To use pcli manual state transition: + +``` +bazel run //tools/pcli:pcli -- state-transition --blockPath /path/to/block --preStatePath /path/to/state +``` + diff --git a/tools/pcli/main.go b/tools/pcli/main.go index 46679aeb0a48..d296db5e2740 100644 --- a/tools/pcli/main.go +++ b/tools/pcli/main.go @@ -32,7 +32,7 @@ func main() { app := cli.App{} app.Name = "pcli" - app.Usage = "A command line utility to run eth2 specific actions" + app.Usage = "A command line utility to run eth2 specific commands" app.Version = version.GetVersion() app.Commands = []*cli.Command{{ Name: "state-transition", From ab44a1b7bc7181218432e4d8f3430a3fa7b80b81 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 27 Apr 2020 20:15:01 +0800 Subject: [PATCH 5/7] readme --- tools/pcli/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pcli/README.md b/tools/pcli/README.md index 55c58eea38cf..a1a5f76769c8 100644 --- a/tools/pcli/README.md +++ b/tools/pcli/README.md @@ -1,11 +1,11 @@ ## Pcli (Prysm CLI) -This is a utility to help users perform eth2 specific actions. +This is a utility to help users perform eth2 specific commands. ### Usage *Name:* - **pcli** - A command line utility to run eth2 specific actions + **pcli** - A command line utility to run eth2 specific commands *Usage:* pcli [global options] command [command options] [arguments...] From ebd006deedabec4ffe32ccb926f8268c5b644d62 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 27 Apr 2020 10:20:52 -0500 Subject: [PATCH 6/7] pending feedback --- tools/pcli/README.md | 8 ++++---- tools/pcli/main.go | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tools/pcli/README.md b/tools/pcli/README.md index a1a5f76769c8..6a4a461ce89e 100644 --- a/tools/pcli/README.md +++ b/tools/pcli/README.md @@ -28,9 +28,9 @@ This is a utility to help users perform eth2 specific commands. *State Transition Flags:* - --blockPath value Path to block file(ssz) - --preStatePath value Path to pre state file(ssz) - --expectedPostStatePath value Path to expected post state file(ssz) + --block-path value Path to block file(ssz) + --pre-state-patch value Path to pre state file(ssz) + --expected-post-state-path value Path to expected post state file(ssz) --help, -h show help (default: false) @@ -40,6 +40,6 @@ This is a utility to help users perform eth2 specific commands. To use pcli manual state transition: ``` -bazel run //tools/pcli:pcli -- state-transition --blockPath /path/to/block --preStatePath /path/to/state +bazel run //tools/pcli:pcli -- state-transition --block-path /path/to/block.ssz --pre-state-path /path/to/state.ssz ``` diff --git a/tools/pcli/main.go b/tools/pcli/main.go index d296db5e2740..acb9722047e5 100644 --- a/tools/pcli/main.go +++ b/tools/pcli/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "fmt" "io/ioutil" "os" "strings" @@ -40,17 +41,17 @@ func main() { Usage: "Subcommand to run manual state transitions", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "blockPath", + Name: "block-path", Usage: "Path to block file(ssz)", Destination: &blockPath, }, &cli.StringFlag{ - Name: "preStatePath", + Name: "pre-state-path", Usage: "Path to pre state file(ssz)", Destination: &preStatePath, }, &cli.StringFlag{ - Name: "expectedPostStatePath", + Name: "expected-post-state-path", Usage: "Path to expected post state file(ssz)", Destination: &expectedPostStatePath, }, @@ -102,14 +103,20 @@ func main() { if err != nil { log.Fatal(err) } - log.Infof("Performing State Transition with a block root of %#x and pre state root of %#x", - blkRoot, preStateRoot) + log.WithFields(logrus.Fields{ + "blockSlot": fmt.Sprintf("%d", block.Slot), + "preStateSlot": fmt.Sprintf("%d", stateObj.Slot()), + }).Infof( + "Performing state transition with a block root of %#x and pre state root of %#x", + blkRoot, + preStateRoot, + ) postState, err := state.ExecuteStateTransition(context.Background(), stateObj, block) if err != nil { log.Fatal(err) } postRoot, err := postState.HashTreeRoot(context.Background()) - log.Infof("Finished Performing State Transition with Post state root of %#x", postRoot) + log.Infof("Finished state transition with post state root of %#x", postRoot) // Diff the state if a post state is provided. if expectedPostStatePath != "" { From d22aafef1cfc7d7ed660ae9c9b1c6b888ad83857 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 27 Apr 2020 10:25:14 -0500 Subject: [PATCH 7/7] builds --- tools/pcli/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/pcli/main.go b/tools/pcli/main.go index acb9722047e5..7a80120da8a2 100644 --- a/tools/pcli/main.go +++ b/tools/pcli/main.go @@ -103,8 +103,8 @@ func main() { if err != nil { log.Fatal(err) } - log.WithFields(logrus.Fields{ - "blockSlot": fmt.Sprintf("%d", block.Slot), + log.WithFields(log.Fields{ + "blockSlot": fmt.Sprintf("%d", block.Block.Slot), "preStateSlot": fmt.Sprintf("%d", stateObj.Slot()), }).Infof( "Performing state transition with a block root of %#x and pre state root of %#x",