Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

shed command to unpack miner info dumps #5800

Merged
merged 3 commits into from
Mar 12, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/lotus-shed/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ func main() {
mpoolCmd,
genesisVerifyCmd,
mathCmd,
minerCmd,
mpoolStatsCmd,
exportChainCmd,
consensusCmd,
Expand Down
113 changes: 113 additions & 0 deletions cmd/lotus-shed/miner.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package main

import (
"bufio"
"io"
"os"
"path/filepath"
"strings"

"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
)

var minerCmd = &cli.Command{
Name: "miner",
Usage: "miner-related utilities",
Subcommands: []*cli.Command{
minerUnpackInfoCmd,
},
}

var minerUnpackInfoCmd = &cli.Command{
Name: "unpack-info",
Usage: "unpack miner info all dump",
ArgsUsage: "[allinfo.txt] [dir]",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: call dir output direcctory (i.e., make it clear that this is the output).

Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 2 {
return xerrors.Errorf("expected 2 args")
}

src, err := homedir.Expand(cctx.Args().Get(0))
if err != nil {
return xerrors.Errorf("expand src: %w", err)
}

f, err := os.Open(src)
if err != nil {
return xerrors.Errorf("open file: %w", err)
}
defer f.Close() // nolint

dest, err := homedir.Expand(cctx.Args().Get(1))
if err != nil {
return xerrors.Errorf("expand dest: %w", err)
}

var outf *os.File

r := bufio.NewReader(f)
for {
l, _, err := r.ReadLine()
if err == io.EOF {
if outf != nil {
return outf.Close()
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe return an explicit "we did nothing" error?

}
if err != nil {
return xerrors.Errorf("read line: %w", err)
}
sl := string(l)

if strings.HasPrefix(sl, "#") {
if strings.Contains(sl, "..") {
return xerrors.Errorf("bad name %s", sl)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we trying to prevent directory traversal? (maybe a comment)

}

if strings.HasPrefix(sl, "#: ") {
if outf != nil {
if err := outf.Close(); err != nil {
return xerrors.Errorf("close out file: %w", err)
}
}
p := filepath.Join(dest, sl[len("#: "):])
if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
return xerrors.Errorf("mkdir: %w", err)
}
outf, err = os.Create(p)
if err != nil {
return xerrors.Errorf("create out file: %w", err)
}
continue
}

if strings.HasPrefix(sl, "##: ") {
if outf != nil {
if err := outf.Close(); err != nil {
return xerrors.Errorf("close out file: %w", err)
}
}
p := filepath.Join(dest, "Per Sector Infos", sl[len("##: "):])
if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
return xerrors.Errorf("mkdir: %w", err)
}
outf, err = os.Create(p)
if err != nil {
return xerrors.Errorf("create out file: %w", err)
}
continue
}
}

if outf != nil {
if _, err := outf.Write(l); err != nil {
return xerrors.Errorf("write line: %w", err)
}
if _, err := outf.Write([]byte("\n")); err != nil {
return xerrors.Errorf("write line end: %w", err)
}
Comment on lines +104 to +109
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: fmt.Fprintln(outf, l) is a bit nicer.

}
}
},
}
40 changes: 20 additions & 20 deletions cmd/lotus-storage-miner/info_all.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,88 +35,88 @@ var infoAllCmd = &cli.Command{

fmt.Println("#: Version")
if err := lcli.VersionCmd.Action(cctx); err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to annotate these errors with what failed (even just a line number).

return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Miner Info")
if err := infoCmdAct(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

// Verbose info

fmt.Println("\n#: Storage List")
if err := storageListCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Worker List")
if err := sealingWorkersCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: PeerID")
if err := lcli.NetId.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Listen Addresses")
if err := lcli.NetListen.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Reachability")
if err := lcli.NetReachability.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

// Very Verbose info
fmt.Println("\n#: Peers")
if err := lcli.NetPeers.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Sealing Jobs")
if err := sealingJobsCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Sched Diag")
if err := sealingSchedDiagCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Storage Ask")
if err := getAskCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Storage Deals")
if err := dealsListCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Retrieval Deals")
if err := retrievalDealsListCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Sector List")
if err := sectorsListCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

fmt.Println("\n#: Sector Refs")
if err := sectorsRefsCmd.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}

// Very Very Verbose info
fmt.Println("\n#: Per Sector Info")

list, err := nodeApi.SectorsList(ctx)
if err != nil {
return err
fmt.Println("ERROR: ", err)
}

sort.Slice(list, func(i, j int) bool {
Expand All @@ -129,11 +129,11 @@ var infoAllCmd = &cli.Command{
fs := &flag.FlagSet{}
for _, f := range sectorsStatusCmd.Flags {
if err := f.Apply(fs); err != nil {
return err
fmt.Println("ERROR: ", err)
}
}
if err := fs.Parse([]string{"--log", "--on-chain-info", fmt.Sprint(s)}); err != nil {
return err
fmt.Println("ERROR: ", err)
}

if err := sectorsStatusCmd.Action(cli.NewContext(cctx.App, fs, cctx)); err != nil {
Expand All @@ -144,7 +144,7 @@ var infoAllCmd = &cli.Command{

fs = &flag.FlagSet{}
if err := fs.Parse([]string{fmt.Sprint(s)}); err != nil {
return err
fmt.Println("ERROR: ", err)
}

if err := storageFindCmd.Action(cli.NewContext(cctx.App, fs, cctx)); err != nil {
Expand All @@ -155,7 +155,7 @@ var infoAllCmd = &cli.Command{
if !_test {
fmt.Println("\n#: Goroutines")
if err := lcli.PprofGoroutines.Action(cctx); err != nil {
return err
fmt.Println("ERROR: ", err)
}
}

Expand Down