Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/#2403 snap #2404

Merged
merged 9 commits into from
Aug 28, 2023
Merged

Feature/#2403 snap #2404

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a1457cd
Path snapshot command
ARR552 Aug 11, 2023
bc9ae51
restore
ARR552 Aug 11, 2023
34290c7
readme
ARR552 Aug 11, 2023
f1cfe46
Merge remote-tracking branch 'origin/release/v0.2.6' into feature/#24…
ARR552 Aug 24, 2023
e168636
options used by dbeaver
ARR552 Aug 24, 2023
0d2e60b
#2429_forkID_improvement: #2429_forkID_improvement:
ARR552 Aug 24, 2023
a4a3341
fix
ARR552 Aug 27, 2023
a167e05
fix postgres version to v15
ARR552 Aug 28, 2023
3c4fd90
fix permissionless init script
ARR552 Aug 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/readme.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

### Create snapshots
```
go run ./cmd snapshot --cfg config/environments/local/local.node.config.toml
go run ./cmd snapshot --cfg config/environments/local/local.node.config.toml --output ./folder/
```

### Restore snapshots
```
go run ./cmd restore --cfg config/environments/local/local.node.config.toml -is zkevmpubliccorestatedb_1685614455_v0.1.0_undefined.sql.tar.gz -ih zkevmpublicstatedb_1685615051_v0.1.0_undefined.sql.tar.gz
go run ./cmd restore --cfg config/environments/local/local.node.config.toml -is ./folder/zkevmpubliccorestatedb_1685614455_v0.1.0_undefined.sql.tar.gz -ih ./folder/zkevmpublicstatedb_1685615051_v0.1.0_undefined.sql.tar.gz
```
70 changes: 58 additions & 12 deletions cmd/restore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package main

import (
"bufio"
"errors"
"fmt"
"io"
"os"
"os/exec"
"strconv"
"strings"

Expand Down Expand Up @@ -45,9 +50,6 @@ func restore(ctx *cli.Context) error {
return errors.New("stateDB input file must end in .sql.tar.gz")
}

// Run migrations to create schemas and tables
runStateMigrations(c.StateDB)

port, err := strconv.Atoi(c.StateDB.Port)
if err != nil {
log.Error("error converting port to int. Error: ", err)
Expand All @@ -64,12 +66,11 @@ func restore(ctx *cli.Context) error {
log.Error("error: ", err)
return err
}
restore.Role = c.StateDB.User
restore.Schemas = append(restore.Schemas, "state")
params := []string{"--no-owner", "--no-acl", "--clean", "--if-exists", "--format=c --exit-on-error"}
log.Info("Restore stateDB snapshot started, please wait...")
restoreExec := restore.Exec(inputFileStateDB, pg.ExecOptions{StreamPrint: false})
restoreExec := execCommand(restore, inputFileStateDB, pg.ExecOptions{StreamPrint: false}, params)
if restoreExec.Error != nil {
log.Error("error restoring snapshot. Error: ", restoreExec.Error.Err)
log.Error("error restoring stateDB snapshot. Error: ", restoreExec.Error.Err)
log.Debug("restoreExec.Output: ", restoreExec.Output)
return err
}
Expand Down Expand Up @@ -105,17 +106,62 @@ func restore(ctx *cli.Context) error {
log.Error("error: ", err)
return err
}
restore.Role = c.HashDB.User
restore.Schemas = append(restore.Schemas, "state")
restore.Options = []string{"--no-owner", "--no-acl"}

log.Info("Restore HashDB snapshot started, please wait...")
restoreExec = restore.Exec(inputFileHashDB, pg.ExecOptions{StreamPrint: false})
restoreExec = execCommand(restore, inputFileHashDB, pg.ExecOptions{StreamPrint: false}, params)
if restoreExec.Error != nil {
log.Error("error restoring snapshot. Error: ", restoreExec.Error.Err)
log.Error("error restoring hashDB snapshot. Error: ", restoreExec.Error.Err)
log.Debug("restoreExec.Output: ", restoreExec.Output)
return err
}

log.Info("Restore HashDB snapshot success")
return nil
}

func execCommand(x *pg.Restore, filename string, opts pg.ExecOptions, params []string) pg.Result {
result := pg.Result{}
options := append(params, x.Postgres.Parse()...)
options = append(options, fmt.Sprintf("%s%s", x.Path, filename))
log.Debug("Options: ", options)

result.FullCommand = strings.Join(options, " ")
cmd := exec.Command(pg.PGRestoreCmd, options...) //nolint:gosec
cmd.Env = append(os.Environ(), x.EnvPassword)
stderrIn, _ := cmd.StderrPipe()
go func(stderrIn io.ReadCloser, opts pg.ExecOptions, result *pg.Result) {
output := ""
reader := bufio.NewReader(stderrIn)
for {
line, err := reader.ReadString('\n')
if err != nil {
if errors.Is(err, io.EOF) {
result.Output = output
break
}
result.Error = &pg.ResultError{Err: fmt.Errorf("error reading output: %w", err), CmdOutput: output}
break
}

if opts.StreamPrint {
_, err = fmt.Fprint(opts.StreamDestination, line)
if err != nil {
result.Error = &pg.ResultError{Err: fmt.Errorf("error writing output: %w", err), CmdOutput: output}
break
}
}

output += line
}
}(stderrIn, opts, &result)
err := cmd.Start()
if err != nil {
result.Error = &pg.ResultError{Err: err, CmdOutput: result.Output}
}
err = cmd.Wait()
if exitError, ok := err.(*exec.ExitError); ok {
result.Error = &pg.ResultError{Err: err, ExitCode: exitError.ExitCode(), CmdOutput: result.Output}
}

return result
}
3 changes: 3 additions & 0 deletions cmd/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

var snapshotFlags = []cli.Flag{
&configFileFlag,
&outputFileFlag,
}

func snapshot(ctx *cli.Context) error {
Expand Down Expand Up @@ -43,6 +44,7 @@ func snapshot(ctx *cli.Context) error {
}
dump.Options = append(dump.Options, "-Z 9")
log.Info("StateDB snapshot is being created...")
dump.Path = ctx.String(config.FlagOutputFile)
dump.SetFileName(fmt.Sprintf(`%v_%v_%v_%v.sql.tar.gz`, dump.DB, time.Now().Unix(), zkevm.Version, zkevm.GitRev))
dumpExec := dump.Exec(pg.ExecOptions{StreamPrint: false})
if dumpExec.Error != nil {
Expand Down Expand Up @@ -71,6 +73,7 @@ func snapshot(ctx *cli.Context) error {
}
dump.Options = append(dump.Options, "-Z 9")
log.Info("HashDB snapshot is being created...")
dump.Path = ctx.String(config.FlagOutputFile)
dump.SetFileName(fmt.Sprintf(`%v_%v_%v_%v.sql.tar.gz`, dump.DB, time.Now().Unix(), zkevm.Version, zkevm.GitRev))
dumpExec = dump.Exec(pg.ExecOptions{StreamPrint: false})
if dumpExec.Error != nil {
Expand Down
Loading