Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimization package #343

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
4ab95b1
add caching for event IDs
uprendis Jun 29, 2022
00ddec3
add caching for LLR voting weights
uprendis Jun 29, 2022
aae378a
fix keeping mutating memory from it.Value()
uprendis Jun 29, 2022
d224554
erase support for outdated DB migrations
uprendis Jun 29, 2022
7985f36
adjust genesis processing ETA logging
uprendis Jun 28, 2022
16bc869
fullsync syncs LLR only if synced up events
uprendis Jun 28, 2022
13f3e01
stricter synced condition
uprendis Jun 28, 2022
5d4da77
add support of multidb config
uprendis Mar 26, 2022
8fea1c2
add `db migrate` command
uprendis Mar 26, 2022
0b6e1cd
adopt fixdity command for multidb feature
uprendis Jun 12, 2022
c5657e1
add batching for genesis processing
uprendis Jun 29, 2022
82d1815
use piecefunc from lachesis-base
uprendis Jul 25, 2022
d697ba7
refactor evm.Commit args
uprendis Jul 25, 2022
00186e4
add support of 3 DB integrity modes (flushable, flagged, direct)
uprendis Jul 25, 2022
430a0d8
add automatic EVM recover after abrupt stopping
uprendis Jul 25, 2022
b568d12
add automatic migration from pre-multi-db layout
uprendis Jul 25, 2022
e369a09
rename `db migrate` to `db transform`
uprendis Jul 25, 2022
b408c39
rename `db fix` to `db heal`
uprendis Jul 25, 2022
60f6524
tip to use `db heal` command
uprendis Jul 25, 2022
c192e0c
reduce scaling of MaxNonFlushedSize with RAM size
uprendis Jul 25, 2022
bbcd5e3
fix race condition in asyncflushproducer.OpenDB when the same DB is c…
uprendis Jul 26, 2022
d97c649
refactor DBs producers in launcher
uprendis Jul 26, 2022
333008d
fix 'db compact' command after multidb changes
uprendis Jul 26, 2022
7507e84
support both leveldb and pebble during multidb migration
uprendis Jul 26, 2022
51b490c
prevent damaging DBs during multidb migration when disk space is low
uprendis Jul 26, 2022
7ed2dea
refactor multidb migration
uprendis Jul 26, 2022
4d6b08b
fix compacting source DB during multidb migration
uprendis Jul 26, 2022
685575e
increase db cache during multidb migration
uprendis Jul 26, 2022
a63b372
Refactor CheckEvm call
uprendis Jul 27, 2022
cd47958
rm duplicate of dbCommand
uprendis Jul 27, 2022
391a9a6
fix compilation of config_custom_test.go
uprendis Jul 28, 2022
53eddab
fix RecoverEVM with pruned blocks
uprendis Jul 28, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 3 additions & 9 deletions cmd/opera/launcher/check.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package launcher

import (
"path"
"time"

"github.com/Fantom-foundation/lachesis-base/inter/idx"
Expand All @@ -10,7 +9,6 @@ import (
"github.com/ethereum/go-ethereum/log"
"gopkg.in/urfave/cli.v1"

"github.com/Fantom-foundation/go-opera/integration"
"github.com/Fantom-foundation/go-opera/inter"
)

Expand All @@ -21,11 +19,8 @@ func checkEvm(ctx *cli.Context) error {

cfg := makeAllConfigs(ctx)

rawProducer := integration.DBProducer(path.Join(cfg.Node.DataDir, "chaindata"), cfg.cachescale)
gdb, err := makeRawGossipStore(rawProducer, cfg)
if err != nil {
log.Crit("DB opening error", "datadir", cfg.Node.DataDir, "err", err)
}
rawDbs := makeDirectDBsProducer(cfg)
gdb := makeGossipStore(rawDbs, cfg)
defer gdb.Close()
evms := gdb.EvmStore()

Expand Down Expand Up @@ -62,8 +57,7 @@ func checkEvm(ctx *cli.Context) error {
})
}

err = evms.CheckEvm(checkBlocks)
if err != nil {
if err := evms.CheckEvm(checkBlocks); err != nil {
return err
}
log.Info("EVM storage is verified", "last", prevPoint, "elapsed", common.PrettyDuration(time.Since(start)))
Expand Down
15 changes: 13 additions & 2 deletions cmd/opera/launcher/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ type config struct {
Lachesis abft.Config
LachesisStore abft.StoreConfig
VectorClock vecmt.IndexConfig
cachescale cachescale.Func
DBs integration.DBsConfig
}

func (c *config) AppConfigs() integration.Configs {
Expand All @@ -158,6 +158,7 @@ func (c *config) AppConfigs() integration.Configs {
Lachesis: c.Lachesis,
LachesisStore: c.LachesisStore,
VectorClock: c.VectorClock,
DBs: c.DBs,
}
}

Expand Down Expand Up @@ -369,7 +370,6 @@ func mayMakeAllConfigs(ctx *cli.Context) (*config, error) {
Lachesis: abft.DefaultConfig(),
LachesisStore: abft.DefaultStoreConfig(cacheRatio),
VectorClock: vecmt.DefaultConfig(cacheRatio),
cachescale: cacheRatio,
}

if ctx.GlobalIsSet(FakeNetFlag.Name) {
Expand All @@ -388,6 +388,17 @@ func mayMakeAllConfigs(ctx *cli.Context) (*config, error) {
return &cfg, err
}
}
// apply default for DB config if it wasn't touched by config file
dbDefault := integration.DefaultDBsConfig(cacheRatio.U64, uint64(utils.MakeDatabaseHandles()))
if len(cfg.DBs.Routing.Table) == 0 {
cfg.DBs.Routing = dbDefault.Routing
}
if len(cfg.DBs.GenesisCache.Table) == 0 {
cfg.DBs.GenesisCache = dbDefault.GenesisCache
}
if len(cfg.DBs.RuntimeCache.Table) == 0 {
cfg.DBs.RuntimeCache = dbDefault.RuntimeCache
}

// Apply flags (high priority)
var err error
Expand Down
1 change: 0 additions & 1 deletion cmd/opera/launcher/config_custom_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ func TestConfigFile(t *testing.T) {
Lachesis: abft.DefaultConfig(),
LachesisStore: abft.DefaultStoreConfig(cacheRatio),
VectorClock: vecmt.DefaultConfig(cacheRatio),
cachescale: cacheRatio,
}

canonical := func(nn []*enode.Node) []*enode.Node {
Expand Down
2 changes: 1 addition & 1 deletion cmd/opera/launcher/consolecmd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const (
// Tests that a node embedded within a console can be started up properly and
// then terminated by closing the input stream.
func TestConsoleWelcome(t *testing.T) {
// Start a opera console, make sure it's cleaned up and terminate the console
// Start an opera console, make sure it's cleaned up and terminate the console
cli := exec(t,
"--fakenet", "0/1", "--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none",
"console")
Expand Down
Loading