Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add flag checking for pruning waypoints #10468

Merged
merged 1 commit into from
May 25, 2024
Merged

add flag checking for pruning waypoints #10468

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 0 additions & 112 deletions core/rawdb/accessors_chain.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"context"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"math"
"math/big"
Expand All @@ -42,7 +41,6 @@ import (

"github.com/ledgerwatch/erigon/core/types"
"github.com/ledgerwatch/erigon/ethdb/cbor"
"github.com/ledgerwatch/erigon/polygon/heimdall"
"github.com/ledgerwatch/erigon/rlp"
)

Expand Down Expand Up @@ -1085,116 +1083,6 @@ func PruneBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int) error {
return nil
}

// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
// keeps genesis in db: [1, to)
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
c, err := tx.Cursor(kv.BorEventNums)
if err != nil {
return err
}
defer c.Close()
var blockNumBytes [8]byte
binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
k, v, err := c.Seek(blockNumBytes[:])
if err != nil {
return err
}
var eventIdTo uint64 = math.MaxUint64
if k != nil {
eventIdTo = binary.BigEndian.Uint64(v)
}
c1, err := tx.RwCursor(kv.BorEvents)
if err != nil {
return err
}
defer c1.Close()
counter := blocksDeleteLimit
for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
eventId := binary.BigEndian.Uint64(k)
if eventId >= eventIdTo {
break
}
if err = c1.DeleteCurrent(); err != nil {
return err
}
counter--
}
if err != nil {
return err
}
firstSpanToKeep := SpanIdAt(blockTo)
c2, err := tx.RwCursor(kv.BorSpans)
if err != nil {
return err
}
defer c2.Close()
counter = blocksDeleteLimit
for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
spanId := binary.BigEndian.Uint64(k)
if spanId >= firstSpanToKeep {
break
}
if err = c2.DeleteCurrent(); err != nil {
return err
}
counter--
}

checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
if err != nil {
return err
}

defer checkpointCursor.Close()
lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)

if err != nil {
return err
}

var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
return err
}
}

milestoneCursor, err := tx.RwCursor(kv.BorMilestones)

if err != nil {
return err
}

defer milestoneCursor.Close()

var lastMilestoneToRemove heimdall.MilestoneId

for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))

if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
return err
} else {
if blockCount == blocksDeleteLimit-1 {
return nil
}
}
}

var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
return err
}
}

return nil
}

func TruncateCanonicalChain(ctx context.Context, db kv.RwTx, from uint64) error {
return db.ForEach(kv.HeaderCanonical, hexutility.EncodeTs(from), func(k, _ []byte) error {
return db.Delete(kv.HeaderCanonical, k)
Expand Down
3 changes: 2 additions & 1 deletion core/rawdb/blockio/block_writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/polygon/bor/bordb"
"github.com/ledgerwatch/log/v3"
)

Expand Down Expand Up @@ -122,5 +123,5 @@ func (w *BlockWriter) PruneBlocks(ctx context.Context, tx kv.RwTx, blockTo uint6
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
func (w *BlockWriter) PruneBorBlocks(ctx context.Context, tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
defer mxPruneTookBor.ObserveDuration(time.Now())
return rawdb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
return bordb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
}
83 changes: 44 additions & 39 deletions eth/stagedsync/stage_bor_heimdall.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/ledgerwatch/erigon/polygon/bor/borcfg"
"github.com/ledgerwatch/erigon/polygon/bor/finality"
"github.com/ledgerwatch/erigon/polygon/bor/finality/whitelist"
borsnaptype "github.com/ledgerwatch/erigon/polygon/bor/snaptype"
"github.com/ledgerwatch/erigon/polygon/bor/valset"
"github.com/ledgerwatch/erigon/polygon/heimdall"
"github.com/ledgerwatch/erigon/turbo/services"
Expand Down Expand Up @@ -898,63 +899,67 @@ func BorHeimdallUnwind(u *UnwindState, ctx context.Context, _ *StageState, tx kv
}

// Removing checkpoints
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "checkpoints") {
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
if borsnaptype.CheckpointsEnabled() {
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "checkpoints") {
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)

if err != nil {
return err
}

defer checkpointCursor.Close()
lastCheckpointToKeep, err := heimdall.CheckpointIdAt(tx, u.UnwindPoint)
hasCheckpoints := true

if err != nil {
if !errors.Is(err, heimdall.ErrCheckpointNotFound) {
if err != nil {
return err
}

hasCheckpoints = false
}
defer checkpointCursor.Close()
lastCheckpointToKeep, err := heimdall.CheckpointIdAt(tx, u.UnwindPoint)
hasCheckpoints := true

if hasCheckpoints {
var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToKeep+1))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Next() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
if err != nil {
if !errors.Is(err, heimdall.ErrCheckpointNotFound) {
return err
}
}
}
}

// Removing milestones
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "milestones") {
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
hasCheckpoints = false
}

if err != nil {
return err
if hasCheckpoints {
var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToKeep+1))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Next() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
return err
}
}
}
}
}

defer milestoneCursor.Close()
lastMilestoneToKeep, err := heimdall.MilestoneIdAt(tx, u.UnwindPoint)
hasMilestones := true
if borsnaptype.MilestonesEnabled() {
// Removing milestones
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "milestones") {
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)

if err != nil {
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
if err != nil {
return err
}

hasMilestones = false
}
defer milestoneCursor.Close()
lastMilestoneToKeep, err := heimdall.MilestoneIdAt(tx, u.UnwindPoint)
hasMilestones := true

if hasMilestones {
var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToKeep+1))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Next() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
if err != nil {
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
return err
}

hasMilestones = false
}

if hasMilestones {
var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToKeep+1))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Next() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
return err
}
}
}
}
}
Expand Down
125 changes: 125 additions & 0 deletions polygon/bor/bordb/prune.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package bordb

import (
"encoding/binary"
"errors"
"math"

"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/polygon/bor/snaptype"
"github.com/ledgerwatch/erigon/polygon/heimdall"
)

// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
// keeps genesis in db: [1, to)
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
c, err := tx.Cursor(kv.BorEventNums)
if err != nil {
return err
}
defer c.Close()
var blockNumBytes [8]byte
binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
k, v, err := c.Seek(blockNumBytes[:])
if err != nil {
return err
}
var eventIdTo uint64 = math.MaxUint64
if k != nil {
eventIdTo = binary.BigEndian.Uint64(v)
}
c1, err := tx.RwCursor(kv.BorEvents)
if err != nil {
return err
}
defer c1.Close()
counter := blocksDeleteLimit
for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
eventId := binary.BigEndian.Uint64(k)
if eventId >= eventIdTo {
break
}
if err = c1.DeleteCurrent(); err != nil {
return err
}
counter--
}
if err != nil {
return err
}
firstSpanToKeep := SpanIdAt(blockTo)
c2, err := tx.RwCursor(kv.BorSpans)
if err != nil {
return err
}
defer c2.Close()
counter = blocksDeleteLimit
for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
spanId := binary.BigEndian.Uint64(k)
if spanId >= firstSpanToKeep {
break
}
if err = c2.DeleteCurrent(); err != nil {
return err
}
counter--
}

if snaptype.CheckpointsEnabled() {
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
if err != nil {
return err
}

defer checkpointCursor.Close()
lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)

if err != nil {
return err
}

var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
return err
}
}
}

if snaptype.MilestonesEnabled() {
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)

if err != nil {
return err
}

defer milestoneCursor.Close()

var lastMilestoneToRemove heimdall.MilestoneId

for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))

if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
return err
} else {
if blockCount == blocksDeleteLimit-1 {
return nil
}
}
}

var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
return err
}
}
}

return nil
}
Loading
Loading