Skip to content

Commit

Permalink
add flag checking for pruning waypoints (#10508)
Browse files Browse the repository at this point in the history
Cherry pick PR #10468 into the release branch

Co-authored-by: Mark Holt <[email protected]>
  • Loading branch information
yperbasis and mh0lt authored May 28, 2024
1 parent 608a940 commit e9840ad
Show file tree
Hide file tree
Showing 5 changed files with 191 additions and 152 deletions.
112 changes: 0 additions & 112 deletions core/rawdb/accessors_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"context"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"math"
"math/big"
Expand All @@ -42,7 +41,6 @@ import (

"github.com/ledgerwatch/erigon/core/types"
"github.com/ledgerwatch/erigon/ethdb/cbor"
"github.com/ledgerwatch/erigon/polygon/heimdall"
"github.com/ledgerwatch/erigon/rlp"
)

Expand Down Expand Up @@ -1096,116 +1094,6 @@ func PruneBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int) error {
return nil
}

// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
// keeps genesis in db: [1, to)
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
c, err := tx.Cursor(kv.BorEventNums)
if err != nil {
return err
}
defer c.Close()
var blockNumBytes [8]byte
binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
k, v, err := c.Seek(blockNumBytes[:])
if err != nil {
return err
}
var eventIdTo uint64 = math.MaxUint64
if k != nil {
eventIdTo = binary.BigEndian.Uint64(v)
}
c1, err := tx.RwCursor(kv.BorEvents)
if err != nil {
return err
}
defer c1.Close()
counter := blocksDeleteLimit
for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
eventId := binary.BigEndian.Uint64(k)
if eventId >= eventIdTo {
break
}
if err = c1.DeleteCurrent(); err != nil {
return err
}
counter--
}
if err != nil {
return err
}
firstSpanToKeep := SpanIdAt(blockTo)
c2, err := tx.RwCursor(kv.BorSpans)
if err != nil {
return err
}
defer c2.Close()
counter = blocksDeleteLimit
for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
spanId := binary.BigEndian.Uint64(k)
if spanId >= firstSpanToKeep {
break
}
if err = c2.DeleteCurrent(); err != nil {
return err
}
counter--
}

checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
if err != nil {
return err
}

defer checkpointCursor.Close()
lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)

if err != nil {
return err
}

var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
return err
}
}

milestoneCursor, err := tx.RwCursor(kv.BorMilestones)

if err != nil {
return err
}

defer milestoneCursor.Close()

var lastMilestoneToRemove heimdall.MilestoneId

for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))

if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
return err
} else {
if blockCount == blocksDeleteLimit-1 {
return nil
}
}
}

var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
return err
}
}

return nil
}

func TruncateCanonicalChain(ctx context.Context, db kv.RwTx, from uint64) error {
return db.ForEach(kv.HeaderCanonical, hexutility.EncodeTs(from), func(k, _ []byte) error {
return db.Delete(kv.HeaderCanonical, k)
Expand Down
3 changes: 2 additions & 1 deletion core/rawdb/blockio/block_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/polygon/bor/bordb"
"github.com/ledgerwatch/log/v3"
)

Expand Down Expand Up @@ -116,5 +117,5 @@ func (w *BlockWriter) PruneBlocks(ctx context.Context, tx kv.RwTx, blockTo uint6
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
func (w *BlockWriter) PruneBorBlocks(ctx context.Context, tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
return rawdb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
return bordb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
}
83 changes: 44 additions & 39 deletions eth/stagedsync/stage_bor_heimdall.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/ledgerwatch/erigon/polygon/bor/borcfg"
"github.com/ledgerwatch/erigon/polygon/bor/finality"
"github.com/ledgerwatch/erigon/polygon/bor/finality/whitelist"
borsnaptype "github.com/ledgerwatch/erigon/polygon/bor/snaptype"
"github.com/ledgerwatch/erigon/polygon/bor/valset"
"github.com/ledgerwatch/erigon/polygon/heimdall"
"github.com/ledgerwatch/erigon/turbo/services"
Expand Down Expand Up @@ -890,63 +891,67 @@ func BorHeimdallUnwind(u *UnwindState, ctx context.Context, _ *StageState, tx kv
}

// Removing checkpoints
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "checkpoints") {
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
if borsnaptype.CheckpointsEnabled() {
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "checkpoints") {
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)

if err != nil {
return err
}

defer checkpointCursor.Close()
lastCheckpointToKeep, err := heimdall.CheckpointIdAt(tx, u.UnwindPoint)
hasCheckpoints := true

if err != nil {
if !errors.Is(err, heimdall.ErrCheckpointNotFound) {
if err != nil {
return err
}

hasCheckpoints = false
}
defer checkpointCursor.Close()
lastCheckpointToKeep, err := heimdall.CheckpointIdAt(tx, u.UnwindPoint)
hasCheckpoints := true

if hasCheckpoints {
var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToKeep+1))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Next() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
if err != nil {
if !errors.Is(err, heimdall.ErrCheckpointNotFound) {
return err
}
}
}
}

// Removing milestones
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "milestones") {
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
hasCheckpoints = false
}

if err != nil {
return err
if hasCheckpoints {
var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToKeep+1))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Next() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
return err
}
}
}
}
}

defer milestoneCursor.Close()
lastMilestoneToKeep, err := heimdall.MilestoneIdAt(tx, u.UnwindPoint)
hasMilestones := true
if borsnaptype.MilestonesEnabled() {
// Removing milestones
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "milestones") {
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)

if err != nil {
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
if err != nil {
return err
}

hasMilestones = false
}
defer milestoneCursor.Close()
lastMilestoneToKeep, err := heimdall.MilestoneIdAt(tx, u.UnwindPoint)
hasMilestones := true

if hasMilestones {
var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToKeep+1))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Next() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
if err != nil {
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
return err
}

hasMilestones = false
}

if hasMilestones {
var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToKeep+1))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Next() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
return err
}
}
}
}
}
Expand Down
125 changes: 125 additions & 0 deletions polygon/bor/bordb/prune.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
package bordb

import (
"encoding/binary"
"errors"
"math"

"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon/polygon/bor/snaptype"
"github.com/ledgerwatch/erigon/polygon/heimdall"
)

// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
// keeps genesis in db: [1, to)
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
c, err := tx.Cursor(kv.BorEventNums)
if err != nil {
return err
}
defer c.Close()
var blockNumBytes [8]byte
binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
k, v, err := c.Seek(blockNumBytes[:])
if err != nil {
return err
}
var eventIdTo uint64 = math.MaxUint64
if k != nil {
eventIdTo = binary.BigEndian.Uint64(v)
}
c1, err := tx.RwCursor(kv.BorEvents)
if err != nil {
return err
}
defer c1.Close()
counter := blocksDeleteLimit
for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
eventId := binary.BigEndian.Uint64(k)
if eventId >= eventIdTo {
break
}
if err = c1.DeleteCurrent(); err != nil {
return err
}
counter--
}
if err != nil {
return err
}
firstSpanToKeep := SpanIdAt(blockTo)
c2, err := tx.RwCursor(kv.BorSpans)
if err != nil {
return err
}
defer c2.Close()
counter = blocksDeleteLimit
for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
spanId := binary.BigEndian.Uint64(k)
if spanId >= firstSpanToKeep {
break
}
if err = c2.DeleteCurrent(); err != nil {
return err
}
counter--
}

if snaptype.CheckpointsEnabled() {
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
if err != nil {
return err
}

defer checkpointCursor.Close()
lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)

if err != nil {
return err
}

var checkpointIdBytes [8]byte
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
if err = checkpointCursor.DeleteCurrent(); err != nil {
return err
}
}
}

if snaptype.MilestonesEnabled() {
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)

if err != nil {
return err
}

defer milestoneCursor.Close()

var lastMilestoneToRemove heimdall.MilestoneId

for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))

if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
return err
} else {
if blockCount == blocksDeleteLimit-1 {
return nil
}
}
}

var milestoneIdBytes [8]byte
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
if err = milestoneCursor.DeleteCurrent(); err != nil {
return err
}
}
}

return nil
}
Loading

0 comments on commit e9840ad

Please sign in to comment.