From 765dab9ef0614e9b9124a760aecae142d488355b Mon Sep 17 00:00:00 2001 From: weiihann Date: Thu, 17 Oct 2024 16:50:44 +0800 Subject: [PATCH 1/6] check for unknown KV, fix upper bound bug --- cmd/juno/dbcmd.go | 18 ++++++++++++++++-- db/pebble/db.go | 27 ++++++++++++++++++++++++--- db/pebble/db_test.go | 8 ++++---- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/cmd/juno/dbcmd.go b/cmd/juno/dbcmd.go index d9892b6be1..653f032cf3 100644 --- a/cmd/juno/dbcmd.go +++ b/cmd/juno/dbcmd.go @@ -203,9 +203,10 @@ func dbSize(cmd *cobra.Command, args []string) error { items [][]string ) - for _, b := range db.BucketValues() { + buckets := db.BucketValues() + for _, b := range buckets { fmt.Fprintf(cmd.OutOrStdout(), "Calculating size of %s, remaining buckets: %d\n", b, len(db.BucketValues())-int(b)-1) - bucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(b)}) + bucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(b)}, true) if err != nil { return err } @@ -228,6 +229,19 @@ func dbSize(cmd *cobra.Command, args []string) error { } } + // check if there is any data left in the db + lastBucket := buckets[len(buckets)-1] + lastBucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(lastBucket + 1)}, false) + if err != nil { + return err + } + + if lastBucketItem.Count > 0 { + items = append(items, []string{"Unknown", lastBucketItem.Size.String(), fmt.Sprintf("%d", lastBucketItem.Count)}) + totalSize += lastBucketItem.Size + totalCount += lastBucketItem.Count + } + table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Bucket", "Size", "Count"}) table.AppendBulk(items) diff --git a/db/pebble/db.go b/db/pebble/db.go index 77aed603d7..b91576d75f 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -17,6 +17,7 @@ const ( // minCache is the minimum amount of memory in megabytes to allocate to pebble read and write caching. // This is also pebble's default value. minCacheSizeMB = 8 + byteLimit = 0xff ) var ( @@ -128,7 +129,7 @@ func (i *Item) add(size utils.DataSize) { i.Size += size } -func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte) (*Item, error) { +func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte, withUpperBound bool) (*Item, error) { var ( err error v []byte @@ -136,9 +137,13 @@ func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte) (*Item, er item = &Item{} ) - const upperBoundofPrefix = 0xff pebbleDB := pDB.Impl().(*pebble.DB) - it, err := pebbleDB.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: append(prefix, upperBoundofPrefix)}) + iterOpt := &pebble.IterOptions{LowerBound: prefix} + if withUpperBound { + iterOpt.UpperBound = upperBound(prefix) + } + + it, err := pebbleDB.NewIter(iterOpt) if err != nil { // No need to call utils.RunAndWrapOnError() since iterator couldn't be created return nil, err @@ -158,3 +163,19 @@ func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte) (*Item, er return item, utils.RunAndWrapOnError(it.Close, err) } + +func upperBound(prefix []byte) []byte { + var ub []byte + + for i := len(prefix) - 1; i >= 0; i-- { + if prefix[i] == byteLimit { + continue + } + ub = make([]byte, i+1) + copy(ub, prefix) + ub[i]++ + return ub + } + + return nil +} diff --git a/db/pebble/db_test.go b/db/pebble/db_test.go index 896d5fac5c..d1c5d72f7c 100644 --- a/db/pebble/db_test.go +++ b/db/pebble/db_test.go @@ -424,7 +424,7 @@ func TestCalculatePrefixSize(t *testing.T) { t.Run("empty db", func(t *testing.T) { testDB := pebble.NewMemTest(t).(*pebble.DB) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB, []byte("0")) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB, []byte("0"), true) require.NoError(t, err) assert.Zero(t, s.Count) assert.Zero(t, s.Size) @@ -435,7 +435,7 @@ func TestCalculatePrefixSize(t *testing.T) { require.NoError(t, testDB.Update(func(txn db.Transaction) error { return txn.Set(append([]byte("0"), []byte("randomKey")...), []byte("someValue")) })) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), []byte("1")) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), []byte("1"), true) require.NoError(t, err) assert.Zero(t, s.Count) assert.Zero(t, s.Size) @@ -455,7 +455,7 @@ func TestCalculatePrefixSize(t *testing.T) { return txn.Set(k3, v3) })) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), p) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), p, true) require.NoError(t, err) assert.Equal(t, uint(3), s.Count) assert.Equal(t, utils.DataSize(expectedSize), s.Size) @@ -464,7 +464,7 @@ func TestCalculatePrefixSize(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s, err := pebble.CalculatePrefixSize(ctx, testDB.(*pebble.DB), p) + s, err := pebble.CalculatePrefixSize(ctx, testDB.(*pebble.DB), p, true) assert.EqualError(t, err, context.Canceled.Error()) assert.Zero(t, s.Count) assert.Zero(t, s.Size) From 9021e42208c28d932399ea224f53559b8ef3aa7b Mon Sep 17 00:00:00 2001 From: weiihann Date: Thu, 17 Oct 2024 16:58:44 +0800 Subject: [PATCH 2/6] minor chore --- cmd/juno/dbcmd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/juno/dbcmd.go b/cmd/juno/dbcmd.go index 653f032cf3..b09522877b 100644 --- a/cmd/juno/dbcmd.go +++ b/cmd/juno/dbcmd.go @@ -231,6 +231,7 @@ func dbSize(cmd *cobra.Command, args []string) error { // check if there is any data left in the db lastBucket := buckets[len(buckets)-1] + fmt.Fprintln(cmd.OutOrStdout(), "Calculating remaining data in the db") lastBucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(lastBucket + 1)}, false) if err != nil { return err From e4b412b8a3906bc4ef8f67e1aad2e36c7d7e6dd5 Mon Sep 17 00:00:00 2001 From: weiihann Date: Tue, 17 Dec 2024 10:02:00 +0800 Subject: [PATCH 3/6] remove upperbound option --- cmd/juno/dbcmd.go | 4 ++-- db/pebble/db.go | 8 ++------ db/pebble/db_test.go | 8 ++++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/cmd/juno/dbcmd.go b/cmd/juno/dbcmd.go index b09522877b..6adbd6ed95 100644 --- a/cmd/juno/dbcmd.go +++ b/cmd/juno/dbcmd.go @@ -206,7 +206,7 @@ func dbSize(cmd *cobra.Command, args []string) error { buckets := db.BucketValues() for _, b := range buckets { fmt.Fprintf(cmd.OutOrStdout(), "Calculating size of %s, remaining buckets: %d\n", b, len(db.BucketValues())-int(b)-1) - bucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(b)}, true) + bucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(b)}) if err != nil { return err } @@ -232,7 +232,7 @@ func dbSize(cmd *cobra.Command, args []string) error { // check if there is any data left in the db lastBucket := buckets[len(buckets)-1] fmt.Fprintln(cmd.OutOrStdout(), "Calculating remaining data in the db") - lastBucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(lastBucket + 1)}, false) + lastBucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(lastBucket + 1)}) if err != nil { return err } diff --git a/db/pebble/db.go b/db/pebble/db.go index b91576d75f..8a310c0f9f 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -129,7 +129,7 @@ func (i *Item) add(size utils.DataSize) { i.Size += size } -func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte, withUpperBound bool) (*Item, error) { +func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte) (*Item, error) { var ( err error v []byte @@ -138,12 +138,8 @@ func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte, withUpperB ) pebbleDB := pDB.Impl().(*pebble.DB) - iterOpt := &pebble.IterOptions{LowerBound: prefix} - if withUpperBound { - iterOpt.UpperBound = upperBound(prefix) - } - it, err := pebbleDB.NewIter(iterOpt) + it, err := pebbleDB.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: upperBound(prefix)}) if err != nil { // No need to call utils.RunAndWrapOnError() since iterator couldn't be created return nil, err diff --git a/db/pebble/db_test.go b/db/pebble/db_test.go index d1c5d72f7c..896d5fac5c 100644 --- a/db/pebble/db_test.go +++ b/db/pebble/db_test.go @@ -424,7 +424,7 @@ func TestCalculatePrefixSize(t *testing.T) { t.Run("empty db", func(t *testing.T) { testDB := pebble.NewMemTest(t).(*pebble.DB) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB, []byte("0"), true) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB, []byte("0")) require.NoError(t, err) assert.Zero(t, s.Count) assert.Zero(t, s.Size) @@ -435,7 +435,7 @@ func TestCalculatePrefixSize(t *testing.T) { require.NoError(t, testDB.Update(func(txn db.Transaction) error { return txn.Set(append([]byte("0"), []byte("randomKey")...), []byte("someValue")) })) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), []byte("1"), true) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), []byte("1")) require.NoError(t, err) assert.Zero(t, s.Count) assert.Zero(t, s.Size) @@ -455,7 +455,7 @@ func TestCalculatePrefixSize(t *testing.T) { return txn.Set(k3, v3) })) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), p, true) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), p) require.NoError(t, err) assert.Equal(t, uint(3), s.Count) assert.Equal(t, utils.DataSize(expectedSize), s.Size) @@ -464,7 +464,7 @@ func TestCalculatePrefixSize(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s, err := pebble.CalculatePrefixSize(ctx, testDB.(*pebble.DB), p, true) + s, err := pebble.CalculatePrefixSize(ctx, testDB.(*pebble.DB), p) assert.EqualError(t, err, context.Canceled.Error()) assert.Zero(t, s.Count) assert.Zero(t, s.Size) From 28af36334372f3b6ffe3edab487a1ae3c4b8ce23 Mon Sep 17 00:00:00 2001 From: weiihann Date: Tue, 17 Dec 2024 17:22:03 +0800 Subject: [PATCH 4/6] Revert "remove upperbound option" This reverts commit e4b412b8a3906bc4ef8f67e1aad2e36c7d7e6dd5. --- cmd/juno/dbcmd.go | 4 ++-- db/pebble/db.go | 8 ++++++-- db/pebble/db_test.go | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cmd/juno/dbcmd.go b/cmd/juno/dbcmd.go index 6adbd6ed95..b09522877b 100644 --- a/cmd/juno/dbcmd.go +++ b/cmd/juno/dbcmd.go @@ -206,7 +206,7 @@ func dbSize(cmd *cobra.Command, args []string) error { buckets := db.BucketValues() for _, b := range buckets { fmt.Fprintf(cmd.OutOrStdout(), "Calculating size of %s, remaining buckets: %d\n", b, len(db.BucketValues())-int(b)-1) - bucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(b)}) + bucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(b)}, true) if err != nil { return err } @@ -232,7 +232,7 @@ func dbSize(cmd *cobra.Command, args []string) error { // check if there is any data left in the db lastBucket := buckets[len(buckets)-1] fmt.Fprintln(cmd.OutOrStdout(), "Calculating remaining data in the db") - lastBucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(lastBucket + 1)}) + lastBucketItem, err := pebble.CalculatePrefixSize(cmd.Context(), pebbleDB.(*pebble.DB), []byte{byte(lastBucket + 1)}, false) if err != nil { return err } diff --git a/db/pebble/db.go b/db/pebble/db.go index 8a310c0f9f..b91576d75f 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -129,7 +129,7 @@ func (i *Item) add(size utils.DataSize) { i.Size += size } -func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte) (*Item, error) { +func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte, withUpperBound bool) (*Item, error) { var ( err error v []byte @@ -138,8 +138,12 @@ func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte) (*Item, er ) pebbleDB := pDB.Impl().(*pebble.DB) + iterOpt := &pebble.IterOptions{LowerBound: prefix} + if withUpperBound { + iterOpt.UpperBound = upperBound(prefix) + } - it, err := pebbleDB.NewIter(&pebble.IterOptions{LowerBound: prefix, UpperBound: upperBound(prefix)}) + it, err := pebbleDB.NewIter(iterOpt) if err != nil { // No need to call utils.RunAndWrapOnError() since iterator couldn't be created return nil, err diff --git a/db/pebble/db_test.go b/db/pebble/db_test.go index 896d5fac5c..d1c5d72f7c 100644 --- a/db/pebble/db_test.go +++ b/db/pebble/db_test.go @@ -424,7 +424,7 @@ func TestCalculatePrefixSize(t *testing.T) { t.Run("empty db", func(t *testing.T) { testDB := pebble.NewMemTest(t).(*pebble.DB) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB, []byte("0")) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB, []byte("0"), true) require.NoError(t, err) assert.Zero(t, s.Count) assert.Zero(t, s.Size) @@ -435,7 +435,7 @@ func TestCalculatePrefixSize(t *testing.T) { require.NoError(t, testDB.Update(func(txn db.Transaction) error { return txn.Set(append([]byte("0"), []byte("randomKey")...), []byte("someValue")) })) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), []byte("1")) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), []byte("1"), true) require.NoError(t, err) assert.Zero(t, s.Count) assert.Zero(t, s.Size) @@ -455,7 +455,7 @@ func TestCalculatePrefixSize(t *testing.T) { return txn.Set(k3, v3) })) - s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), p) + s, err := pebble.CalculatePrefixSize(context.Background(), testDB.(*pebble.DB), p, true) require.NoError(t, err) assert.Equal(t, uint(3), s.Count) assert.Equal(t, utils.DataSize(expectedSize), s.Size) @@ -464,7 +464,7 @@ func TestCalculatePrefixSize(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - s, err := pebble.CalculatePrefixSize(ctx, testDB.(*pebble.DB), p) + s, err := pebble.CalculatePrefixSize(ctx, testDB.(*pebble.DB), p, true) assert.EqualError(t, err, context.Canceled.Error()) assert.Zero(t, s.Count) assert.Zero(t, s.Size) From 7a8a159649c5d9b9034d94279b6ce1a123699c35 Mon Sep 17 00:00:00 2001 From: Ng Wei Han <47109095+weiihann@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:01:55 +0700 Subject: [PATCH 5/6] Update db/pebble/db.go Co-authored-by: Pawel Nowosielski Signed-off-by: Ng Wei Han <47109095+weiihann@users.noreply.github.com> --- db/pebble/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/pebble/db.go b/db/pebble/db.go index b91576d75f..a4143625c2 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -17,7 +17,7 @@ const ( // minCache is the minimum amount of memory in megabytes to allocate to pebble read and write caching. // This is also pebble's default value. minCacheSizeMB = 8 - byteLimit = 0xff + maxByte = ^byte(0) ) var ( From 15c33bcec4ef6c50620eb218ab25504c638237d1 Mon Sep 17 00:00:00 2001 From: weiihann Date: Tue, 17 Dec 2024 19:36:39 +0800 Subject: [PATCH 6/6] fix --- db/pebble/db.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/pebble/db.go b/db/pebble/db.go index a4143625c2..dc9a62d4c1 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -17,7 +17,7 @@ const ( // minCache is the minimum amount of memory in megabytes to allocate to pebble read and write caching. // This is also pebble's default value. minCacheSizeMB = 8 - maxByte = ^byte(0) + maxByte = ^byte(0) ) var ( @@ -168,7 +168,7 @@ func upperBound(prefix []byte) []byte { var ub []byte for i := len(prefix) - 1; i >= 0; i-- { - if prefix[i] == byteLimit { + if prefix[i] == maxByte { continue } ub = make([]byte, i+1)