Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Record the amount of free pages instead of hashmapFreeCount calculation #585

Merged
merged 2 commits into from
Oct 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,14 @@ lint:
.PHONY: test
test:
@echo "hashmap freelist test"
TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT}
TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./internal/...
TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt
BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT}
BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./internal/...
BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} ./cmd/bbolt

@echo "array freelist test"
TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT}
TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/...
TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt
BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout ${TESTFLAGS_TIMEOUT}
BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./internal/...
BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} ./cmd/bbolt

.PHONY: coverage
coverage:
Expand All @@ -76,8 +76,8 @@ install-gofail:
.PHONY: test-failpoint
test-failpoint:
@echo "[failpoint] hashmap freelist test"
TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint
BBOLT_VERIFY=all TEST_FREELIST_TYPE=hashmap go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint

@echo "[failpoint] array freelist test"
TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint
BBOLT_VERIFY=all TEST_FREELIST_TYPE=array go test -v ${TESTFLAGS} -timeout 30m ./tests/failpoint

1 change: 1 addition & 0 deletions freelist.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type freelist struct {
freemaps map[uint64]pidSet // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size
forwardMap map[common.Pgid]uint64 // key is start pgid, value is its span size
backwardMap map[common.Pgid]uint64 // key is end pgid, value is its span size
freePagesCount uint64 // count of free pages(hashmap version)
allocate func(txid common.Txid, n int) common.Pgid // the freelist allocate func
free_count func() int // the function which gives you free page number
mergeSpans func(ids common.Pgids) // the mergeSpan func
Expand Down
14 changes: 13 additions & 1 deletion freelist_hmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,15 @@ import (

// hashmapFreeCount returns count of free pages(hashmap version)
func (f *freelist) hashmapFreeCount() int {
// use the forwardMap to get the total count
common.Verify(func() {
expectedFreePageCount := f.hashmapFreeCountSlow()
common.Assert(int(f.freePagesCount) == expectedFreePageCount,
"freePagesCount (%d) is out of sync with free pages map (%d)", f.freePagesCount, expectedFreePageCount)
})
return int(f.freePagesCount)
}

func (f *freelist) hashmapFreeCountSlow() int {
count := 0
for _, size := range f.forwardMap {
count += int(size)
Expand Down Expand Up @@ -142,6 +150,7 @@ func (f *freelist) addSpan(start common.Pgid, size uint64) {
}

f.freemaps[size][start] = struct{}{}
f.freePagesCount += size
}

func (f *freelist) delSpan(start common.Pgid, size uint64) {
Expand All @@ -151,6 +160,7 @@ func (f *freelist) delSpan(start common.Pgid, size uint64) {
if len(f.freemaps[size]) == 0 {
delete(f.freemaps, size)
}
f.freePagesCount -= size
}

// initial from pgids using when use hashmap version
Expand All @@ -162,6 +172,8 @@ func (f *freelist) init(pgids []common.Pgid) {

size := uint64(1)
start := pgids[0]
// reset the counter when freelist init
f.freePagesCount = 0

if !sort.SliceIsSorted([]common.Pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) {
panic("pgids not sorted")
Expand Down
1 change: 1 addition & 0 deletions freelist_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,7 @@ func Test_freelist_hashmapGetFreePageIDs(t *testing.T) {
val = rand.Int31n(1000)
fm[common.Pgid(i)] = uint64(val)
i += val
f.freePagesCount += uint64(val)
}

f.forwardMap = fm
Expand Down
7 changes: 0 additions & 7 deletions internal/common/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,6 @@ import (
"unsafe"
)

// Assert will panic with a given formatted message if the given condition is false.
func Assert(condition bool, msg string, v ...interface{}) {
if !condition {
panic(fmt.Sprintf("assertion failed: "+msg, v...))
}
}

func LoadBucket(buf []byte) *InBucket {
return (*InBucket)(unsafe.Pointer(&buf[0]))
}
Expand Down
67 changes: 67 additions & 0 deletions internal/common/verify.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copied from https://github.com/etcd-io/etcd/blob/main/client/pkg/verify/verify.go
package common

import (
"fmt"
"os"
"strings"
)

const ENV_VERIFY = "BBOLT_VERIFY"

type VerificationType string

const (
ENV_VERIFY_VALUE_ALL VerificationType = "all"
ENV_VERIFY_VALUE_ASSERT VerificationType = "assert"
)

func getEnvVerify() string {
return strings.ToLower(os.Getenv(ENV_VERIFY))
}

func IsVerificationEnabled(verification VerificationType) bool {
env := getEnvVerify()
return env == string(ENV_VERIFY_VALUE_ALL) || env == strings.ToLower(string(verification))
}

// EnableVerifications sets `ENV_VERIFY` and returns a function that
// can be used to bring the original settings.
func EnableVerifications(verification VerificationType) func() {
previousEnv := getEnvVerify()
os.Setenv(ENV_VERIFY, string(verification))
return func() {
os.Setenv(ENV_VERIFY, previousEnv)
}
}

// EnableAllVerifications enables verification and returns a function
// that can be used to bring the original settings.
func EnableAllVerifications() func() {
return EnableVerifications(ENV_VERIFY_VALUE_ALL)
}

// DisableVerifications unsets `ENV_VERIFY` and returns a function that
// can be used to bring the original settings.
func DisableVerifications() func() {
previousEnv := getEnvVerify()
os.Unsetenv(ENV_VERIFY)
return func() {
os.Setenv(ENV_VERIFY, previousEnv)
}
}

// Verify performs verification if the assertions are enabled.
// In the default setup running in tests and skipped in the production code.
func Verify(f func()) {
if IsVerificationEnabled(ENV_VERIFY_VALUE_ASSERT) {
f()
}
}

// Assert will panic with a given formatted message if the given condition is false.
func Assert(condition bool, msg string, v ...any) {
if !condition {
panic(fmt.Sprintf("assertion failed: "+msg, v...))
}
}
Loading