From 94e44a9c8edb64f514b6f3b7f7001db0cfeb2d70 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Tue, 18 Apr 2017 15:23:24 -0700 Subject: [PATCH] runtime: preallocate some overflow buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When allocating a non-small array of buckets for a map, also preallocate some overflow buckets. The estimate of the number of overflow buckets is based on a simulation of putting mid=(low+high)/2 elements into a map, where low is the minimum number of elements needed to reach this value of b (according to overLoadFactor), and high is the maximum number of elements possible to put in this value of b (according to overLoadFactor). This estimate is surprisingly reliable and accurate. The number of overflow buckets needed is quadratic, for a fixed value of b. Using this mid estimate means that we will overallocate a few too many overflow buckets when the actual number of elements is near low, and underallocate significantly too few overflow buckets when the actual number of elements is near high. The mechanism introduced in this CL can be re-used for other overflow bucket optimizations. For example, given an initial size hint, we could estimate quite precisely the number of overflow buckets. This is #19931. We could also change from "non-nil means end-of-list" to "pointer-to-hmap.buckets means end-of-list", and then create a linked list of reusable overflow buckets when they are freed by map growth. That is #19992. We could also use a similar mechanism to do bulk allocation of overflow buckets. All these uses can co-exist with only the one additional pointer in mapextra, given a little care. name old time/op new time/op delta MapPopulate/1-8 60.1ns ± 2% 60.3ns ± 2% ~ (p=0.278 n=19+20) MapPopulate/10-8 577ns ± 1% 578ns ± 1% ~ (p=0.140 n=20+20) MapPopulate/100-8 8.06µs ± 1% 8.19µs ± 1% +1.67% (p=0.000 n=20+20) MapPopulate/1000-8 104µs ± 1% 104µs ± 1% ~ (p=0.317 n=20+20) MapPopulate/10000-8 891µs ± 1% 888µs ± 1% ~ (p=0.101 n=19+20) MapPopulate/100000-8 8.61ms ± 1% 8.58ms ± 0% -0.34% (p=0.009 n=20+17) name old alloc/op new alloc/op delta MapPopulate/1-8 0.00B 0.00B ~ (all equal) MapPopulate/10-8 179B ± 0% 179B ± 0% ~ (all equal) MapPopulate/100-8 3.33kB ± 0% 3.38kB ± 0% +1.48% (p=0.000 n=20+16) MapPopulate/1000-8 55.5kB ± 0% 53.4kB ± 0% -3.84% (p=0.000 n=19+20) MapPopulate/10000-8 432kB ± 0% 428kB ± 0% -1.06% (p=0.000 n=19+20) MapPopulate/100000-8 3.65MB ± 0% 3.62MB ± 0% -0.70% (p=0.000 n=20+20) name old allocs/op new allocs/op delta MapPopulate/1-8 0.00 0.00 ~ (all equal) MapPopulate/10-8 1.00 ± 0% 1.00 ± 0% ~ (all equal) MapPopulate/100-8 18.0 ± 0% 17.0 ± 0% -5.56% (p=0.000 n=20+20) MapPopulate/1000-8 96.0 ± 0% 72.6 ± 1% -24.38% (p=0.000 n=20+20) MapPopulate/10000-8 625 ± 0% 319 ± 0% -48.86% (p=0.000 n=20+20) MapPopulate/100000-8 6.23k ± 0% 4.00k ± 0% -35.79% (p=0.000 n=20+20) Change-Id: I01f41cb1374bdb99ccedbc00d04fb9ae43daa204 Reviewed-on: https://go-review.googlesource.com/40979 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/hashmap.go | 71 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/src/runtime/hashmap.go b/src/runtime/hashmap.go index 5243236f016230..0c9505e655be64 100644 --- a/src/runtime/hashmap.go +++ b/src/runtime/hashmap.go @@ -130,6 +130,9 @@ type mapextra struct { // overflow[1] contains overflow buckets for hmap.oldbuckets. // The indirection allows to store a pointer to the slice in hiter. overflow [2]*[]*bmap + + // nextOverflow holds a pointer to a free overflow bucket. + nextOverflow *bmap } // A bucket for a Go map. @@ -205,7 +208,24 @@ func (h *hmap) incrnoverflow() { } func (h *hmap) newoverflow(t *maptype, b *bmap) *bmap { - ovf := (*bmap)(newobject(t.bucket)) + var ovf *bmap + if h.extra != nil && h.extra.nextOverflow != nil { + // We have preallocated overflow buckets available. + // See makeBucketArray for more details. + ovf = h.extra.nextOverflow + if ovf.overflow(t) == nil { + // We're not at the end of the preallocated overflow buckets. Bump the pointer. + h.extra.nextOverflow = (*bmap)(add(unsafe.Pointer(ovf), uintptr(t.bucketsize))) + } else { + // This is the last preallocated overflow bucket. + // Reset the overflow pointer on this bucket, + // which was set to a non-nil sentinel value. + ovf.setoverflow(t, nil) + h.extra.nextOverflow = nil + } + } else { + ovf = (*bmap)(newobject(t.bucket)) + } h.incrnoverflow() if t.bucket.kind&kindNoPointers != 0 { h.createOverflow() @@ -287,8 +307,14 @@ func makemap(t *maptype, hint int64, h *hmap, bucket unsafe.Pointer) *hmap { // if B == 0, the buckets field is allocated lazily later (in mapassign) // If hint is large zeroing this memory could take a while. buckets := bucket + var extra *mapextra if B != 0 { - buckets = newarray(t.bucket, 1<= 4 { + // Add on the estimated number of overflow buckets + // required to insert the median number of elements + // used with this value of b. + nbuckets += 1 << (b - 4) + sz := t.bucket.size * nbuckets + up := roundupsize(sz) + if up != sz { + nbuckets = up / t.bucket.size + } + } + buckets = newarray(t.bucket, int(nbuckets)) + if base != nbuckets { + // We preallocated some overflow buckets. + // To keep the overhead of tracking these overflow buckets to a minimum, + // we use the convention that if a preallocated overflow bucket's overflow + // pointer is nil, then there are more available by bumping the pointer. + // We need a safe non-nil pointer for the last overflow bucket; just use buckets. + nextOverflow = (*bmap)(add(buckets, base*uintptr(t.bucketsize))) + last := (*bmap)(add(buckets, (nbuckets-1)*uintptr(t.bucketsize))) + last.setoverflow(t, (*bmap)(buckets)) + } + return buckets, nextOverflow +} + func hashGrow(t *maptype, h *hmap) { // If we've hit the load factor, get bigger. // Otherwise, there are too many overflow buckets, @@ -893,7 +949,8 @@ func hashGrow(t *maptype, h *hmap) { h.flags |= sameSizeGrow } oldbuckets := h.buckets - newbuckets := newarray(t.bucket, 1<<(h.B+bigger)) + newbuckets, nextOverflow := makeBucketArray(t, h.B+bigger) + flags := h.flags &^ (iterator | oldIterator) if h.flags&iterator != 0 { flags |= oldIterator @@ -914,6 +971,12 @@ func hashGrow(t *maptype, h *hmap) { h.extra.overflow[1] = h.extra.overflow[0] h.extra.overflow[0] = nil } + if nextOverflow != nil { + if h.extra == nil { + h.extra = new(mapextra) + } + h.extra.nextOverflow = nextOverflow + } // the actual copying of the hash table data is done incrementally // by growWork() and evacuate().