runtime: define maximum supported physical page and huge page sizes

This change defines a maximum supported physical and huge page size in
the runtime based on the new page allocator's implementation, and uses
them where appropriate.

Furthemore, if the system exceeds the maximum supported huge page
size, we simply ignore it silently.

It also fixes a huge-page-related test which is only triggered by a
condition which is definitely wrong.

Finally, it adds a few TODOs related to code clean-up and supporting
larger huge page sizes.

Updates #35112.
Fixes #35431.

Change-Id: Ie4348afb6bf047cce2c1433576d1514720d8230f
Reviewed-on: https://go-review.googlesource.com/c/go/+/205937
Run-TryBot: Michael Knyszek <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Keith Randall <[email protected]>
Reviewed-by: Cherry Zhang <[email protected]>

src/runtime/malloc.go

@@ -436,6 +436,10 @@ func mallocinit() {
 		// The OS init code failed to fetch the physical page size.
 		throw("failed to get system page size")
 	}
+	if physPageSize > maxPhysPageSize {
+		print("system page size (", physPageSize, ") is larger than maximum page size (", maxPhysPageSize, ")\n")
+		throw("bad system page size")
+	}
 	if physPageSize < minPhysPageSize {
 		print("system page size (", physPageSize, ") is smaller than minimum page size (", minPhysPageSize, ")\n")
 		throw("bad system page size")
@@ -448,6 +452,13 @@ func mallocinit() {
 		print("system huge page size (", physHugePageSize, ") must be a power of 2\n")
 		throw("bad system huge page size")
 	}
+	if physHugePageSize > maxPhysHugePageSize {
+		// physHugePageSize is greater than the maximum supported huge page size.
+		// Don't throw here, like in the other cases, since a system configured
+		// in this way isn't wrong, we just don't have the code to support them.
+		// Instead, silently set the huge page size to zero.
+		physHugePageSize = 0
+	}
 	if physHugePageSize != 0 {
 		// Since physHugePageSize is a power of 2, it suffices to increase
 		// physHugePageShift until 1<<physHugePageShift == physHugePageSize.

src/runtime/mgcscavenge.go

@@ -76,6 +76,10 @@ const (
 	// incurs an additional cost), to account for heap fragmentation and
 	// the ever-changing layout of the heap.
 	retainExtraPercent = 10
+
+	// maxPagesPerPhysPage is the maximum number of supported runtime pages per
+	// physical page, based on maxPhysPageSize.
+	maxPagesPerPhysPage = maxPhysPageSize / pageSize
 )
 
 // heapRetained returns an estimate of the current heap RSS.
@@ -498,7 +502,7 @@ func (s *pageAlloc) scavengeRangeLocked(ci chunkIdx, base, npages uint) {
 //
 // Note that if m == 1, this is a no-op.
 //
-// m must be a power of 2 <= 64.
+// m must be a power of 2 <= maxPagesPerPhysPage.
 func fillAligned(x uint64, m uint) uint64 {
 	apply := func(x uint64, c uint64) uint64 {
 		// The technique used it here is derived from
@@ -533,8 +537,10 @@ func fillAligned(x uint64, m uint) uint64 {
 		x = apply(x, 0x7fff7fff7fff7fff)
 	case 32:
 		x = apply(x, 0x7fffffff7fffffff)
-	case 64:
+	case 64: // == maxPagesPerPhysPage
 		x = apply(x, 0x7fffffffffffffff)
+	default:
+		throw("bad m value")
 	}
 	// Now, the top bit of each m-aligned group in x is set
 	// that group was all zero in the original x.
@@ -552,14 +558,14 @@ func fillAligned(x uint64, m uint) uint64 {
 // min pages of free-and-unscavenged memory in the region represented by this
 // pallocData.
 //
-// min must be a non-zero power of 2 <= 64.
+// min must be a non-zero power of 2 <= maxPagesPerPhysPage.
 func (m *pallocData) hasScavengeCandidate(min uintptr) bool {
 	if min&(min-1) != 0 || min == 0 {
 		print("runtime: min = ", min, "\n")
 		throw("min must be a non-zero power of 2")
-	} else if min > 64 {
+	} else if min > maxPagesPerPhysPage {
 		print("runtime: min = ", min, "\n")
-		throw("physical page sizes > 512 KiB are not supported")
+		throw("min too large")
 	}
 
 	// The goal of this search is to see if the chunk contains any free and unscavenged memory.
@@ -590,7 +596,7 @@ func (m *pallocData) hasScavengeCandidate(min uintptr) bool {
 // min indicates a hard minimum size and alignment for runs of pages. That is,
 // findScavengeCandidate will not return a region smaller than min pages in size,
 // or that is min pages or greater in size but not aligned to min. min must be
-// a non-zero power of 2 <= 64.
+// a non-zero power of 2 <= maxPagesPerPhysPage.
 //
 // max is a hint for how big of a region is desired. If max >= pallocChunkPages, then
 // findScavengeCandidate effectively returns entire free and unscavenged regions.
@@ -603,9 +609,9 @@ func (m *pallocData) findScavengeCandidate(searchIdx uint, min, max uintptr) (ui
 	if min&(min-1) != 0 || min == 0 {
 		print("runtime: min = ", min, "\n")
 		throw("min must be a non-zero power of 2")
-	} else if min > 64 {
+	} else if min > maxPagesPerPhysPage {
 		print("runtime: min = ", min, "\n")
-		throw("physical page sizes > 512 KiB are not supported")
+		throw("min too large")
 	}
 	// max is allowed to be less than min, but we need to ensure
 	// we never truncate further than min.
@@ -660,6 +666,11 @@ func (m *pallocData) findScavengeCandidate(searchIdx uint, min, max uintptr) (ui
 	}
 	start := end - size
 
+	// Each huge page is guaranteed to fit in a single palloc chunk.
+	//
+	// TODO(mknyszek): Support larger huge page sizes.
+	// TODO(mknyszek): Consider taking pages-per-huge-page as a parameter
+	// so we can write tests for this.
 	if physHugePageSize > pageSize && physHugePageSize > physPageSize {
 		// We have huge pages, so let's ensure we don't break one by scavenging
 		// over a huge page boundary. If the range [start, start+size) overlaps with

src/runtime/mgcscavenge_test.go

@@ -229,8 +229,8 @@ func TestPallocDataFindScavengeCandidate(t *testing.T) {
 			max:   3, // Make it so that max would have us try to break the huge page.
 			want:  BitRange{0, bits + 2},
 		}
-		if bits >= 3*PallocChunkPages {
-			// We need at least 3 huge pages in an arena for this test to make sense.
+		if 3*bits < PallocChunkPages {
+			// We need at least 3 huge pages in a chunk for this test to make sense.
 			tests["PreserveHugePageMiddle"] = test{
 				alloc: []BitRange{{0, bits - 10}, {2*bits + 10, PallocChunkPages - (2*bits + 10)}},
 				min:   1,

src/runtime/mheap.go

@@ -15,10 +15,19 @@ import (
 	"unsafe"
 )
 
-// minPhysPageSize is a lower-bound on the physical page size. The
-// true physical page size may be larger than this. In contrast,
-// sys.PhysPageSize is an upper-bound on the physical page size.
-const minPhysPageSize = 4096
+const (
+	// minPhysPageSize is a lower-bound on the physical page size. The
+	// true physical page size may be larger than this. In contrast,
+	// sys.PhysPageSize is an upper-bound on the physical page size.
+	minPhysPageSize = 4096
+
+	// maxPhysPageSize is the maximum page size the runtime supports.
+	maxPhysPageSize = 512 << 10
+
+	// maxPhysHugePageSize sets an upper-bound on the maximum huge page size
+	// that the runtime supports.
+	maxPhysHugePageSize = pallocChunkBytes
+)
 
 // Main malloc heap.
 // The heap itself is the "free" and "scav" treaps,