From cb41c066665a5008f0f64ab3283b0ca373cc0de7 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Fri, 28 Dec 2018 09:59:28 +0100 Subject: [PATCH] Add precise gc implementation --- src/core/bitop.d | 36 ++- src/core/memory.d | 8 +- src/gc/bits.d | 362 +++++++++++++++++++++++++++- src/gc/config.d | 4 +- src/gc/impl/conservative/gc.d | 439 ++++++++++++++++++++++++++++------ src/gc/proxy.d | 7 +- src/object.d | 28 ++- src/rt/aaA.d | 87 ++++++- src/rt/dmain2.d | 5 +- src/rt/lifetime.d | 4 +- src/rt/typeinfo/ti_byte.d | 2 + src/rt/typeinfo/ti_cdouble.d | 2 + src/rt/typeinfo/ti_cfloat.d | 2 + src/rt/typeinfo/ti_char.d | 2 + src/rt/typeinfo/ti_creal.d | 2 + src/rt/typeinfo/ti_dchar.d | 2 + src/rt/typeinfo/ti_delegate.d | 2 + src/rt/typeinfo/ti_double.d | 2 + src/rt/typeinfo/ti_float.d | 2 + src/rt/typeinfo/ti_int.d | 2 + src/rt/typeinfo/ti_long.d | 2 + src/rt/typeinfo/ti_ptr.d | 2 + src/rt/typeinfo/ti_real.d | 2 + src/rt/typeinfo/ti_short.d | 2 + src/rt/typeinfo/ti_ubyte.d | 2 + src/rt/typeinfo/ti_uint.d | 2 + src/rt/typeinfo/ti_ulong.d | 2 + src/rt/typeinfo/ti_ushort.d | 2 + src/rt/typeinfo/ti_wchar.d | 2 + 29 files changed, 918 insertions(+), 100 deletions(-) diff --git a/src/core/bitop.d b/src/core/bitop.d index 0daee55c7e9..fc4d7ef0083 100644 --- a/src/core/bitop.d +++ b/src/core/bitop.d @@ -396,6 +396,32 @@ struct BitRange } } + /** + * Construct a BitRange. + * + * Params: + * bitarr = The array of bits to iterate over + * numBits = The total number of valid bits in the given bit array + * startBit = The initial start index to start searching + */ + this(const(size_t)* bitarr, size_t numBits, size_t startBit) @system + { + assert(startBit <= numBits); + bits = bitarr; + len = numBits; + idx = startBit; + if (len) + { + bits += idx / bitsPerWord; + auto curbit = idx % bitsPerWord; + // prime the first bit + cur = *bits++ ^ (size_t(1) << curbit); + if (curbit) + cur &= ~((size_t(1) << curbit) - 1); // clear skipped lower bits + popFront(); + } + } + /// Range functions size_t front() { @@ -453,6 +479,11 @@ struct BitRange bts(bitArr, 95); bts(bitArr, 78); + assert(BitRange(bitArr, 100).front() == 24); + + bts(bitArr, 0); + assert(BitRange(bitArr, 100).front() == 0); + enum sum = 48 + 24 + 95 + 78; // iterate @@ -465,7 +496,10 @@ struct BitRange } assert(testSum == sum); - assert(nBits == 4); + assert(nBits == 5); + + assert(BitRange(bitArr, 100, 50).front() == 78); + assert(BitRange(bitArr, 100, 48).front() == 48); } @system unittest diff --git a/src/core/memory.d b/src/core/memory.d index 5d35b78daa4..1be3df0ee93 100644 --- a/src/core/memory.d +++ b/src/core/memory.d @@ -239,7 +239,9 @@ struct GC Example: ---- // Allocate the underlying array. - int* pToArray = cast(int*)GC.malloc(10 * int.sizeof, GC.BlkAttr.NO_SCAN | GC.BlkAttr.APPENDABLE); + BlkInfo info = GC.qalloc(10 * int.sizeof, GC.BlkAttr.NO_SCAN | GC.BlkAttr.APPENDABLE); + info.base[0..info.size] = 0; + int* pToArray = cast(int*)info.base; // Bind a slice. Check the slice has capacity information. int[] slice = pToArray[0 .. 0]; assert(capacity(slice) > 0); @@ -248,6 +250,10 @@ struct GC slice ~= 1; assert(slice.ptr == p); ---- + Please be aware that large allocations (> 2 kB) work slightly differently, as the size + of allocated data is stored at the beginning of the allocation instead of the end. + + A precise GC might take advantage of the actual memory layout if APPENDABLE is set. */ APPENDABLE = 0b0000_1000, diff --git a/src/gc/bits.d b/src/gc/bits.d index 21d43c4f4f0..e26ceba32e6 100644 --- a/src/gc/bits.d +++ b/src/gc/bits.d @@ -1,12 +1,12 @@ /** * Contains a bitfield used by the GC. * - * Copyright: Copyright Digital Mars 2005 - 2013. + * Copyright: Copyright Digital Mars 2005 - 2016. * License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). - * Authors: Walter Bright, David Friedman, Sean Kelly + * Authors: Walter Bright, David Friedman, Sean Kelly, Rainer Schuetze */ -/* Copyright Digital Mars 2005 - 2013. +/* Copyright Digital Mars 2005 - 2016. * Distributed under the Boost Software License, Version 1.0. * (See accompanying file LICENSE or copy at * http://www.boost.org/LICENSE_1_0.txt) @@ -19,6 +19,11 @@ import core.stdc.string; import core.stdc.stdlib; import core.exception : onOutOfMemoryError; + +// use version bitwise to disable optimizations that use word operands +// on bulk operation copyRange, setRange, clrRange, etc. +// version = bitwise; + struct GCBits { alias size_t wordtype; @@ -26,7 +31,9 @@ struct GCBits enum BITS_PER_WORD = (wordtype.sizeof * 8); enum BITS_SHIFT = (wordtype.sizeof == 8 ? 6 : 5); enum BITS_MASK = (BITS_PER_WORD - 1); + enum BITS_0 = cast(wordtype)0; enum BITS_1 = cast(wordtype)1; + enum BITS_2 = cast(wordtype)2; wordtype* data; size_t nbits; @@ -78,6 +85,355 @@ struct GCBits return core.bitop.btr(data, i); } + mixin template RangeVars() + { + size_t firstWord = (target >> BITS_SHIFT); + size_t firstOff = target & BITS_MASK; + size_t last = target + len - 1; + size_t lastWord = (last >> BITS_SHIFT); + size_t lastOff = last & BITS_MASK; + } + + // extract loops to allow inlining the rest + void clearWords(size_t firstWord, size_t lastWord) nothrow + { + for (size_t w = firstWord; w < lastWord; w++) + data[w] = 0; + } + + void setWords(size_t firstWord, size_t lastWord) nothrow + { + for (size_t w = firstWord; w < lastWord; w++) + data[w] = ~0; + } + + void copyWords(size_t firstWord, size_t lastWord, const(wordtype)* source) nothrow + { + for (size_t w = firstWord; w < lastWord; w++) + data[w] = source[w - firstWord]; + } + + void copyWordsShifted(size_t firstWord, size_t cntWords, size_t firstOff, const(wordtype)* source) nothrow + { + wordtype mask = ~BITS_0 << firstOff; + data[firstWord] = (data[firstWord] & ~mask) | (source[0] << firstOff); + for (size_t w = 1; w < cntWords; w++) + data[firstWord + w] = (source[w - 1] >> (BITS_PER_WORD - firstOff)) | (source[w] << firstOff); + } + + // target = the biti to start the copy to + // destlen = the number of bits to copy from source + pragma(inline,true) + void copyRange(size_t target, size_t len, const(wordtype)* source) nothrow + { + version (bitwise) + { + for (size_t i = 0; i < len; i++) + if (source[(i >> BITS_SHIFT)] & (BITS_1 << (i & BITS_MASK))) + set(target+i); + else + clear(target+i); + } + else + { + if (len > 0) + copyRangeZ(target, len, source); + } + } + + //pragma(inline,true) + void copyRangeZ(size_t target, size_t len, const(wordtype)* source) nothrow + { + mixin RangeVars!(); + + if (firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] = (data[firstWord] & ~mask) | ((source[0] << firstOff) & mask); + } + else if (firstOff == 0) + { + copyWords(firstWord, lastWord, source); + + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] = (data[lastWord] & ~mask) | (source[lastWord - firstWord] & mask); + } + else + { + size_t cntWords = lastWord - firstWord; + copyWordsShifted(firstWord, cntWords, firstOff, source); + + wordtype src = (source[cntWords - 1] >> (BITS_PER_WORD - firstOff)) | (source[cntWords] << firstOff); + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] = (data[lastWord] & ~mask) | (src & mask); + } + } + + void copyRangeRepeating(size_t target, size_t destlen, const(wordtype)* source, size_t sourcelen) nothrow + { + version (bitwise) + { + for (size_t i=0; i < destlen; i++) + { + bool b; + size_t j = i % sourcelen; + b = (source[j >> BITS_SHIFT] & (BITS_1 << (j & BITS_MASK))) != 0; + if (b) set(target+i); + else clear(target+i); + } + } + else + { + if (destlen > 4 * sourcelen && destlen > 4 * BITS_PER_WORD) + { + // precalculate the number of words where a bit pattern of the + // source length repeats on word alignment + static size_t gcd(size_t a, size_t b) + { + // euclidean algorithm + while (b != 0) + { + auto t = b; + b = a % b; + a = t; + } + return a; + } + // lowest common multiple (with BITS_PER_WORD) + static ubyte lcm(ubyte i) + { + // calc lcm(i,BITS_PER_WORD)/BITS_PER_WORD + return cast(ubyte)(BITS_PER_WORD / gcd(i, BITS_PER_WORD)); + } + static struct ut { unittest { assert(lcm(3) == BITS_PER_WORD); } } + static calcRepLength() + { + ubyte[BITS_PER_WORD] rep; + for (ubyte i = 0; i < BITS_PER_WORD; i++) + rep[i] = lcm(i); + return rep; + } + static immutable repLength = calcRepLength(); + + // make some initial copies until we have a pattern that + // repeats on word boundary + size_t rep = repLength[sourcelen & BITS_MASK]; + size_t repwords = (sourcelen * rep) >> BITS_SHIFT; + size_t alignbits = (target & BITS_MASK ? BITS_PER_WORD - (target & BITS_MASK) : 0); + size_t initbits = BITS_PER_WORD * repwords + alignbits; + + if (initbits < destlen) + { + while (initbits > sourcelen) + { + copyRange(target, sourcelen, source); + target += sourcelen; + destlen -= sourcelen; + initbits -= sourcelen; + } + copyRange(target, initbits, source); + target += initbits; + destlen -= initbits; + assert((target & BITS_MASK) == 0); + + size_t tpos = target >> BITS_SHIFT; + while (destlen >= BITS_PER_WORD) + { + data[tpos] = data[tpos - repwords]; + destlen -= BITS_PER_WORD; + tpos++; + } + + if (destlen > 0) + { + wordtype mask = (BITS_1 << destlen) - 1; + data[tpos] = (data[tpos] & ~mask) | (data[tpos - repwords] & mask); + } + return; + } + } + + while (destlen > sourcelen) + { + copyRange(target, sourcelen, source); + target += sourcelen; + destlen -= sourcelen; + } + copyRange(target, destlen, source); + } + } + + unittest + { + // simulate broken array append test case in vibe.d + GCBits bits; + bits.alloc(10000); + auto data = bits.data; + + GCBits src; + src.alloc(67); + src.data[0] = 0x4; + + bits.copyRangeRepeating(2, 10000, src.data, 67); + + foreach (i; 0 .. 10000) + if ((i - 2) % 67 == 2) + assert(bits.test(i)); + else + assert(!bits.test(i)); + } + + //pragma(inline,true) + void setRange(size_t target, size_t len) nothrow + { + version (bitwise) + { + for (size_t i = 0; i < len; i++) + set(target+i); + } + else + { + if (len > 0) + setRangeZ(target, len); + } + } + + //pragma(inline,true) + void setRangeZ(size_t target, size_t len) nothrow + { + mixin RangeVars!(); + + if (firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] |= mask; + } + else + { + data[firstWord] |= ~BITS_0 << firstOff; + setWords(firstWord + 1, lastWord); + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] |= mask; + } + } + + //pragma(inline,true) + void clrRange(size_t target, size_t len) nothrow + { + version (bitwise) + { + for (size_t i = 0; i < len; i++) + clear(target+i); + } + else + { + if (len > 0) + clrRangeZ(target, len); + } + } + + //pragma(inline,true) + void clrRangeZ(size_t target, size_t len) nothrow + { + mixin RangeVars!(); + if (firstWord == lastWord) + { + wordtype mask = ((BITS_2 << (lastOff - firstOff)) - 1) << firstOff; + data[firstWord] &= ~mask; + } + else + { + data[firstWord] &= ~(~BITS_0 << firstOff); + clearWords(firstWord + 1, lastWord); + wordtype mask = (BITS_2 << lastOff) - 1; + data[lastWord] &= ~mask; + } + } + + unittest + { + GCBits bits; + bits.alloc(1000); + auto data = bits.data; + + bits.setRange(0,1); + assert(data[0] == 1); + + bits.clrRange(0,1); + assert(data[0] == 0); + + bits.setRange(BITS_PER_WORD-1,1); + assert(data[0] == BITS_1 << (BITS_PER_WORD-1)); + + bits.clrRange(BITS_PER_WORD-1,1); + assert(data[0] == 0); + + bits.setRange(12,7); + assert(data[0] == 0b0111_1111_0000_0000_0000); + + bits.clrRange(14,4); + assert(data[0] == 0b0100_0011_0000_0000_0000); + + bits.clrRange(0,BITS_PER_WORD); + assert(data[0] == 0); + + bits.setRange(0,BITS_PER_WORD); + assert(data[0] == ~0); + assert(data[1] == 0); + + bits.setRange(BITS_PER_WORD,BITS_PER_WORD); + assert(data[0] == ~0); + assert(data[1] == ~0); + assert(data[2] == 0); + bits.clrRange(BITS_PER_WORD/2,BITS_PER_WORD); + assert(data[0] == (BITS_1 << (BITS_PER_WORD/2)) - 1); + assert(data[1] == ~data[0]); + assert(data[2] == 0); + + bits.setRange(8*BITS_PER_WORD+1,4*BITS_PER_WORD-2); + assert(data[8] == ~0 << 1); + assert(data[9] == ~0); + assert(data[10] == ~0); + assert(data[11] == cast(wordtype)~0 >> 1); + + bits.clrRange(9*BITS_PER_WORD+1,2*BITS_PER_WORD); + assert(data[8] == ~0 << 1); + assert(data[9] == 1); + assert(data[10] == 0); + assert(data[11] == ((cast(wordtype)~0 >> 1) & ~1)); + + wordtype[4] src = [ 0xa, 0x5, 0xaa, 0x55 ]; + + void testCopyRange(size_t start, size_t len, int repeat = 1) + { + bits.setRange(0, bits.nbits); + if (repeat > 1) + bits.copyRangeRepeating(start, repeat * len, src.ptr, len); + else + bits.copyRange(start, len, src.ptr); + foreach (i; 0 .. start) + assert(bits.test(i)); + foreach (r; 0 .. repeat) + foreach (i; 0 .. len) + assert(!bits.test(start + r*len + i) == !core.bitop.bt(src.ptr, i)); + foreach (i; start + repeat*len .. 10*BITS_PER_WORD) + assert(bits.test(i)); + } + + testCopyRange(20, 10); // short copy range within same word + testCopyRange(50, 20); // short copy range spanning two words + testCopyRange(64, 3 * BITS_PER_WORD + 3); // aligned copy range + testCopyRange(77, 2 * BITS_PER_WORD + 15); // unaligned copy range + testCopyRange(64, 127); // copy range within critical end alignment + + testCopyRange(10, 4, 5); // repeating small range within same word + testCopyRange(20, 5, 10); // repeating small range spanning two words + testCopyRange(40, 21, 7); // repeating medium range + testCopyRange(73, 2 * BITS_PER_WORD + 15, 5); // repeating multi-word range + + testCopyRange(2, 3, 166); // failed with assert + } + void zero() nothrow { memset(data, 0, nwords * wordtype.sizeof); diff --git a/src/gc/config.d b/src/gc/config.d index 1a3631bd887..4306c694cd0 100644 --- a/src/gc/config.d +++ b/src/gc/config.d @@ -16,7 +16,7 @@ struct Config { bool disable; // start disabled ubyte profile; // enable profiling with summary when terminating program - string gc = "conservative"; // select gc implementation conservative|manual + string gc = "precise"; // select gc implementation conservative|precise|manual size_t initReserve; // initial reserve (MB) size_t minPoolSize = 1; // initial and minimum pool size (MB) @@ -36,7 +36,7 @@ struct Config string s = "GC options are specified as whitespace separated assignments: disable:0|1 - start disabled (%d) profile:0|1|2 - enable profiling with summary when terminating program (%d) - gc:conservative|manual - select gc implementation (default = conservative) + gc:conservative|precise|manual - select gc implementation (default = conservative) initReserve:N - initial memory to reserve in MB (%lld) minPoolSize:N - initial and minimum pool size in MB (%lld) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index d7b93b0093f..84d2f0cc89d 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -19,6 +19,7 @@ module gc.impl.conservative.gc; //debug = PRINTF; // turn on printf's //debug = COLLECT_PRINTF; // turn on printf's +//debug = MARK_PRINTF; // turn on printf's //debug = PRINTF_TO_FILE; // redirect printf's ouptut to file "gcx.log" //debug = LOGGING; // log allocations / frees //debug = MEMSTOMP; // stomp on memory @@ -57,6 +58,7 @@ debug(PRINTF_TO_FILE) { private __gshared MonoTime gcStartTick; private __gshared FILE* gcx_fh; + private __gshared bool hadNewline = false; private int printf(ARGS...)(const char* fmt, ARGS args) nothrow { @@ -70,7 +72,7 @@ debug(PRINTF_TO_FILE) { len = fprintf(gcx_fh, "before init: "); } - else + else if (hadNewline) { if (gcStartTick == MonoTime.init) gcStartTick = MonoTime.currTime; @@ -80,6 +82,8 @@ debug(PRINTF_TO_FILE) } len += fprintf(gcx_fh, fmt, args); fflush(gcx_fh); + import core.stdc.string; + hadNewline = fmt && fmt[0] && fmt[strlen(fmt) - 1] == '\n'; return len; } } @@ -243,6 +247,39 @@ debug (LOGGING) /* ============================ GC =============================== */ + +debug(PRINTF) +void printGCBits(GCBits* bits) +{ + for (size_t i = 0; i < bits.nwords; i++) + { + if (i % 32 == 0) printf("\n\t"); + printf("%x ", bits.data[i]); + } + printf("\n"); +} + +// we can assume the name is always from a literal, so it is zero terminated +debug(PRINTF) +string debugTypeName(const(TypeInfo) ti) nothrow +{ + string name; + if (ti is null) + name = "null"; + else if (auto ci = cast(TypeInfo_Class)ti) + name = ci.name; + else if (auto si = cast(TypeInfo_Struct)ti) + name = si.name; + else if (auto ci = cast(TypeInfo_Const)ti) + static if (__traits(compiles,ci.base)) // different whether compiled with object.di or object.d + return debugTypeName(ci.base); + else + return debugTypeName(ci.next); + else + name = ti.classinfo.name; + return name; +} + class ConservativeGC : GC { // For passing to debug code (not thread safe) @@ -254,6 +291,7 @@ class ConservativeGC : GC import core.internal.spinlock; static gcLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy); static bool _inFinalizer; + __gshared bool isPrecise = false; // lock GC, throw InvalidMemoryOperationError on recursive locking during finalization static void lockNR() @nogc nothrow @@ -268,8 +306,11 @@ class ConservativeGC : GC { import core.stdc.string: memcpy; - if (config.gc != "conservative") - return; + if ((config.gc != "precise") && (config.gc != "conservative")) + return; + + if (config.gc == "precise") + isPrecise = true; auto p = cstdlib.malloc(__traits(classInstanceSize,ConservativeGC)); @@ -287,8 +328,8 @@ class ConservativeGC : GC static void finalize(ref GC gc) { - if (config.gc != "conservative") - return; + if ((config.gc != "precise") && (config.gc != "conservative")) + return; auto instance = cast(ConservativeGC) gc; instance.Dtor(); @@ -504,11 +545,13 @@ class ConservativeGC : GC { assert(size != 0); - debug(PRINTF) printf("GC.mallocNoSync(size = %d)\n", size); + debug(PRINTF) + printf("GC::malloc(gcx = %p, size = %d bits = %x, ti = %s)\n", gcx, size, bits, debugTypeName(ti).ptr); + assert(gcx); //debug(PRINTF) printf("gcx.self = %x, pthread_self() = %x\n", gcx.self, pthread_self()); - auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits); + auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits, ti); if (!p) onOutOfMemoryErrorNoGC(); @@ -604,7 +647,7 @@ class ConservativeGC : GC { void *p2; size_t psize; - //debug(PRINTF) printf("GC::realloc(p = %p, size = %zu)\n", p, size); + //debug(PRINTF) printf("GC::realloc(p = %p, size = %llu)\n", p, cast(long)size); debug (SENTINEL) { sentinel_Invariant(p); @@ -719,7 +762,11 @@ class ConservativeGC : GC p = p2; } else + { alloc_size = psize; + if (isPrecise) + pool.setPointerBitmapSmall(p, size, psize, bits, ti); + } } } return p; @@ -1393,7 +1440,8 @@ struct Gcx roots.removeAll(); ranges.removeAll(); - toscan.reset(); + toscanConservative.reset(); + toscanPrecise.reset(); } @@ -1418,8 +1466,12 @@ struct Gcx for (size_t i = 0; i < B_NUMSMALL; i++) { + size_t j = 0; + List* prev, pprev, ppprev; // keep a short history to inspect in the debugger for (auto list = cast(List*)bucket[i]; list; list = list.next) { + ppprev = pprev; pprev = prev; prev = list; + j++; } } } @@ -1689,14 +1741,15 @@ struct Gcx return isLowOnMem(mappedPages * PAGESIZE); } - void* alloc(size_t size, ref size_t alloc_size, uint bits) nothrow + void* alloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow { - return size <= 2048 ? smallAlloc(binTable[size], alloc_size, bits) - : bigAlloc(size, alloc_size, bits); + return size <= PAGESIZE/2 ? smallAlloc(size, alloc_size, bits, ti) + : bigAlloc(size, alloc_size, bits, ti); } - void* smallAlloc(Bins bin, ref size_t alloc_size, uint bits) nothrow + void* smallAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti) nothrow { + immutable bin = binTable[size]; alloc_size = binsize[bin]; void* p; @@ -1743,6 +1796,14 @@ struct Gcx pool.setBits((p - pool.baseAddr) >> pool.shiftBy, bits); //debug(PRINTF) printf("\tmalloc => %p\n", p); debug (MEMSTOMP) memset(p, 0xF0, alloc_size); + + if (ConservativeGC.isPrecise) + { + debug(SENTINEL) + pool.setPointerBitmapSmall(sentinel_add(p), size - SENTINEL_EXTRA, size - SENTINEL_EXTRA, bits, ti); + else + pool.setPointerBitmapSmall(p, size, alloc_size, bits, ti); + } return p; } @@ -1826,6 +1887,20 @@ struct Gcx if (bits) pool.setBits(pn, bits); + + if (ConservativeGC.isPrecise) + { + // an array of classes is in fact an array of pointers + immutable(void)* rtinfo; + if (!ti) + rtinfo = rtinfoHasPointers; + else if ((bits & BlkAttr.APPENDABLE) && (typeid(ti) is typeid(TypeInfo_Class))) + rtinfo = rtinfoHasPointers; + else + rtinfo = ti.rtInfo(); + pool.rtinfo[pn] = cast(immutable(size_t)*)rtinfo; + } + return p; } @@ -1908,13 +1983,19 @@ struct Gcx return null; } - static struct ScanRange + static struct ScanRange(bool precise) { void* pbot; void* ptop; + static if (precise) + { + void** pbase; // start of memory described by ptrbitmap + size_t* ptrbmp; // bits from is_pointer or rtinfo + size_t bmplength; // number of valid bits + } } - static struct ToScanStack + static struct ToScanStack(RANGE) { nothrow: @disable this(this); @@ -1922,25 +2003,28 @@ struct Gcx void reset() { _length = 0; - os_mem_unmap(_p, _cap * ScanRange.sizeof); - _p = null; + if (_p) + { + os_mem_unmap(_p, _cap * RANGE.sizeof); + _p = null; + } _cap = 0; } - void push(ScanRange rng) + void push(RANGE rng) { if (_length == _cap) grow(); _p[_length++] = rng; } - ScanRange pop() + RANGE pop() in { assert(!empty); } do { return _p[--_length]; } - ref inout(ScanRange) opIndex(size_t idx) inout + ref inout(RANGE) opIndex(size_t idx) inout in { assert(idx < _length); } do { @@ -1956,40 +2040,52 @@ struct Gcx pragma(inline, false); enum initSize = 64 * 1024; // Windows VirtualAlloc granularity - immutable ncap = _cap ? 2 * _cap : initSize / ScanRange.sizeof; - auto p = cast(ScanRange*)os_mem_map(ncap * ScanRange.sizeof); + immutable ncap = _cap ? 2 * _cap : initSize / RANGE.sizeof; + auto p = cast(RANGE*)os_mem_map(ncap * RANGE.sizeof); if (p is null) onOutOfMemoryErrorNoGC(); if (_p !is null) { p[0 .. _length] = _p[0 .. _length]; - os_mem_unmap(_p, _cap * ScanRange.sizeof); + os_mem_unmap(_p, _cap * RANGE.sizeof); } _p = p; _cap = ncap; } size_t _length; - ScanRange* _p; + RANGE* _p; size_t _cap; } - ToScanStack toscan; + ToScanStack!(ScanRange!false) toscanConservative; + ToScanStack!(ScanRange!true) toscanPrecise; /** * Search a range of memory values and mark any pointers into the GC pool. */ - void mark(void *pbot, void *ptop) scope nothrow + void mark(bool precise)(void *pbot, void *ptop) scope nothrow { + static if (precise) + alias toscan = toscanPrecise; + else + alias toscan = toscanConservative; + + debug(MARK_PRINTF) + printf("marking range: [%p..%p] (%#llx)\n", pbot, ptop, cast(long)(ptop - pbot)); + if (pbot >= ptop) return; - void **p1 = cast(void **)pbot; - void **p2 = cast(void **)ptop; + ScanRange!precise rng = void; + rng.pbot = cast(void **)pbot; + rng.ptop = cast(void **)ptop; + static if (precise) + rng.pbase = null; // always starting from a non-heap root // limit the amount of ranges added to the toscan stack enum FANOUT_LIMIT = 32; size_t stackPos; - ScanRange[FANOUT_LIMIT] stack = void; + ScanRange!precise[FANOUT_LIMIT] stack = void; size_t pcache = 0; @@ -1998,41 +2094,61 @@ struct Gcx const highpool = pooltable.npools - 1; const minAddr = pooltable.minAddr; size_t memSize = pooltable.maxAddr - minAddr; + Pool* pool = null; - void* base = void; - void* top = void; + // properties of allocation pointed to + ScanRange!precise tgt = void; - //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); for (;;) { - auto p = *p1; + auto p = *cast(void**)(rng.pbot); + + debug(MARK_PRINTF) printf("\tmark %p: %p\n", rng.pbot, p); - //if (log) debug(PRINTF) printf("\tmark %p\n", p); if (cast(size_t)(p - minAddr) < memSize && (cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) != pcache) { - Pool* pool = void; - size_t low = 0; - size_t high = highpool; - while (true) + static if (precise) if (rng.pbase) { - size_t mid = (low + high) >> 1; - pool = pools[mid]; - if (p < pool.baseAddr) - high = mid - 1; - else if (p >= pool.topAddr) - low = mid + 1; - else break; - - if (low > high) + size_t bitpos = cast(void**)rng.pbot - rng.pbase; + while (bitpos >= rng.bmplength) + { + bitpos -= rng.bmplength; + rng.pbase += rng.bmplength; + } + import core.bitop; + if (!core.bitop.bt(rng.ptrbmp, bitpos)) + { + debug(MARK_PRINTF) printf("\t\tskipping non-pointer\n"); goto LnextPtr; + } + } + + if (!pool || p < pool.baseAddr || p >= pool.topAddr) + { + size_t low = 0; + size_t high = highpool; + while (true) + { + size_t mid = (low + high) >> 1; + pool = pools[mid]; + if (p < pool.baseAddr) + high = mid - 1; + else if (p >= pool.topAddr) + low = mid + 1; + else break; + + if (low > high) + goto LnextPtr; + } } size_t offset = cast(size_t)(p - pool.baseAddr); size_t biti = void; size_t pn = offset / PAGESIZE; size_t bin = pool.pagetable[pn]; // not Bins to avoid multiple size extension instructions - //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); + debug(MARK_PRINTF) + printf("\t\tfound pool %p, base=%p, pn = %lld, bin = %d\n", pool, pool.baseAddr, cast(long)pn, bin); // Adjust bit to be at start of allocated memory block if (bin < B_PAGE) @@ -2045,8 +2161,14 @@ struct Gcx if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - base = pool.baseAddr + offsetBase; - top = base + binsize[bin]; + tgt.pbot = pool.baseAddr + offsetBase; + tgt.ptop = tgt.pbot + binsize[bin]; + static if (precise) + { + tgt.pbase = cast(void**)pool.baseAddr; + tgt.ptrbmp = pool.is_pointer.data; + tgt.bmplength = size_t.max; // no repetition + } goto LaddRange; } } @@ -2056,18 +2178,18 @@ struct Gcx //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); - base = cast(void*)pcache; + tgt.pbot = cast(void*)pcache; // For the NO_INTERIOR attribute. This tracks whether // the pointer is an interior pointer or points to the // base address of a block. - if (base != sentinel_sub(p) && pool.nointerior.nbits && pool.nointerior.test(biti)) + if (tgt.pbot != sentinel_sub(p) && pool.nointerior.nbits && pool.nointerior.test(biti)) goto LnextPtr; if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - top = base + pool.bPageOffsets[pn] * PAGESIZE; - goto LaddRange; + tgt.ptop = tgt.pbot + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddLargeRange; } } else if (bin == B_PAGEPLUS) @@ -2081,8 +2203,40 @@ struct Gcx if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - base = pool.baseAddr + (pn * PAGESIZE); - top = base + pool.bPageOffsets[pn] * PAGESIZE; + tgt.pbot = pool.baseAddr + (pn * PAGESIZE); + tgt.ptop = tgt.pbot + pool.bPageOffsets[pn] * PAGESIZE; + LaddLargeRange: + static if (precise) + { + auto rtinfo = pool.rtinfo[biti]; + if (rtinfo is rtinfoNoPointers) + goto LnextPtr; // only if inconsistent with noscan + if (rtinfo is rtinfoHasPointers) + { + tgt.pbase = null; // conservative + } + else + { + tgt.ptrbmp = cast(size_t*)rtinfo; + size_t element_size = *tgt.ptrbmp++; + tgt.bmplength = (element_size + (void*).sizeof - 1) / (void*).sizeof; + assert(tgt.bmplength); + + if (pool.appendable.test(biti)) + { + // take advantage of knowing array layout in rt.lifetime + void* arrtop = tgt.pbot + 16 + *cast(size_t*)tgt.pbot; + assert (arrtop > tgt.pbot && arrtop <= tgt.ptop); + tgt.pbot += 16; + tgt.ptop = arrtop; + } + else + { + tgt.ptop = tgt.pbot + element_size; + } + tgt.pbase = cast(void**)tgt.pbot; + } + } goto LaddRange; } } @@ -2093,22 +2247,20 @@ struct Gcx } } LnextPtr: - if (++p1 < p2) + rng.pbot += (void*).sizeof; + if (rng.pbot < rng.ptop) continue; + LnextRange: if (stackPos) { // pop range from local stack and recurse - auto next = &stack[--stackPos]; - p1 = cast(void**)next.pbot; - p2 = cast(void**)next.ptop; + rng = stack[--stackPos]; } else if (!toscan.empty) { // pop range from global stack and recurse - auto next = toscan.pop(); - p1 = cast(void**)next.pbot; - p2 = cast(void**)next.ptop; + rng = toscan.pop(); } else { @@ -2116,32 +2268,43 @@ struct Gcx break; } // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); - pcache = 0; - continue; + goto LcontRange; LaddRange: - if (++p1 < p2) + rng.pbot += (void*).sizeof; + if (rng.pbot < rng.ptop) { if (stackPos < stack.length) { - stack[stackPos].pbot = base; - stack[stackPos].ptop = top; + stack[stackPos] = tgt; stackPos++; continue; } - toscan.push(ScanRange(p1, p2)); + toscan.push(rng); // reverse order for depth-first-order traversal - foreach_reverse (ref rng; stack) - toscan.push(rng); + foreach_reverse (ref range; stack) + toscan.push(range); stackPos = 0; } + LendOfRange: // continue with last found range - p1 = cast(void**)base; - p2 = cast(void**)top; + rng = tgt; + + LcontRange: pcache = 0; } } + void markConservative(void *pbot, void *ptop) scope nothrow + { + mark!false(pbot, ptop); + } + + void markPrecise(void *pbot, void *ptop) scope nothrow + { + mark!true(pbot, ptop); + } + // collection step 1: prepare freebits and mark bits void prepare() nothrow { @@ -2181,20 +2344,20 @@ struct Gcx } // collection step 2: mark roots and heap - void markAll(bool nostack) nothrow + void markAll(alias markFn)(bool nostack) nothrow { if (!nostack) { debug(COLLECT_PRINTF) printf("\tscan stacks.\n"); // Scan stacks and registers for each paused thread - thread_scanAll(&mark); + thread_scanAll(&markFn); } // Scan roots[] debug(COLLECT_PRINTF) printf("\tscan roots[]\n"); foreach (root; roots) { - mark(cast(void*)&root.proot, cast(void*)(&root.proot + 1)); + markFn(cast(void*)&root.proot, cast(void*)(&root.proot + 1)); } // Scan ranges[] @@ -2203,7 +2366,7 @@ struct Gcx foreach (range; ranges) { debug(COLLECT_PRINTF) printf("\t\t%p .. %p\n", range.pbot, range.ptop); - mark(range.pbot, range.ptop); + markFn(range.pbot, range.ptop); } //log--; } @@ -2443,7 +2606,10 @@ struct Gcx start = stop; } - markAll(nostack); + if (ConservativeGC.isPrecise) + markAll!markPrecise(nostack); + else + markAll!markConservative(nostack); thread_processGCMarks(&isMarked); thread_resumeAll(); @@ -2664,6 +2830,7 @@ struct Pool GCBits appendable; // entries that are appendable GCBits nointerior; // interior pointers should be ignored. // Only implemented for large object pools. + GCBits is_pointer; // precise GC only: per-word, not per-block like the rest of them (SmallObjectPool only) size_t npages; size_t freepages; // The number of pages not in use. ubyte* pagetable; @@ -2680,9 +2847,12 @@ struct Pool // This tracks how far back we have to go to find the nearest B_PAGE at // a smaller address than a B_PAGEPLUS. To save space, we use a uint. // This limits individual allocations to 16 terabytes, assuming a 4k - // pagesize. + // pagesize. (LargeObjectPool only) uint* bPageOffsets; + // precise GC: TypeInfo.rtInfo for allocation (LargeObjectPool only) + immutable(size_t)** rtinfo; + // This variable tracks a conservative estimate of where the first free // page in this pool is, so that if a lot of pages towards the beginning // are occupied, we can bypass them in O(1). @@ -2717,6 +2887,21 @@ struct Pool auto nbits = cast(size_t)poolsize >> shiftBy; mark.alloc(nbits); + if (ConservativeGC.isPrecise) + { + if (isLargeObject) + { + rtinfo = cast(immutable(size_t)**)cstdlib.malloc(npages * (size_t*).sizeof); + if (!rtinfo) + onOutOfMemoryErrorNoGC(); + memset(rtinfo, 0, npages * (size_t*).sizeof); + } + else + { + is_pointer.alloc(cast(size_t)poolsize/(void*).sizeof); + is_pointer.clrRange(0, is_pointer.nbits); + } + } // pagetable already keeps track of what's free for the large object // pool. @@ -2774,6 +2959,13 @@ struct Pool cstdlib.free(bPageOffsets); mark.Dtor(); + if (ConservativeGC.isPrecise) + { + if (isLargeObject) + cstdlib.free(rtinfo); + else + is_pointer.Dtor(); + } if (isLargeObject) { nointerior.Dtor(); @@ -2978,6 +3170,95 @@ struct Pool } } } + + pragma(inline,true) + void setPointerBitmapSmall(void* p, size_t s, size_t allocSize, uint attr, const TypeInfo ti) nothrow + { + if (!(attr & BlkAttr.NO_SCAN)) + setPointerBitmap(p, s, allocSize, ti, attr); + } + + pragma(inline,false) + void setPointerBitmap(void* p, size_t s, size_t allocSize, const TypeInfo ti, uint attr) nothrow + { + size_t offset = p - baseAddr; + //debug(PRINTF) printGCBits(&pool.is_pointer); + + debug(PRINTF) + printf("Setting a pointer bitmap for %s at %p + %llu\n", debugTypeName(ti).ptr, p, cast(ulong)s); + + if (ti) + { + if (attr & BlkAttr.APPENDABLE) + { + // an array of classes is in fact an array of pointers + if (typeid(ti) is typeid(TypeInfo_Class)) + goto L_conservative; + s = allocSize; + } + + auto rtInfo = cast(const(size_t)*)ti.rtInfo(); + + if (rtInfo is rtinfoNoPointers) + { + debug(PRINTF) printf("\tCompiler generated rtInfo: no pointers\n"); + is_pointer.clrRange(offset/(void*).sizeof, s/(void*).sizeof); + } + else if (rtInfo is rtinfoHasPointers) + { + debug(PRINTF) printf("\tCompiler generated rtInfo: has pointers\n"); + is_pointer.setRange(offset/(void*).sizeof, s/(void*).sizeof); + } + else + { + const(size_t)* bitmap = cast (size_t*) rtInfo; + //first element of rtInfo is the size of the object the bitmap encodes + size_t element_size = * bitmap; + bitmap++; + size_t tocopy; + if (attr & BlkAttr.APPENDABLE) + { + tocopy = s/(void*).sizeof; + is_pointer.copyRangeRepeating(offset/(void*).sizeof, tocopy, bitmap, element_size/(void*).sizeof); + } + else + { + tocopy = (s < element_size ? s : element_size)/(void*).sizeof; + is_pointer.copyRange(offset/(void*).sizeof, tocopy, bitmap); + } + + debug(PRINTF) printf("\tSetting bitmap for new object (%s)\n\t\tat %p\t\tcopying from %p + %llu: ", + debugTypeName(ti).ptr, p, bitmap, cast(ulong)element_size); + debug(PRINTF) + for (size_t i = 0; i < element_size/((void*).sizeof); i++) + printf("%d", (bitmap[i/(8*size_t.sizeof)] >> (i%(8*size_t.sizeof))) & 1); + debug(PRINTF) printf("\n"); + + if (tocopy * (void*).sizeof < s) // better safe than sorry: if allocated more, assume pointers inside + { + debug(PRINTF) printf(" Appending %d pointer bits\n", s/(void*).sizeof - tocopy); + is_pointer.setRange(offset/(void*).sizeof + tocopy, s/(void*).sizeof - tocopy); + } + } + + if (s < allocSize) + { + offset = (offset + s + (void*).sizeof - 1) & ~((void*).sizeof - 1); + is_pointer.clrRange(offset/(void*).sizeof, (allocSize - s)/(void*).sizeof); + } + } + else + { + L_conservative: + // limit pointers to actual size of allocation? might fail for arrays that append + // without notifying the GC + s = allocSize; + + debug(PRINTF) printf("Allocating a block without TypeInfo\n"); + is_pointer.setRange(offset/(void*).sizeof, s/(void*).sizeof); + } + //debug(PRINTF) printGCBits(&pool.is_pointer); + } } struct LargeObjectPool diff --git a/src/gc/proxy.d b/src/gc/proxy.d index fe74db8e09e..fb30231f415 100644 --- a/src/gc/proxy.d +++ b/src/gc/proxy.d @@ -36,13 +36,18 @@ private extern (C) { + // must be called before any other initialization that might depend on GC configuration + void gc_config() @nogc + { + config.initialize(); + } + void gc_init() { instanceLock.lock(); if (!isInstanceInit) { auto protoInstance = instance; - config.initialize(); ManualGC.initialize(instance); ConservativeGC.initialize(instance); diff --git a/src/object.d b/src/object.d index 4aaf3548ad6..13fc706e55e 100644 --- a/src/object.d +++ b/src/object.d @@ -1306,7 +1306,7 @@ class TypeInfo /** Return info used by the garbage collector to do precise collection. */ - @property immutable(void)* rtInfo() nothrow pure const @safe @nogc { return null; } + @property immutable(void)* rtInfo() nothrow pure const @safe @nogc { return rtinfoHasPointers; } // better safe than sorry } class TypeInfo_Enum : TypeInfo @@ -1502,6 +1502,8 @@ class TypeInfo_Array : TypeInfo arg2 = typeid(void*); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return RTInfo!(void[]); } } class TypeInfo_StaticArray : TypeInfo @@ -1626,6 +1628,9 @@ class TypeInfo_StaticArray : TypeInfo arg1 = typeid(void*); return 0; } + + // just return the rtInfo of the element, we have no generic type T to run RTInfo!T on + override @property immutable(void)* rtInfo() nothrow pure const @safe { return value.rtInfo(); } } class TypeInfo_AssociativeArray : TypeInfo @@ -1752,6 +1757,8 @@ class TypeInfo_Function : TypeInfo return null; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } + TypeInfo next; /** @@ -1843,6 +1850,8 @@ class TypeInfo_Delegate : TypeInfo arg2 = typeid(void*); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return RTInfo!(int delegate()); } } /** @@ -2199,7 +2208,7 @@ class TypeInfo_Struct : TypeInfo uint m_align; - override @property immutable(void)* rtInfo() const { return m_RTInfo; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return m_RTInfo; } version (X86_64) { @@ -4147,11 +4156,24 @@ void __ctfeWrite(scope const(char)[] s) @nogc @safe pure nothrow {} * Create RTInfo for type T */ +template RTInfoImpl(size_t[] pointers) +{ + immutable size_t[pointers.length] data = pointers[]; + immutable RTInfoImpl = data.ptr; +} + template RTInfo(T) { - enum RTInfo = null; + enum RTInfo = RTInfoImpl!(__traits(getPointerBitmap, T)); } +/** +* shortcuts for the precise GC, also generated by the compiler +* used instead of the actual pointer bitmap +*/ +enum immutable(void)* rtinfoNoPointers = null; +enum immutable(void)* rtinfoHasPointers = cast(void*)1; + // Compiler hook into the runtime implementation of array (vector) operations. template _arrayOp(Args...) { diff --git a/src/rt/aaA.d b/src/rt/aaA.d index 11662e7c7b5..286b930d1c2 100644 --- a/src/rt/aaA.d +++ b/src/rt/aaA.d @@ -200,6 +200,14 @@ Bucket[] allocBuckets(size_t dim) @trusted pure nothrow // Entry //------------------------------------------------------------------------------ +immutable bool gc_precise; + +extern(C) void aaa_init() @nogc +{ + import gc.config; + cast() gc_precise = (config.gc == "precise"); +} + private void* allocEntry(in Impl* aa, in void* pkey) { import rt.lifetime : _d_newitemU; @@ -250,12 +258,14 @@ TypeInfo_Struct fakeEntryTI(const TypeInfo keyti, const TypeInfo valti) auto kti = unqualify(keyti); auto vti = unqualify(valti); - if (!hasDtor(kti) && !hasDtor(vti)) + + bool entryHasDtor = hasDtor(kti) || hasDtor(vti); + if (!gc_precise && !entryHasDtor) return null; // save kti and vti after type info for struct enum sizeti = __traits(classInstanceSize, TypeInfo_Struct); - void* p = GC.malloc(sizeti + 2 * (void*).sizeof); + void* p = GC.malloc(sizeti + (gc_precise ? 4 : 2) * (void*).sizeof); import core.stdc.string : memcpy; memcpy(p, typeid(TypeInfo_Struct).initializer().ptr, sizeti); @@ -268,23 +278,86 @@ TypeInfo_Struct fakeEntryTI(const TypeInfo keyti, const TypeInfo valti) static immutable tiName = __MODULE__ ~ ".Entry!(...)"; ti.name = tiName; + if (gc_precise) + { + auto rtinfoData = cast(size_t[2]*) (extra + 2); + ti.m_RTInfo = rtinfoEntry(keyti, valti, *rtinfoData); + ti.m_flags = ti.m_RTInfo is rtinfoNoPointers ? cast(TypeInfo_Struct.StructFlags)0 : TypeInfo_Struct.StructFlags.hasPointers; + } + else + { + ti.m_RTInfo = null; + ti.m_flags = cast(TypeInfo_Struct.StructFlags)(keyti.flags | valti.flags) & TypeInfo_Struct.StructFlags.hasPointers; + } // we don't expect the Entry objects to be used outside of this module, so we have control // over the non-usage of the callback methods and other entries and can keep these null // xtoHash, xopEquals, xopCmp, xtoString and xpostblit - ti.m_RTInfo = null; immutable entrySize = talign(kti.tsize, vti.talign) + vti.tsize; ti.m_init = (cast(ubyte*) null)[0 .. entrySize]; // init length, but not ptr - // xdtor needs to be built from the dtors of key and value for the GC - ti.xdtorti = &entryDtor; + if (entryHasDtor) + { + // xdtor needs to be built from the dtors of key and value for the GC + ti.xdtorti = &entryDtor; + ti.m_flags |= TypeInfo_Struct.StructFlags.isDynamicType; + } - ti.m_flags = TypeInfo_Struct.StructFlags.isDynamicType; - ti.m_flags |= (keyti.flags | valti.flags) & TypeInfo_Struct.StructFlags.hasPointers; ti.m_align = cast(uint) max(kti.talign, vti.talign); return ti; } +// build type info with appropriate RTInfo at runtime +immutable(void)* rtinfoEntry(const TypeInfo keyti, const TypeInfo valti, return ref size_t[2] rtinfoData) +{ + static bool isNoClass(const TypeInfo ti) { return ti && typeid(ti) !is typeid(TypeInfo_Class); } + + immutable(size_t)* keyinfo = cast(immutable(size_t)*) (isNoClass(keyti) ? keyti.rtInfo : rtinfoHasPointers); + immutable(size_t)* valinfo = cast(immutable(size_t)*) (isNoClass(valti) ? valti.rtInfo : rtinfoHasPointers); + + enum maxSupportedSize = 8 * (void*).sizeof * (void*).sizeof; + + size_t rtinfosize = 0; + size_t valuesize = valti.tsize(); + + size_t valbits; + if (valinfo is rtinfoNoPointers) + valbits = 0; + else if (rtinfosize + valuesize > maxSupportedSize) + return rtinfoHasPointers; + else if (valinfo is rtinfoHasPointers) + valbits = (1 << (valuesize / (void*).sizeof)) - 1; + else + valbits = valinfo[1]; + + if (valbits != 0) + rtinfosize += valuesize; + + size_t keybits; + size_t keysize = keyti.tsize; + if (keyinfo is rtinfoNoPointers) + keybits = 0; + else if (rtinfosize + keysize > maxSupportedSize) + return rtinfoHasPointers; + else if (keyinfo is rtinfoHasPointers) + keybits = (1 << (keysize / (void*).sizeof)) - 1; + else + keybits = keyinfo[1]; + + if (valbits == 0 && keybits == 0) + return rtinfoNoPointers; + + if (valbits != 0 || keybits != 0) + rtinfosize += keysize; + + rtinfoData[0] = rtinfosize; + + size_t valshift = (keysize + (void*).sizeof - 1) / (void*).sizeof; + rtinfoData[1] = keybits | (valbits << valshift); + + return cast(immutable(void)*) rtinfoData.ptr; +} + //============================================================================== // Helper functions //------------------------------------------------------------------------------ diff --git a/src/rt/dmain2.d b/src/rt/dmain2.d index 901b9595894..edef6910c55 100644 --- a/src/rt/dmain2.d +++ b/src/rt/dmain2.d @@ -65,11 +65,12 @@ extern (C) void _d_monitor_staticctor(); extern (C) void _d_monitor_staticdtor(); extern (C) void _d_critical_init(); extern (C) void _d_critical_term(); -extern (C) void gc_init(); +extern (C) void gc_config() @nogc; extern (C) void gc_term(); extern (C) void thread_init() @nogc; extern (C) void thread_term() @nogc; extern (C) void lifetime_init(); +extern (C) void aaa_init() @nogc; extern (C) void rt_moduleCtor(); extern (C) void rt_moduleTlsCtor(); extern (C) void rt_moduleDtor(); @@ -203,10 +204,12 @@ extern (C) int rt_init() // this initializes mono time before anything else to allow usage // in other druntime systems. _d_initMonoTime(); + gc_config(); thread_init(); // TODO: fixme - calls GC.addRange -> Initializes GC initStaticDataGC(); lifetime_init(); + aaa_init(); rt_moduleCtor(); rt_moduleTlsCtor(); return 1; diff --git a/src/rt/lifetime.d b/src/rt/lifetime.d index 2f5c99a2bff..79eb073e135 100644 --- a/src/rt/lifetime.d +++ b/src/rt/lifetime.d @@ -425,7 +425,7 @@ BlkInfo __arrayAlloc(size_t arrsize, const TypeInfo ti, const TypeInfo tinext) n uint attr = (!(tinext.flags & 1) ? BlkAttr.NO_SCAN : 0) | BlkAttr.APPENDABLE; if (typeInfoSize) attr |= BlkAttr.STRUCTFINAL | BlkAttr.FINALIZE; - return GC.qalloc(padded_size, attr, ti); + return GC.qalloc(padded_size, attr, tinext); } BlkInfo __arrayAlloc(size_t arrsize, ref BlkInfo info, const TypeInfo ti, const TypeInfo tinext) @@ -442,7 +442,7 @@ BlkInfo __arrayAlloc(size_t arrsize, ref BlkInfo info, const TypeInfo ti, const return BlkInfo(); } - return GC.qalloc(padded_size, info.attr, ti); + return GC.qalloc(padded_size, info.attr, tinext); } /** diff --git a/src/rt/typeinfo/ti_byte.d b/src/rt/typeinfo/ti_byte.d index 2cc3f98388d..c35bc82e940 100644 --- a/src/rt/typeinfo/ti_byte.d +++ b/src/rt/typeinfo/ti_byte.d @@ -58,4 +58,6 @@ class TypeInfo_g : TypeInfo *cast(byte *)p1 = *cast(byte *)p2; *cast(byte *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_cdouble.d b/src/rt/typeinfo/ti_cdouble.d index 939edda5791..ce5764acec6 100644 --- a/src/rt/typeinfo/ti_cdouble.d +++ b/src/rt/typeinfo/ti_cdouble.d @@ -71,4 +71,6 @@ class TypeInfo_r : TypeInfo arg2 = typeid(double); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_cfloat.d b/src/rt/typeinfo/ti_cfloat.d index 6c856310d60..8c29453e304 100644 --- a/src/rt/typeinfo/ti_cfloat.d +++ b/src/rt/typeinfo/ti_cfloat.d @@ -70,4 +70,6 @@ class TypeInfo_q : TypeInfo arg1 = typeid(double); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_char.d b/src/rt/typeinfo/ti_char.d index c804711af2b..9633bb9eaca 100644 --- a/src/rt/typeinfo/ti_char.d +++ b/src/rt/typeinfo/ti_char.d @@ -59,4 +59,6 @@ class TypeInfo_a : TypeInfo return (&c)[0 .. 1]; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_creal.d b/src/rt/typeinfo/ti_creal.d index 5ed045d1ac0..a236f86437f 100644 --- a/src/rt/typeinfo/ti_creal.d +++ b/src/rt/typeinfo/ti_creal.d @@ -71,4 +71,6 @@ class TypeInfo_c : TypeInfo arg2 = typeid(real); return 0; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_dchar.d b/src/rt/typeinfo/ti_dchar.d index 0051dfc4acc..7329946199b 100644 --- a/src/rt/typeinfo/ti_dchar.d +++ b/src/rt/typeinfo/ti_dchar.d @@ -59,4 +59,6 @@ class TypeInfo_w : TypeInfo return (&c)[0 .. 1]; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_delegate.d b/src/rt/typeinfo/ti_delegate.d index fa39593442f..d712be08a92 100644 --- a/src/rt/typeinfo/ti_delegate.d +++ b/src/rt/typeinfo/ti_delegate.d @@ -60,4 +60,6 @@ class TypeInfo_D : TypeInfo { return 1; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return RTInfo!(dg); } } diff --git a/src/rt/typeinfo/ti_double.d b/src/rt/typeinfo/ti_double.d index f9a95a6fff1..e371bc652fa 100644 --- a/src/rt/typeinfo/ti_double.d +++ b/src/rt/typeinfo/ti_double.d @@ -65,6 +65,8 @@ class TypeInfo_d : TypeInfo return F.alignof; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } + version (Windows) { } diff --git a/src/rt/typeinfo/ti_float.d b/src/rt/typeinfo/ti_float.d index 790d759fc02..e161addae2e 100644 --- a/src/rt/typeinfo/ti_float.d +++ b/src/rt/typeinfo/ti_float.d @@ -60,6 +60,8 @@ class TypeInfo_f : TypeInfo return (&r)[0 .. 1]; } + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } + version (Windows) { } diff --git a/src/rt/typeinfo/ti_int.d b/src/rt/typeinfo/ti_int.d index a4c47d3af6f..d3ff4540d10 100644 --- a/src/rt/typeinfo/ti_int.d +++ b/src/rt/typeinfo/ti_int.d @@ -62,4 +62,6 @@ class TypeInfo_i : TypeInfo *cast(int *)p1 = *cast(int *)p2; *cast(int *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_long.d b/src/rt/typeinfo/ti_long.d index 3f8ba8e655d..bc7e79fb537 100644 --- a/src/rt/typeinfo/ti_long.d +++ b/src/rt/typeinfo/ti_long.d @@ -70,4 +70,6 @@ class TypeInfo_l : TypeInfo { return long.alignof; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ptr.d b/src/rt/typeinfo/ti_ptr.d index 453c945d6eb..c27d907dc5a 100644 --- a/src/rt/typeinfo/ti_ptr.d +++ b/src/rt/typeinfo/ti_ptr.d @@ -62,4 +62,6 @@ class TypeInfo_P : TypeInfo } override @property uint flags() nothrow pure const { return 1; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoHasPointers; } } diff --git a/src/rt/typeinfo/ti_real.d b/src/rt/typeinfo/ti_real.d index 6ca240caebd..19839adf7ce 100644 --- a/src/rt/typeinfo/ti_real.d +++ b/src/rt/typeinfo/ti_real.d @@ -64,4 +64,6 @@ class TypeInfo_e : TypeInfo { return F.alignof; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_short.d b/src/rt/typeinfo/ti_short.d index da445c2744d..4a06c6ed35a 100644 --- a/src/rt/typeinfo/ti_short.d +++ b/src/rt/typeinfo/ti_short.d @@ -58,4 +58,6 @@ class TypeInfo_s : TypeInfo *cast(short *)p1 = *cast(short *)p2; *cast(short *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ubyte.d b/src/rt/typeinfo/ti_ubyte.d index 09232fd1c3f..860592fdf8b 100644 --- a/src/rt/typeinfo/ti_ubyte.d +++ b/src/rt/typeinfo/ti_ubyte.d @@ -57,6 +57,8 @@ class TypeInfo_h : TypeInfo *cast(ubyte *)p1 = *cast(ubyte *)p2; *cast(ubyte *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } class TypeInfo_b : TypeInfo_h diff --git a/src/rt/typeinfo/ti_uint.d b/src/rt/typeinfo/ti_uint.d index 8f8e42fa919..3ae043277de 100644 --- a/src/rt/typeinfo/ti_uint.d +++ b/src/rt/typeinfo/ti_uint.d @@ -61,4 +61,6 @@ class TypeInfo_k : TypeInfo *cast(uint *)p1 = *cast(uint *)p2; *cast(uint *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ulong.d b/src/rt/typeinfo/ti_ulong.d index 7382e487adb..6b067f2c496 100644 --- a/src/rt/typeinfo/ti_ulong.d +++ b/src/rt/typeinfo/ti_ulong.d @@ -70,4 +70,6 @@ class TypeInfo_m : TypeInfo { return ulong.alignof; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_ushort.d b/src/rt/typeinfo/ti_ushort.d index 3a95eb4646d..f90b4ab9722 100644 --- a/src/rt/typeinfo/ti_ushort.d +++ b/src/rt/typeinfo/ti_ushort.d @@ -57,4 +57,6 @@ class TypeInfo_t : TypeInfo *cast(ushort *)p1 = *cast(ushort *)p2; *cast(ushort *)p2 = t; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } } diff --git a/src/rt/typeinfo/ti_wchar.d b/src/rt/typeinfo/ti_wchar.d index a1ba04196ff..7d278abaac0 100644 --- a/src/rt/typeinfo/ti_wchar.d +++ b/src/rt/typeinfo/ti_wchar.d @@ -59,4 +59,6 @@ class TypeInfo_u : TypeInfo return (&c)[0 .. 1]; } + + override @property immutable(void)* rtInfo() nothrow pure const @safe { return rtinfoNoPointers; } }