From 5e55aca2ae6a7223aceab4b429620e6f50be753d Mon Sep 17 00:00:00 2001 From: Cheng Jin Date: Fri, 28 Oct 2022 16:00:07 -0400 Subject: [PATCH] [Jtreg/FFI]Fix the OOM issue with thunk allocation The changes aim to address the OOM issue with thunk allocation on the platforms with a small page size (e.g. 4KB) by maintaining a thunk heap list in which case a new thunk heap is created only when there is no free thunk heap available in the list to allocate thunk. Signed-off-by: Cheng Jin --- .../foreign/abi/InternalUpcallHandler.java | 41 +---- .../jdk_internal_foreign_abi_UpcallStubs.cpp | 33 ++-- runtime/oti/j9nonbuilder.h | 24 +-- runtime/oti/jclprots.h | 2 +- ...rnal_foreign_abi_InternalUpcallHandler.cpp | 2 +- runtime/vm/UpcallThunkMem.cpp | 151 ++++++++++++------ runtime/vm/j9vm.tdf | 2 + runtime/vm/jvminit.c | 42 +++-- runtime/vm/vmthinit.c | 8 +- 9 files changed, 178 insertions(+), 127 deletions(-) diff --git a/jcl/src/java.base/share/classes/openj9/internal/foreign/abi/InternalUpcallHandler.java b/jcl/src/java.base/share/classes/openj9/internal/foreign/abi/InternalUpcallHandler.java index 44186ecafda..f085de39dd5 100644 --- a/jcl/src/java.base/share/classes/openj9/internal/foreign/abi/InternalUpcallHandler.java +++ b/jcl/src/java.base/share/classes/openj9/internal/foreign/abi/InternalUpcallHandler.java @@ -55,14 +55,6 @@ public final class InternalUpcallHandler { private final long thunkAddr; private UpcallMHMetaData metaData; - /* The generated thunk address are cached & shared in multiple upcalls/threads within the same session/scope */ - private static final HashMap cachedUpcallThunkAddr = new HashMap<>(); - - private static final class PrivateUpcallClassLock { - PrivateUpcallClassLock() {} - } - private static final Object privateUpcallClassLock = new PrivateUpcallClassLock(); - /*[IF JAVA_SPEC_VERSION >= 19]*/ /** * The constructor creates an upcall handler specific to the requested java method @@ -131,7 +123,7 @@ private long getUpcallThunkAddr(MethodHandle target, ResourceScope sessionOrScop /*[ENDIF] JAVA_SPEC_VERSION >= 19 */ { int argLayoutCount = argLayoutArray.length; - /* The last element of the native signature array is for the return type */ + /* The last element of the native signature array is for the return type. */ String[] nativeSignatureStrs = new String[argLayoutCount + 1]; for (int argIndex = 0; argIndex < argLayoutCount; argIndex++) { MemoryLayout argLayout = argLayoutArray[argIndex]; @@ -148,31 +140,12 @@ private long getUpcallThunkAddr(MethodHandle target, ResourceScope sessionOrScop nativeSignatureStrs[argLayoutCount] = LayoutStrPreprocessor.getSimplifiedLayoutString(realReturnLayout, false); } - long addr = 0; - synchronized(privateUpcallClassLock) { - /* The generated thunks are shared across upcalls or threads only when - * the target method handles are identical within the same session/scope - * given a thunk plus the associated metadata is released automatically - * in native via freeUpcallStub() in OpenJDK when the corresponding - * session/scope is terminated. - */ - String targetHashScopeStr = target.hashCode() + "#" + sessionOrScope.toString(); //$NON-NLS-1$ - Integer upcallThunkAddrKey = Integer.valueOf(targetHashScopeStr.hashCode()); - Long upcallThunkAddr = cachedUpcallThunkAddr.get(upcallThunkAddrKey); - if (upcallThunkAddr != null) { - addr = upcallThunkAddr.longValue(); - } else { - /* The thunk must be created for each upcall handler given the UpcallMHMetaData object uniquely bound to the thunk - * is only alive for a MemorySession(JDK19)/ResourceScope(JDK17/18) specified in java, which means the upcall handler - * and its UpcallMHMetaData object will be cleaned up automatically once their session/scope is closed. - */ - metaData = new UpcallMHMetaData(target, sessionOrScope); - addr = allocateUpcallStub(metaData, nativeSignatureStrs); - cachedUpcallThunkAddr.put(upcallThunkAddrKey, Long.valueOf(addr)); - } - } - - return addr; + /* The thunk must be created for each upcall handler given the UpcallMHMetaData object uniquely bound to the thunk + * is only alive for a MemorySession(JDK19)/ResourceScope(JDK17/18) specified in java, which means the upcall handler + * and its UpcallMHMetaData object will be cleaned up automatically once their session/scope is closed. + */ + metaData = new UpcallMHMetaData(target, sessionOrScope); + return allocateUpcallStub(metaData, nativeSignatureStrs); } /* This native requests the JIT to generate an upcall thunk of the specified java method diff --git a/runtime/jcl/common/jdk_internal_foreign_abi_UpcallStubs.cpp b/runtime/jcl/common/jdk_internal_foreign_abi_UpcallStubs.cpp index a04e70297c5..57b11a39950 100644 --- a/runtime/jcl/common/jdk_internal_foreign_abi_UpcallStubs.cpp +++ b/runtime/jcl/common/jdk_internal_foreign_abi_UpcallStubs.cpp @@ -48,36 +48,35 @@ extern "C" { * to be allocated on the heap for thunk (especially on AIX only with 4KB virtual memory). */ jboolean JNICALL -Java_jdk_internal_foreign_abi_UpcallStubs_freeUpcallStub0(JNIEnv *env, jobject receiver, jlong address) +Java_jdk_internal_foreign_abi_UpcallStubs_freeUpcallStub0(JNIEnv *env, jclass clazz, jlong address) { J9VMThread *currentThread = (J9VMThread *)env; J9JavaVM *vm = currentThread->javaVM; const J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; -#if defined(AIXPPC) || (defined(S390) && defined(zOS)) +#if defined(AIXPPC) || defined(J9ZOS390) /* The first element of the function pointer holds the thunk memory address, - * as specified in vm/ap64/UpcallThunkGen.cpp + * as specified in vm/ap64/UpcallThunkGen.cpp and vm/mz64/UpcallThunkGen.cpp. */ - UDATA *functionPtr = (UDATA *)address; - void *thunkAddr = (void *)functionPtr[0]; -#else /* defined(AIXPPC) || (defined(S390) && defined(zOS)) */ - void *thunkAddr = (void *)(UDATA)address; -#endif /* defined(AIXPPC) || (defined(S390) && defined(zOS)) */ + void **functionPtr = (void **)(intptr_t)address; + void *thunkAddr = *functionPtr; +#else /* defined(AIXPPC) || defined(J9ZOS390) */ + void *thunkAddr = (void *)(intptr_t)address; +#endif /* defined(AIXPPC) || defined(J9ZOS390) */ PORT_ACCESS_FROM_JAVAVM(vm); - omrthread_monitor_enter(vm->thunkHeapWrapperMutex); + omrthread_monitor_enter(vm->thunkHeapListMutex); if (NULL != thunkAddr) { - J9UpcallThunkHeapWrapper *thunkHeapWrapper = vm->thunkHeapWrapper; - J9HashTable *metaDataHashTable = thunkHeapWrapper->metaDataHashTable; - J9Heap *thunkHeap = thunkHeapWrapper->heap; + J9HashTable *metaDataHashTable = vm->thunkHeapHead->metaDataHashTable; if (NULL != metaDataHashTable) { J9UpcallMetaDataEntry metaDataEntry = {0}; - metaDataEntry.thunkAddrValue = (UDATA)(uintptr_t)thunkAddr; + metaDataEntry.thunkAddrValue = (UDATA)thunkAddr; metaDataEntry.upcallMetaData = NULL; J9UpcallMetaDataEntry * result = (J9UpcallMetaDataEntry *)hashTableFind(metaDataHashTable, &metaDataEntry); if (NULL != result) { J9UpcallMetaData *metaData = result->upcallMetaData; + J9Heap *thunkHeap = metaData->thunkHeapWrapper->heap; J9UpcallNativeSignature *nativeFuncSig = metaData->nativeFuncSignature; if (NULL != nativeFuncSig) { j9mem_free_memory(nativeFuncSig->sigArray); @@ -93,18 +92,18 @@ Java_jdk_internal_foreign_abi_UpcallStubs_freeUpcallStub0(JNIEnv *env, jobject r if (NULL != thunkHeap) { #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(0); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ j9heap_free(thunkHeap, thunkAddr); #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(1); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ } } } } - omrthread_monitor_exit(vm->thunkHeapWrapperMutex); + omrthread_monitor_exit(vm->thunkHeapListMutex); - return true; + return JNI_TRUE; } /** diff --git a/runtime/oti/j9nonbuilder.h b/runtime/oti/j9nonbuilder.h index 1db7ebc12fa..245e5edbdb2 100644 --- a/runtime/oti/j9nonbuilder.h +++ b/runtime/oti/j9nonbuilder.h @@ -5783,8 +5783,8 @@ typedef struct J9JavaVM { omrthread_monitor_t cifNativeCalloutDataCacheMutex; struct J9Pool *cifArgumentTypesCache; omrthread_monitor_t cifArgumentTypesCacheMutex; - struct J9UpcallThunkHeapWrapper *thunkHeapWrapper; - omrthread_monitor_t thunkHeapWrapperMutex; + struct J9UpcallThunkHeapList *thunkHeapHead; + omrthread_monitor_t thunkHeapListMutex; #endif /* JAVA_SPEC_VERSION >= 16 */ struct J9HashTable* ensureHashedClasses; #if JAVA_SPEC_VERSION >= 19 @@ -5933,6 +5933,18 @@ typedef struct J9UpcallNativeSignature { J9UpcallSigType *sigArray; } J9UpcallNativeSignature; +typedef struct J9UpcallThunkHeapWrapper { + J9Heap *heap; + uintptr_t heapSize; + J9PortVmemIdentifier vmemID; +} J9UpcallThunkHeapWrapper; + +typedef struct J9UpcallThunkHeapList { + J9UpcallThunkHeapWrapper *thunkHeapWrapper; + J9HashTable *metaDataHashTable; + struct J9UpcallThunkHeapList *next; +} J9UpcallThunkHeapList; + typedef struct J9UpcallMetaData { J9JavaVM *vm; J9VMThread *downCallThread; /* The thread is mainly used to set exceptions in dispatcher if a native thread is created locally */ @@ -5942,6 +5954,7 @@ typedef struct J9UpcallMetaData { UDATA thunkSize; /* The size of the generated thunk */ J9UpcallNativeSignature *nativeFuncSignature; /* The native function signature extracted from FunctionDescriptor */ UDATA functionPtr[3]; /* The address of the generated thunk on AIX or z/OS */ + J9UpcallThunkHeapWrapper *thunkHeapWrapper; /* The thunk heap associated with the metaData */ } J9UpcallMetaData; typedef struct J9UpcallMetaDataEntry { @@ -5949,13 +5962,6 @@ typedef struct J9UpcallMetaDataEntry { J9UpcallMetaData *upcallMetaData; } J9UpcallMetaDataEntry; -typedef struct J9UpcallThunkHeapWrapper { - J9Heap *heap; - uintptr_t heapSize; - J9PortVmemIdentifier vmemID; - J9HashTable *metaDataHashTable; -} J9UpcallThunkHeapWrapper; - #endif /* JAVA_SPEC_VERSION >= 16 */ /* Data block for JIT instance field watch reporting */ diff --git a/runtime/oti/jclprots.h b/runtime/oti/jclprots.h index 9c5e7a6add8..a558b57af18 100644 --- a/runtime/oti/jclprots.h +++ b/runtime/oti/jclprots.h @@ -1267,7 +1267,7 @@ void JNICALL Java_jdk_internal_foreign_abi_UpcallStubs_registerNatives(JNIEnv *env, jclass clazz); jboolean JNICALL -Java_jdk_internal_foreign_abi_UpcallStubs_freeUpcallStub0(JNIEnv *env, jobject receiver, jlong address); +Java_jdk_internal_foreign_abi_UpcallStubs_freeUpcallStub0(JNIEnv *env, jclass clazz, jlong address); void JNICALL Java_jdk_internal_misc_ScopedMemoryAccess_registerNatives(JNIEnv *env, jclass clazz); diff --git a/runtime/vm/OutOfLineINL_openj9_internal_foreign_abi_InternalUpcallHandler.cpp b/runtime/vm/OutOfLineINL_openj9_internal_foreign_abi_InternalUpcallHandler.cpp index 301ce93a6c1..9c83de69ed8 100644 --- a/runtime/vm/OutOfLineINL_openj9_internal_foreign_abi_InternalUpcallHandler.cpp +++ b/runtime/vm/OutOfLineINL_openj9_internal_foreign_abi_InternalUpcallHandler.cpp @@ -138,7 +138,7 @@ OutOfLineINL_openj9_internal_foreign_abi_InternalUpcallHandler_allocateUpcallStu } done: - VM_OutOfLineINL_Helpers::returnDouble(currentThread, (I_64)(uintptr_t)thunkAddr, 3); + VM_OutOfLineINL_Helpers::returnDouble(currentThread, (I_64)(intptr_t)thunkAddr, 3); return rc; freeAllMemoryThenExit: diff --git a/runtime/vm/UpcallThunkMem.cpp b/runtime/vm/UpcallThunkMem.cpp index 14d6bae57e9..6567d6a34c9 100644 --- a/runtime/vm/UpcallThunkMem.cpp +++ b/runtime/vm/UpcallThunkMem.cpp @@ -34,7 +34,7 @@ extern "C" { #endif static void * subAllocateThunkFromHeap(J9UpcallMetaData *data); -static J9Heap * allocateThunkHeap(J9UpcallMetaData *data); +static J9UpcallThunkHeapList * allocateThunkHeap(J9UpcallMetaData *data); static UDATA upcallMetaDataHashFn(void *key, void *userData); static UDATA upcallMetaDataEqualFn(void *leftKey, void *rightKey, void *userData); @@ -52,7 +52,7 @@ doneUpcallThunkGeneration(J9UpcallMetaData *data, void *thunkAddress) PORT_ACCESS_FROM_JAVAVM(vm); if (NULL != thunkAddress) { - /* Flash the generated thunk to the memory */ + /* Flush the generated thunk to the memory. */ j9cpu_flush_icache(thunkAddress, data->thunkSize); } } @@ -71,9 +71,9 @@ allocateUpcallThunkMemory(J9UpcallMetaData *data) Assert_VM_true(data->thunkSize > 0); - omrthread_monitor_enter(vm->thunkHeapWrapperMutex); + omrthread_monitor_enter(vm->thunkHeapListMutex); thunkAddress = subAllocateThunkFromHeap(data); - omrthread_monitor_exit(vm->thunkHeapWrapperMutex); + omrthread_monitor_exit(vm->thunkHeapListMutex); return thunkAddress; } @@ -87,66 +87,100 @@ subAllocateThunkFromHeap(J9UpcallMetaData *data) { J9JavaVM * vm = data->vm; UDATA thunkSize = data->thunkSize; - J9Heap *thunkHeap = NULL; + J9UpcallThunkHeapList *thunkHeapHead = vm->thunkHeapHead; + J9UpcallThunkHeapList *thunkHeapNode = NULL; void *subAllocThunkPtr = NULL; + bool succeeded = false; PORT_ACCESS_FROM_JAVAVM(vm); Trc_VM_subAllocateThunkFromHeap_Entry(thunkSize); - if (NULL == vm->thunkHeapWrapper) { - thunkHeap = allocateThunkHeap(data); - if (NULL == thunkHeap) { - /* The thunk address is NULL if VM fails to allocate the thunk heap */ + if (NULL == thunkHeapHead) { +retry: + thunkHeapHead = allocateThunkHeap(data); + if (NULL == thunkHeapHead) { + /* The thunk address is NULL if VM fails to allocate the thunk heap. */ goto done; } - } else { - thunkHeap = vm->thunkHeapWrapper->heap; } + thunkHeapNode = thunkHeapHead; #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(0); -#endif - subAllocThunkPtr = j9heap_allocate(thunkHeap, thunkSize); - if (NULL != subAllocThunkPtr) { - J9UpcallMetaDataEntry metaDataEntry = {0}; - metaDataEntry.thunkAddrValue = (UDATA)(uintptr_t)subAllocThunkPtr; - metaDataEntry.upcallMetaData = data; - - /* Store the address of the generated thunk plus the corresponding metadata as an entry to - * a hashtable which will be used to release the memory automatically via freeUpcallStub - * in OpenJDK when their native scope is terminated or VM exits. - */ - if (NULL == hashTableAdd(vm->thunkHeapWrapper->metaDataHashTable, &metaDataEntry)) { - j9heap_free(thunkHeap, subAllocThunkPtr); - subAllocThunkPtr = NULL; - Trc_VM_subAllocateThunkFromHeap_suballoc_thunk_add_hashtable_failed(thunkSize, thunkHeap); +#endif /* defined(OSX) && defined(AARCH64) */ + while (!succeeded) { + J9UpcallThunkHeapWrapper *thunkHeapWrapper = thunkHeapNode->thunkHeapWrapper; + J9Heap *thunkHeap = thunkHeapWrapper->heap; + + subAllocThunkPtr = j9heap_allocate(thunkHeap, thunkSize); + if (NULL != subAllocThunkPtr) { + J9UpcallMetaDataEntry metaDataEntry = {0}; + + data->thunkHeapWrapper = thunkHeapWrapper; + metaDataEntry.thunkAddrValue = (UDATA)subAllocThunkPtr; + metaDataEntry.upcallMetaData = data; + + /* Store the address of the generated thunk plus the corresponding metadata as an entry to + * a hashtable which will be used to release the memory automatically via freeUpcallStub + * in OpenJDK when their native scope is terminated or VM exits. + */ + if (NULL == hashTableAdd(thunkHeapNode->metaDataHashTable, &metaDataEntry)) { + j9heap_free(thunkHeap, subAllocThunkPtr); + subAllocThunkPtr = NULL; + Trc_VM_subAllocateThunkFromHeap_suballoc_thunk_add_hashtable_failed(thunkSize, thunkHeap); + break; + } else { + Trc_VM_subAllocateThunkFromHeap_suballoc_thunk_success(subAllocThunkPtr, thunkSize, thunkHeap); + /* The list head always points to the free thunk heap if thunk + * is successfully allocated from the heap. + */ + vm->thunkHeapHead = thunkHeapNode; + succeeded = true; + } } else { - Trc_VM_subAllocateThunkFromHeap_suballoc_thunk_success(subAllocThunkPtr, thunkSize, thunkHeap); + Trc_VM_subAllocateThunkFromHeap_suballoc_thunk_failed(thunkSize, thunkHeap); + if (thunkHeapNode->next != thunkHeapHead) { + /* Get back to check the previous thunk heaps in the list to see whether + * a free thunk heap (in which part of the previously allocated thunks + * are already released) is available for use. + */ + thunkHeapNode = thunkHeapNode->next; + } else { + /* Attempt to allocate a new thunk heap given all thunk heaps are out of memory. */ +#if defined(OSX) && defined(AARCH64) + pthread_jit_write_protect_np(1); +#endif /* defined(OSX) && defined(AARCH64) */ + goto retry; + } + } - } else { - Trc_VM_subAllocateThunkFromHeap_suballoc_thunk_failed(thunkSize, thunkHeap); } #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(1); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ done: + Trc_VM_subAllocateThunkFromHeap_Exit(subAllocThunkPtr); return subAllocThunkPtr; } /** + * Allocate a new thunk heap and add it to the end of the thunk heap list. + * + * Note: * The memory of the thunk heap will be allocated using vmem. If the overhead * of omrheap precludes using suballocation, omrheap will not be used and the * memory will be used directly instead. */ -static J9Heap * +static J9UpcallThunkHeapList * allocateThunkHeap(J9UpcallMetaData *data) { J9JavaVM * vm = data->vm; PORT_ACCESS_FROM_JAVAVM(vm); UDATA pageSize = j9vmem_supported_page_sizes()[0]; UDATA thunkSize = data->thunkSize; + J9UpcallThunkHeapList *thunkHeapNode = NULL; J9UpcallThunkHeapWrapper *thunkHeapWrapper = NULL; J9HashTable *metaDataHashTable = NULL; void *allocMemPtr = NULL; @@ -155,22 +189,21 @@ allocateThunkHeap(J9UpcallMetaData *data) Trc_VM_allocateThunkHeap_Entry(thunkSize); - /* Create the wrapper struct for the thunk heap */ - thunkHeapWrapper = (J9UpcallThunkHeapWrapper *)j9mem_allocate_memory(sizeof(J9UpcallThunkHeapWrapper), J9MEM_CATEGORY_VM_FFI); - if (NULL == thunkHeapWrapper) { - Trc_VM_allocateThunkHeap_allocate_thunk_heap_wrapper_failed(); + /* Create a thunk heap node of the list. */ + thunkHeapNode = (J9UpcallThunkHeapList *)j9mem_allocate_memory(sizeof(J9UpcallThunkHeapList), J9MEM_CATEGORY_VM_FFI); + if (NULL == thunkHeapNode) { + Trc_VM_allocateThunkHeap_allocate_thunk_heap_node_failed(); goto done; } - metaDataHashTable = hashTableNew(OMRPORT_FROM_J9PORT(vm->portLibrary), "Upcall metadata table", 0, - sizeof(J9UpcallMetaDataEntry), 0, 0, J9MEM_CATEGORY_VM_FFI, upcallMetaDataHashFn, upcallMetaDataEqualFn, NULL, NULL); - if (NULL == metaDataHashTable) { - Trc_VM_allocateThunkHeap_create_metadata_hash_table_failed(); + /* Create the wrapper struct for the thunk heap. */ + thunkHeapWrapper = (J9UpcallThunkHeapWrapper *)j9mem_allocate_memory(sizeof(J9UpcallThunkHeapWrapper), J9MEM_CATEGORY_VM_FFI); + if (NULL == thunkHeapWrapper) { + Trc_VM_allocateThunkHeap_allocate_thunk_heap_wrapper_failed(); goto freeAllMemoryThenExit; } - thunkHeapWrapper->metaDataHashTable = metaDataHashTable; - /* Reserve a block of memory with the fixed page size for heap creation */ + /* Reserve a block of memory with the fixed page size for heap creation. */ allocMemPtr = j9vmem_reserve_memory(NULL, pageSize, &vmemID, J9PORT_VMEM_MEMORY_MODE_READ | J9PORT_VMEM_MEMORY_MODE_WRITE | J9PORT_VMEM_MEMORY_MODE_EXECUTE | J9PORT_VMEM_MEMORY_MODE_COMMIT, pageSize, J9MEM_CATEGORY_VM_FFI); @@ -181,7 +214,7 @@ allocateThunkHeap(J9UpcallMetaData *data) #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(0); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ /* Initialize the allocated memory as a J9Heap */ thunkHeap = j9heap_create(allocMemPtr, pageSize, 0); @@ -190,21 +223,45 @@ allocateThunkHeap(J9UpcallMetaData *data) goto freeAllMemoryThenExit; } - /* Store the heap handle in J9UpcallThunkHeapWrapper if the thunk heap is successfully created */ + /* Store the heap handle in J9UpcallThunkHeapWrapper if the thunk heap is successfully created. */ thunkHeapWrapper->heap = thunkHeap; thunkHeapWrapper->heapSize = pageSize; thunkHeapWrapper->vmemID = vmemID; - vm->thunkHeapWrapper = thunkHeapWrapper; + thunkHeapNode->thunkHeapWrapper = thunkHeapWrapper; + + /* Set the newly created thunk heap node as the list head if it doesn't exist + * or when there is no free thunk heap to be allocated in the list for now. + */ + if (NULL == vm->thunkHeapHead) { + metaDataHashTable = hashTableNew(OMRPORT_FROM_J9PORT(vm->portLibrary), "Upcall metadata table", 0, + sizeof(J9UpcallMetaDataEntry), 0, 0, J9MEM_CATEGORY_VM_FFI, upcallMetaDataHashFn, upcallMetaDataEqualFn, NULL, NULL); + if (NULL == metaDataHashTable) { + Trc_VM_allocateThunkHeap_create_metadata_hash_table_failed(); + goto freeAllMemoryThenExit; + } + thunkHeapNode->metaDataHashTable = metaDataHashTable; + vm->thunkHeapHead = thunkHeapNode; + vm->thunkHeapHead->next = vm->thunkHeapHead; + } else { + /* The hashtable is shared among all thunk heaps in the list. */ + thunkHeapNode->metaDataHashTable = vm->thunkHeapHead->metaDataHashTable; + thunkHeapNode->next = vm->thunkHeapHead->next; + vm->thunkHeapHead->next = thunkHeapNode; + vm->thunkHeapHead = thunkHeapNode; + } #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(1); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ done: Trc_VM_allocateThunkHeap_Exit(thunkHeap); - return thunkHeap; + return thunkHeapNode; freeAllMemoryThenExit: + j9mem_free_memory(thunkHeapNode); + thunkHeapNode = NULL; + if (NULL != thunkHeapWrapper) { if (NULL != metaDataHashTable) { hashTableFree(metaDataHashTable); @@ -216,7 +273,7 @@ allocateThunkHeap(J9UpcallMetaData *data) if (NULL != allocMemPtr) { #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(1); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ j9vmem_free_memory(allocMemPtr, pageSize, &vmemID); allocMemPtr = NULL; } diff --git a/runtime/vm/j9vm.tdf b/runtime/vm/j9vm.tdf index 5034122e655..bacea84cd47 100644 --- a/runtime/vm/j9vm.tdf +++ b/runtime/vm/j9vm.tdf @@ -923,3 +923,5 @@ TraceEvent=Trc_VM_allocateThunkHeap_create_metadata_hash_table_failed NoEnv Over TraceEvent=Trc_VM_allocateThunkHeap_reserve_memory_failed NoEnv Overhead=1 Level=3 Template="allocateThunkHeap - Failed to allocate reserve the memory with the requested page size(%zu)" TraceEvent=Trc_VM_allocateThunkHeap_create_heap_failed NoEnv Overhead=1 Level=3 Template="allocateThunkHeap - Failed to create a heap from the reserved memory (%p) with the requested page size(%zu)" TraceExit=Trc_VM_allocateThunkHeap_Exit NoEnv Overhead=1 Level=3 Template="Exit allocateThunkHeap - The suballocated thunk heap is %p" + +TraceEvent=Trc_VM_allocateThunkHeap_allocate_thunk_heap_node_failed NoEnv Overhead=1 Level=3 Template="allocateThunkHeap - Failed to allocate memory for the thunk heap node" diff --git a/runtime/vm/jvminit.c b/runtime/vm/jvminit.c index b530ef09612..145cb7b9f49 100644 --- a/runtime/vm/jvminit.c +++ b/runtime/vm/jvminit.c @@ -668,17 +668,17 @@ freeJavaVM(J9JavaVM * vm) vm->cifArgumentTypesCache = NULL; } - /* Clean up all resources created via allocateUpcallThunkMemory, including the generatered thunk - * and the corresponding metadata of each entry stored in the hashtable. + /* Empty the thunk heap list by cleaning up all resources created via allocateUpcallThunkMemory, + * including the generated thunk and the corresponding metadata of each entry in the hashtable. * See UpcallThunkMem.cpp for details. */ - if (NULL != vm->thunkHeapWrapper) { - J9UpcallThunkHeapWrapper *thunkHeapWrapper = vm->thunkHeapWrapper; - J9PortVmemIdentifier vmemID = thunkHeapWrapper->vmemID; + if (NULL != vm->thunkHeapHead) { + J9UpcallThunkHeapList *thunkHeapHead = vm->thunkHeapHead; + J9UpcallThunkHeapList *thunkHeapNode = NULL; UDATA byteAmount = j9vmem_supported_page_sizes()[0]; - if (NULL != thunkHeapWrapper->metaDataHashTable) { - J9HashTable *metaDataHashTable = thunkHeapWrapper->metaDataHashTable; + if (NULL != thunkHeapHead->metaDataHashTable) { + J9HashTable *metaDataHashTable = thunkHeapHead->metaDataHashTable; UDATA entryCount = hashTableGetCount(metaDataHashTable); if (entryCount > 0) { hashTableForEachDo(metaDataHashTable, (J9HashTableDoFn)freeUpcallMetaDataDoFn, NULL); @@ -686,9 +686,23 @@ freeJavaVM(J9JavaVM * vm) hashTableFree(metaDataHashTable); metaDataHashTable = NULL; } - j9vmem_free_memory(vmemID.address, byteAmount, &vmemID); - j9mem_free_memory(thunkHeapWrapper); - thunkHeapWrapper = NULL; + + thunkHeapNode = thunkHeapHead->next; + thunkHeapHead->next = NULL; + thunkHeapHead = thunkHeapNode; + while (NULL != thunkHeapHead) { + J9UpcallThunkHeapWrapper *thunkHeapWrapper = thunkHeapHead->thunkHeapWrapper; + if (NULL != thunkHeapWrapper) { + J9PortVmemIdentifier vmemID = thunkHeapWrapper->vmemID; + j9vmem_free_memory(vmemID.address, byteAmount, &vmemID); + j9mem_free_memory(thunkHeapWrapper); + thunkHeapWrapper = NULL; + } + thunkHeapNode = thunkHeapHead; + thunkHeapHead = thunkHeapHead->next; + j9mem_free_memory(thunkHeapNode); + } + vm->thunkHeapHead = NULL; } #endif /* JAVA_SPEC_VERSION >= 16 */ @@ -6745,7 +6759,7 @@ protectedInitializeJavaVM(J9PortLibrary* portLibrary, void * userData) vm->cifNativeCalloutDataCache = NULL; vm->cifArgumentTypesCache = NULL; /* The thunk block should be allocated on demand */ - vm->thunkHeapWrapper = NULL; + vm->thunkHeapHead = NULL; #endif /* JAVA_SPEC_VERSION >= 16 */ #if defined(J9X86) || defined(J9HAMMER) @@ -8399,7 +8413,7 @@ freeUpcallMetaDataDoFn(J9UpcallMetaDataEntry *entry, void *userData) const J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; J9VMThread *currentThread = vmFuncs->currentVMThread(vm); J9UpcallNativeSignature *nativeFuncSig = metaData->nativeFuncSignature; - J9Heap *thunkHeap = vm->thunkHeapWrapper->heap; + J9Heap *thunkHeap = metaData->thunkHeapWrapper->heap; PORT_ACCESS_FROM_JAVAVM(vm); if (NULL != nativeFuncSig) { @@ -8414,11 +8428,11 @@ freeUpcallMetaDataDoFn(J9UpcallMetaDataEntry *entry, void *userData) if (NULL != thunkHeap) { #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(0); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ j9heap_free(thunkHeap, thunkAddr); #if defined(OSX) && defined(AARCH64) pthread_jit_write_protect_np(1); -#endif +#endif /* defined(OSX) && defined(AARCH64) */ } entry->thunkAddrValue = 0; } diff --git a/runtime/vm/vmthinit.c b/runtime/vm/vmthinit.c index 41fb8714760..ecf8267c53e 100644 --- a/runtime/vm/vmthinit.c +++ b/runtime/vm/vmthinit.c @@ -87,7 +87,7 @@ UDATA initializeVMThreading(J9JavaVM *vm) #if JAVA_SPEC_VERSION >= 16 omrthread_monitor_init_with_name(&vm->cifNativeCalloutDataCacheMutex, 0, "CIF cache mutex") || omrthread_monitor_init_with_name(&vm->cifArgumentTypesCacheMutex, 0, "CIF argument types mutex") || - omrthread_monitor_init_with_name(&vm->thunkHeapWrapperMutex, 0, "Wait mutex for allocating the upcall thunk memory") || + omrthread_monitor_init_with_name(&vm->thunkHeapListMutex, 0, "Wait mutex for allocating the upcall thunk memory") || #endif /* JAVA_SPEC_VERSION >= 16 */ #if JAVA_SPEC_VERSION >= 19 @@ -187,9 +187,9 @@ void terminateVMThreading(J9JavaVM *vm) vm->cifArgumentTypesCacheMutex = NULL; } - if (NULL != vm->thunkHeapWrapperMutex) { - omrthread_monitor_destroy(vm->thunkHeapWrapperMutex); - vm->thunkHeapWrapperMutex = NULL; + if (NULL != vm->thunkHeapListMutex) { + omrthread_monitor_destroy(vm->thunkHeapListMutex); + vm->thunkHeapListMutex = NULL; } #endif /* JAVA_SPEC_VERSION >= 16 */