Skip to content

Commit

Permalink
[vulkan] Add conform API methods to memory allocator to fix block all…
Browse files Browse the repository at this point in the history 
…ocations (#8130)

* Add conform API methods to block and region allocator classes
Override conform requests for Vulkan memory allocator
Cleanup memory requirement constraints for Vulkan
Add conform test cases to block_allocator runtime test.

* Clang format/tidy pas

* Fix unsigned int comparisons

* Clang format pass

* Fix other unsigned int comparisons

* Fix mismatched template types for max()

* Fix whitespace for clang format

---------

Co-authored-by: Derek Gerstmann <[email protected]>
  • Loading branch information
@derek-gerstmann
derek-gerstmann and Derek Gerstmann authored Mar 6, 2024
1 parent 10e07e6 commit 754e6ec
Show file tree
Hide file tree
Showing 5 changed files with 566 additions and 184 deletions.
132 changes: 83 additions & 49 deletions src/runtime/internal/block_allocator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,11 @@ class BlockAllocator {

// Public interface methods
MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
int retain(void *user_context, MemoryRegion *region); //< retain the region and increase the usage count
bool collect(void *user_context); //< returns true if any blocks were removed
int conform(void *user_context, MemoryRequest *request) const; //< conform the given request into a suitable allocation
int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
int retain(void *user_context, MemoryRegion *region); //< retain the region and increase the usage count
bool collect(void *user_context); //< returns true if any blocks were removed
int release(void *user_context);
int destroy(void *user_context);

Expand Down Expand Up @@ -86,13 +87,13 @@ class BlockAllocator {
int destroy_region_allocator(void *user_context, RegionAllocator *region_allocator);

// Reserves a block of memory for the requested size and returns the corresponding block entry, or nullptr on failure
BlockEntry *reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated);
BlockEntry *reserve_block_entry(void *user_context, const MemoryRequest &request);

// Locates the "best-fit" block entry for the requested size, or nullptr if none was found
BlockEntry *find_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated);
BlockEntry *find_block_entry(void *user_context, const MemoryRequest &request);

// Creates a new block entry and int the list
BlockEntry *create_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated);
// Creates a new block entry and adds it tos the list
BlockEntry *create_block_entry(void *user_context, const MemoryRequest &request);

// Releases the block entry from being used, and makes it available for further allocations
int release_block_entry(void *user_context, BlockEntry *block_entry);
Expand All @@ -113,7 +114,7 @@ class BlockAllocator {
bool is_compatible_block(const BlockResource *block, const MemoryProperties &properties) const;

// Returns true if the given block is suitable for the request allocation
bool is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryProperties &properties, size_t size, bool dedicated) const;
bool is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryRequest &request) const;

Config config;
LinkedList block_list;
Expand Down Expand Up @@ -162,7 +163,8 @@ MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &r
<< "caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...";
#endif
BlockEntry *block_entry = reserve_block_entry(user_context, request.properties, request.size, request.dedicated);
// Reserve a block entry for use
BlockEntry *block_entry = reserve_block_entry(user_context, request);
if (block_entry == nullptr) {
error(user_context) << "BlockAllocator: Failed to allocate new empty block of requested size ("
<< (int32_t)(request.size) << " bytes)\n";
Expand All @@ -173,11 +175,12 @@ MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &r
halide_abort_if_false(user_context, block != nullptr);
halide_abort_if_false(user_context, block->allocator != nullptr);

// Reserve an initial memory region for the block
MemoryRegion *result = reserve_memory_region(user_context, block->allocator, request);
if (result == nullptr) {

// Unable to reserve region in an existing block ... create a new block and try again.
block_entry = create_block_entry(user_context, request.properties, request.size, request.dedicated);
block_entry = create_block_entry(user_context, request);
if (block_entry == nullptr) {
error(user_context) << "BlockAllocator: Out of memory! Failed to allocate empty block of size ("
<< (int32_t)(request.size) << " bytes)\n";
Expand Down Expand Up @@ -299,8 +302,8 @@ MemoryRegion *BlockAllocator::reserve_memory_region(void *user_context, RegionAl
return result;
}

bool BlockAllocator::is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryProperties &properties, size_t size, bool dedicated) const {
if (!is_compatible_block(block, properties)) {
bool BlockAllocator::is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryRequest &request) const {
if (!is_compatible_block(block, request.properties)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... incompatible properties! ("
<< "block_resource=" << (void *)block << " "
Expand All @@ -309,16 +312,16 @@ bool BlockAllocator::is_block_suitable_for_request(void *user_context, const Blo
<< "block_usage=" << halide_memory_usage_name(block->memory.properties.usage) << " "
<< "block_caching=" << halide_memory_caching_name(block->memory.properties.caching) << " "
<< "block_visibility=" << halide_memory_visibility_name(block->memory.properties.visibility) << " "
<< "request_size=" << (uint32_t)size << " "
<< "request_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "request_size=" << (uint32_t)request.size << " "
<< "request_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
// skip blocks that are using incompatible memory
return false;
}

if (dedicated && (block->reserved > 0)) {
if (request.dedicated && (block->reserved > 0)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... can be used for dedicated allocation! ("
<< "block_resource=" << (void *)block << " "
Expand All @@ -340,31 +343,31 @@ bool BlockAllocator::is_block_suitable_for_request(void *user_context, const Blo
}

size_t available = (block->memory.size - block->reserved);
if (available >= size) {
if (available >= request.size) {
return true;
}

return false;
}

BlockAllocator::BlockEntry *
BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
BlockAllocator::find_block_entry(void *user_context, const MemoryRequest &request) {
BlockEntry *block_entry = block_list.back();
while (block_entry != nullptr) {
BlockEntry *prev_entry = block_entry->prev_ptr;
const BlockResource *block = static_cast<BlockResource *>(block_entry->value);
if (is_block_suitable_for_request(user_context, block, properties, size, dedicated)) {
if (is_block_suitable_for_request(user_context, block, request)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: found suitable block ("
<< "user_context=" << (void *)(user_context) << " "
<< "block_resource=" << (void *)block << " "
<< "block_size=" << (uint32_t)block->memory.size << " "
<< "block_reserved=" << (uint32_t)block->reserved << " "
<< "request_size=" << (uint32_t)size << " "
<< "dedicated=" << (dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "request_size=" << (uint32_t)request.size << " "
<< "request_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "request_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
return block_entry;
}
Expand All @@ -375,37 +378,37 @@ BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &pro
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: couldn't find suitable block! ("
<< "user_context=" << (void *)(user_context) << " "
<< "request_size=" << (uint32_t)size << " "
<< "dedicated=" << (dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "request_size=" << (uint32_t)request.size << " "
<< "request_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "request_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
}
return block_entry;
}

BlockAllocator::BlockEntry *
BlockAllocator::reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
BlockAllocator::reserve_block_entry(void *user_context, const MemoryRequest &request) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: reserving block ... ! ("
<< "requested_size=" << (uint32_t)size << " "
<< "requested_is_dedicated=" << (dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "requested_size=" << (uint32_t)request.size << " "
<< "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
BlockEntry *block_entry = find_block_entry(user_context, properties, size, dedicated);
BlockEntry *block_entry = find_block_entry(user_context, request);
if (block_entry == nullptr) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: creating block ... ! ("
<< "requested_size=" << (uint32_t)size << " "
<< "requested_is_dedicated=" << (dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "requested_size=" << (uint32_t)request.size << " "
<< "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
block_entry = create_block_entry(user_context, properties, size, dedicated);
block_entry = create_block_entry(user_context, request);
}

if (block_entry) {
Expand Down Expand Up @@ -449,7 +452,7 @@ int BlockAllocator::destroy_region_allocator(void *user_context, RegionAllocator
}

BlockAllocator::BlockEntry *
BlockAllocator::create_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
BlockAllocator::create_block_entry(void *user_context, const MemoryRequest &request) {
if (config.maximum_pool_size && (pool_size() >= config.maximum_pool_size)) {
error(user_context) << "BlockAllocator: No free blocks found! Maximum pool size reached ("
<< (int32_t)(config.maximum_pool_size) << " bytes or "
Expand All @@ -476,12 +479,16 @@ BlockAllocator::create_block_entry(void *user_context, const MemoryProperties &p
<< "allocator=" << (void *)(allocators.block.allocate) << ")...";
#endif

// Constrain the request to the a valid block allocation
MemoryRequest block_request = request;
conform(user_context, &block_request);

// Create the block resource itself
BlockResource *block = static_cast<BlockResource *>(block_entry->value);
block->memory.size = constrain_requested_size(size);
block->memory.size = block_request.size;
block->memory.handle = nullptr;
block->memory.properties = properties;
block->memory.properties.nearest_multiple = max(config.nearest_multiple, properties.nearest_multiple);
block->memory.dedicated = dedicated;
block->memory.properties = block_request.properties;
block->memory.dedicated = block_request.dedicated;
block->reserved = 0;
block->allocator = create_region_allocator(user_context, block);
alloc_memory_block(user_context, block);
Expand Down Expand Up @@ -561,6 +568,33 @@ size_t BlockAllocator::constrain_requested_size(size_t size) const {
return actual_size;
}

int BlockAllocator::conform(void *user_context, MemoryRequest *request) const {

request->properties.nearest_multiple = max(config.nearest_multiple, request->properties.nearest_multiple);

if (request->properties.nearest_multiple) {
size_t nm = request->properties.nearest_multiple;
request->size = (((request->size + nm - 1) / nm) * nm); // round up to nearest multiple
}

if (config.minimum_block_size) {
request->size = ((request->size < config.minimum_block_size) ?
config.minimum_block_size :
request->size);
}
if (config.maximum_block_size) {
request->size = ((request->size > config.maximum_block_size) ?
config.maximum_block_size :
request->size);
}

if (allocators.block.conform) {
return allocators.block.conform(user_context, request);
}

return 0;
}

bool BlockAllocator::is_compatible_block(const BlockResource *block, const MemoryProperties &properties) const {
if (properties.caching != MemoryCaching::DefaultCaching) {
if (properties.caching != block->memory.properties.caching) {
Expand Down
4 changes: 4 additions & 0 deletions src/runtime/internal/memory_resources.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,18 +202,22 @@ struct HalideSystemAllocatorFns {

typedef int (*AllocateBlockFn)(void *, MemoryBlock *);
typedef int (*DeallocateBlockFn)(void *, MemoryBlock *);
typedef int (*ConformBlockRequestFn)(void *, MemoryRequest *);

struct MemoryBlockAllocatorFns {
AllocateBlockFn allocate = nullptr;
DeallocateBlockFn deallocate = nullptr;
ConformBlockRequestFn conform = nullptr;
};

typedef int (*AllocateRegionFn)(void *, MemoryRegion *);
typedef int (*DeallocateRegionFn)(void *, MemoryRegion *);
typedef int (*ConformBlockRegionFn)(void *, MemoryRequest *);

struct MemoryRegionAllocatorFns {
AllocateRegionFn allocate = nullptr;
DeallocateRegionFn deallocate = nullptr;
ConformBlockRegionFn conform = nullptr;
};

// --
Expand Down
Loading

0 comments on commit 754e6ec

Please sign in to comment.