Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[vulkan] Add conform API methods to memory allocator to fix block allocations #8130

Merged
merged 7 commits into from
Mar 6, 2024
+566 −184
Merged

[vulkan] Add conform API methods to memory allocator to fix block allocations #8130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
61c0ec7
Add conform API methods to block and region allocator classes
Mar 1, 2024
12cfaed
Clang format/tidy pas
Mar 1, 2024
7a2cf1d
Fix unsigned int comparisons
Mar 4, 2024
e3b362c
Clang format pass
Mar 4, 2024
f36f066
Fix other unsigned int comparisons
Mar 4, 2024
cd2d237
Fix mismatched template types for max()
Mar 4, 2024
9e9859d
Fix whitespace for clang format
Mar 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 83 additions & 49 deletions src/runtime/internal/block_allocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,11 @@ class BlockAllocator {

// Public interface methods
MemoryRegion *reserve(void *user_context, const MemoryRequest &request);
int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
int retain(void *user_context, MemoryRegion *region); //< retain the region and increase the usage count
bool collect(void *user_context); //< returns true if any blocks were removed
int conform(void *user_context, MemoryRequest *request) const; //< conform the given request into a suitable allocation
int release(void *user_context, MemoryRegion *region); //< unmark and cache the region for reuse
int reclaim(void *user_context, MemoryRegion *region); //< free the region and consolidate
int retain(void *user_context, MemoryRegion *region); //< retain the region and increase the usage count
bool collect(void *user_context); //< returns true if any blocks were removed
int release(void *user_context);
int destroy(void *user_context);

Expand Down Expand Up @@ -86,13 +87,13 @@ class BlockAllocator {
int destroy_region_allocator(void *user_context, RegionAllocator *region_allocator);

// Reserves a block of memory for the requested size and returns the corresponding block entry, or nullptr on failure
BlockEntry *reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated);
BlockEntry *reserve_block_entry(void *user_context, const MemoryRequest &request);

// Locates the "best-fit" block entry for the requested size, or nullptr if none was found
BlockEntry *find_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated);
BlockEntry *find_block_entry(void *user_context, const MemoryRequest &request);

// Creates a new block entry and int the list
BlockEntry *create_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated);
// Creates a new block entry and adds it tos the list
BlockEntry *create_block_entry(void *user_context, const MemoryRequest &request);

// Releases the block entry from being used, and makes it available for further allocations
int release_block_entry(void *user_context, BlockEntry *block_entry);
Expand All @@ -113,7 +114,7 @@ class BlockAllocator {
bool is_compatible_block(const BlockResource *block, const MemoryProperties &properties) const;

// Returns true if the given block is suitable for the request allocation
bool is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryProperties &properties, size_t size, bool dedicated) const;
bool is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryRequest &request) const;

Config config;
LinkedList block_list;
Expand Down Expand Up @@ -162,7 +163,8 @@ MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &r
<< "caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(request.properties.visibility) << ") ...";
#endif
BlockEntry *block_entry = reserve_block_entry(user_context, request.properties, request.size, request.dedicated);
// Reserve a block entry for use
BlockEntry *block_entry = reserve_block_entry(user_context, request);
if (block_entry == nullptr) {
error(user_context) << "BlockAllocator: Failed to allocate new empty block of requested size ("
<< (int32_t)(request.size) << " bytes)\n";
Expand All @@ -173,11 +175,12 @@ MemoryRegion *BlockAllocator::reserve(void *user_context, const MemoryRequest &r
halide_abort_if_false(user_context, block != nullptr);
halide_abort_if_false(user_context, block->allocator != nullptr);

// Reserve an initial memory region for the block
MemoryRegion *result = reserve_memory_region(user_context, block->allocator, request);
if (result == nullptr) {

// Unable to reserve region in an existing block ... create a new block and try again.
block_entry = create_block_entry(user_context, request.properties, request.size, request.dedicated);
block_entry = create_block_entry(user_context, request);
if (block_entry == nullptr) {
error(user_context) << "BlockAllocator: Out of memory! Failed to allocate empty block of size ("
<< (int32_t)(request.size) << " bytes)\n";
Expand Down Expand Up @@ -299,8 +302,8 @@ MemoryRegion *BlockAllocator::reserve_memory_region(void *user_context, RegionAl
return result;
}

bool BlockAllocator::is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryProperties &properties, size_t size, bool dedicated) const {
if (!is_compatible_block(block, properties)) {
bool BlockAllocator::is_block_suitable_for_request(void *user_context, const BlockResource *block, const MemoryRequest &request) const {
if (!is_compatible_block(block, request.properties)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... incompatible properties! ("
<< "block_resource=" << (void *)block << " "
Expand All @@ -309,16 +312,16 @@ bool BlockAllocator::is_block_suitable_for_request(void *user_context, const Blo
<< "block_usage=" << halide_memory_usage_name(block->memory.properties.usage) << " "
<< "block_caching=" << halide_memory_caching_name(block->memory.properties.caching) << " "
<< "block_visibility=" << halide_memory_visibility_name(block->memory.properties.visibility) << " "
<< "request_size=" << (uint32_t)size << " "
<< "request_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "request_size=" << (uint32_t)request.size << " "
<< "request_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
// skip blocks that are using incompatible memory
return false;
}

if (dedicated && (block->reserved > 0)) {
if (request.dedicated && (block->reserved > 0)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: skipping block ... can be used for dedicated allocation! ("
<< "block_resource=" << (void *)block << " "
Expand All @@ -340,31 +343,31 @@ bool BlockAllocator::is_block_suitable_for_request(void *user_context, const Blo
}

size_t available = (block->memory.size - block->reserved);
if (available >= size) {
if (available >= request.size) {
return true;
}

return false;
}

BlockAllocator::BlockEntry *
BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
BlockAllocator::find_block_entry(void *user_context, const MemoryRequest &request) {
BlockEntry *block_entry = block_list.back();
while (block_entry != nullptr) {
BlockEntry *prev_entry = block_entry->prev_ptr;
const BlockResource *block = static_cast<BlockResource *>(block_entry->value);
if (is_block_suitable_for_request(user_context, block, properties, size, dedicated)) {
if (is_block_suitable_for_request(user_context, block, request)) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: found suitable block ("
<< "user_context=" << (void *)(user_context) << " "
<< "block_resource=" << (void *)block << " "
<< "block_size=" << (uint32_t)block->memory.size << " "
<< "block_reserved=" << (uint32_t)block->reserved << " "
<< "request_size=" << (uint32_t)size << " "
<< "dedicated=" << (dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "request_size=" << (uint32_t)request.size << " "
<< "request_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "request_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
return block_entry;
}
Expand All @@ -375,37 +378,37 @@ BlockAllocator::find_block_entry(void *user_context, const MemoryProperties &pro
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: couldn't find suitable block! ("
<< "user_context=" << (void *)(user_context) << " "
<< "request_size=" << (uint32_t)size << " "
<< "dedicated=" << (dedicated ? "true" : "false") << " "
<< "usage=" << halide_memory_usage_name(properties.usage) << " "
<< "caching=" << halide_memory_caching_name(properties.caching) << " "
<< "visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "request_size=" << (uint32_t)request.size << " "
<< "request_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "request_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "request_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "request_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
}
return block_entry;
}

BlockAllocator::BlockEntry *
BlockAllocator::reserve_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
BlockAllocator::reserve_block_entry(void *user_context, const MemoryRequest &request) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: reserving block ... ! ("
<< "requested_size=" << (uint32_t)size << " "
<< "requested_is_dedicated=" << (dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "requested_size=" << (uint32_t)request.size << " "
<< "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
BlockEntry *block_entry = find_block_entry(user_context, properties, size, dedicated);
BlockEntry *block_entry = find_block_entry(user_context, request);
if (block_entry == nullptr) {
#ifdef DEBUG_RUNTIME_INTERNAL
debug(user_context) << "BlockAllocator: creating block ... ! ("
<< "requested_size=" << (uint32_t)size << " "
<< "requested_is_dedicated=" << (dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(properties.visibility) << ")";
<< "requested_size=" << (uint32_t)request.size << " "
<< "requested_is_dedicated=" << (request.dedicated ? "true" : "false") << " "
<< "requested_usage=" << halide_memory_usage_name(request.properties.usage) << " "
<< "requested_caching=" << halide_memory_caching_name(request.properties.caching) << " "
<< "requested_visibility=" << halide_memory_visibility_name(request.properties.visibility) << ")";
#endif
block_entry = create_block_entry(user_context, properties, size, dedicated);
block_entry = create_block_entry(user_context, request);
}

if (block_entry) {
Expand Down Expand Up @@ -449,7 +452,7 @@ int BlockAllocator::destroy_region_allocator(void *user_context, RegionAllocator
}

BlockAllocator::BlockEntry *
BlockAllocator::create_block_entry(void *user_context, const MemoryProperties &properties, size_t size, bool dedicated) {
BlockAllocator::create_block_entry(void *user_context, const MemoryRequest &request) {
if (config.maximum_pool_size && (pool_size() >= config.maximum_pool_size)) {
error(user_context) << "BlockAllocator: No free blocks found! Maximum pool size reached ("
<< (int32_t)(config.maximum_pool_size) << " bytes or "
Expand All @@ -476,12 +479,16 @@ BlockAllocator::create_block_entry(void *user_context, const MemoryProperties &p
<< "allocator=" << (void *)(allocators.block.allocate) << ")...";
#endif

// Constrain the request to the a valid block allocation
MemoryRequest block_request = request;
conform(user_context, &block_request);

// Create the block resource itself
BlockResource *block = static_cast<BlockResource *>(block_entry->value);
block->memory.size = constrain_requested_size(size);
block->memory.size = block_request.size;
block->memory.handle = nullptr;
block->memory.properties = properties;
block->memory.properties.nearest_multiple = max(config.nearest_multiple, properties.nearest_multiple);
block->memory.dedicated = dedicated;
block->memory.properties = block_request.properties;
block->memory.dedicated = block_request.dedicated;
block->reserved = 0;
block->allocator = create_region_allocator(user_context, block);
alloc_memory_block(user_context, block);
Expand Down Expand Up @@ -561,6 +568,33 @@ size_t BlockAllocator::constrain_requested_size(size_t size) const {
return actual_size;
}

int BlockAllocator::conform(void *user_context, MemoryRequest *request) const {

request->properties.nearest_multiple = max(config.nearest_multiple, request->properties.nearest_multiple);

if (request->properties.nearest_multiple) {
size_t nm = request->properties.nearest_multiple;
request->size = (((request->size + nm - 1) / nm) * nm); // round up to nearest multiple
}

if (config.minimum_block_size) {
request->size = ((request->size < config.minimum_block_size) ?
config.minimum_block_size :
request->size);
}
if (config.maximum_block_size) {
request->size = ((request->size > config.maximum_block_size) ?
config.maximum_block_size :
request->size);
}

if (allocators.block.conform) {
return allocators.block.conform(user_context, request);
}

return 0;
}

bool BlockAllocator::is_compatible_block(const BlockResource *block, const MemoryProperties &properties) const {
if (properties.caching != MemoryCaching::DefaultCaching) {
if (properties.caching != block->memory.properties.caching) {
Expand Down
4 changes: 4 additions & 0 deletions src/runtime/internal/memory_resources.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,18 +202,22 @@ struct HalideSystemAllocatorFns {

typedef int (*AllocateBlockFn)(void *, MemoryBlock *);
typedef int (*DeallocateBlockFn)(void *, MemoryBlock *);
typedef int (*ConformBlockRequestFn)(void *, MemoryRequest *);

struct MemoryBlockAllocatorFns {
AllocateBlockFn allocate = nullptr;
DeallocateBlockFn deallocate = nullptr;
ConformBlockRequestFn conform = nullptr;
};

typedef int (*AllocateRegionFn)(void *, MemoryRegion *);
typedef int (*DeallocateRegionFn)(void *, MemoryRegion *);
typedef int (*ConformBlockRegionFn)(void *, MemoryRequest *);

struct MemoryRegionAllocatorFns {
AllocateRegionFn allocate = nullptr;
DeallocateRegionFn deallocate = nullptr;
ConformBlockRegionFn conform = nullptr;
};

// --
Expand Down
Loading
Loading