From 9cb4f41bc526adf929dc6adf448473ab445cf1fd Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Tue, 3 Aug 2021 16:16:49 -0700 Subject: [PATCH 01/34] mem_acquire no longer allows returning NULL, initial prototype of file-system traversal with posix implementation. --- include/aws/common/file.h | 99 +++++++++++++++ source/allocator.c | 41 +++--- source/file.c | 117 ++++++++++++++++++ source/posix/file.c | 77 ++++++++++++ tests/CMakeLists.txt | 1 + tests/file_test.c | 50 +++++++- .../first_child_dir/child.txt | 1 + .../dir_traversal_test/root_child.txt | 1 + 8 files changed, 366 insertions(+), 21 deletions(-) create mode 100644 source/file.c create mode 100644 tests/resources/dir_traversal_test/first_child_dir/child.txt create mode 100644 tests/resources/dir_traversal_test/root_child.txt diff --git a/include/aws/common/file.h b/include/aws/common/file.h index ec1332665..389485224 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -4,7 +4,58 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ +#include #include +#include +#include + +#ifdef _WIN32 +static const char AWS_PATH_DELIM = '\\'; +#else +static const char AWS_PATH_DELIM = '/'; +#endif + +enum aws_file_type { + AWS_FILE_TYPE_NONE, + AWS_FILE_TYPE_FILE, + AWS_FILE_TYPE_SYM_LINK, + AWS_FILE_TYPE_DIRECTORY, +}; + +struct aws_directory_entry { + struct aws_allocator *allocator; + struct aws_string *path; + struct aws_string *relative_path; + enum aws_file_type file_type; + int64_t file_size; + struct aws_linked_list children; + struct aws_linked_list_node node; + struct aws_directory_entry *parent; + struct aws_ref_count ref_count; + void *impl; +}; + +/** + * Platform implementation of opening a file and loading its immediate children (not recursive). + * return AWS_OP_SUCCESS if successful. + */ +int aws_directory_entry_open_internal(struct aws_directory_entry *entry); + +/** + * Destroy any resources allocated in aws_directory_entry_open_internal. + */ +void aws_directory_entry_destroy_internal(struct aws_directory_entry *entry); + +/** + * General function for allocating and setting up a directory entry. Use for allocating and setting up the + * platform agnostic portions of aws_directory_entry. + * + * returns an instance of aws_directory_entry on success and NULL on failure. + */ +struct aws_directory_entry *aws_directory_entry_open_base( + struct aws_allocator *allocator, + struct aws_byte_cursor path, + struct aws_byte_cursor relative_path); AWS_EXTERN_C_BEGIN @@ -17,6 +68,54 @@ AWS_EXTERN_C_BEGIN AWS_COMMON_API FILE *aws_fopen(const char *file_path, const char *mode); +/** + * Opens a directory at path. It's ref-count will be initialized to 1 on successful return. + * You must call aws_directory_entry_release() when finished or the resource will leak. + * + * This function allocates resources for children in the directory, but only with a depth of one. To get to the + * next level, you call aws_directory_entry_descend() which will repeat this process. + */ +AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_open( + struct aws_allocator *allocator, + struct aws_byte_cursor path, + struct aws_byte_cursor relative_path); + +/** + * Holds a reference to the entry. Call this before seating an instance of aws_directory_entry(). + */ +AWS_COMMON_API void aws_directory_entry_acquire(struct aws_directory_entry *entry); + +/** + * Releases reference to entry. When the last reference is dropped, the resource will be freed. + */ +AWS_COMMON_API void aws_directory_entry_release(struct aws_directory_entry *entry); + +/** + * Get's the next directory entry at the current directory depth level as entry. Returns NULL when there are no more + * sibling entries. + */ +AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_get_next_sibling(struct aws_directory_entry *entry); + +/** + * Get's the previous directory entry at the current directory depth level as entry. Returns NULL when there are no more + * sibling entries. + */ +AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_get_prev_sibling(struct aws_directory_entry *entry); + +/** + * Descends into the next depth level for entry. Returns the first child in entry. Returns NULL if entry is not a + * directory or it's empty. The return value's reference count is owned by entry. You do not need to acquire unless + * you'll be seating the return value beyond the scope of the root entry. + */ +AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_descend(struct aws_directory_entry *entry); + +/** + * Returns the parent for the current entry unless the entry is the initial root directory from + * aws_directory_entry_open(). In that case it returns NULL. Also note, you do not need to acquire or release a + * reference to the returned value unless you plan on seating it beyond the scope of the root entry. + */ +AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_parent(struct aws_directory_entry *entry); + AWS_EXTERN_C_END #endif /* AWS_COMMON_FILE_H */ diff --git a/source/allocator.c b/source/allocator.c index eb19b8c1f..8437a5b0e 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -119,7 +119,9 @@ void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { void *mem = allocator->mem_acquire(allocator, size); if (!mem) { - aws_raise_error(AWS_ERROR_OOM); + AWS_FATAL_ASSERT( + mem && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " + "allocator implementation"); } return mem; } @@ -135,14 +137,16 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { */ size_t required_bytes; if (aws_mul_size_checked(num, size, &required_bytes)) { - return NULL; + AWS_FATAL_ASSERT("illegal size for calloc()"); } /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { void *mem = allocator->mem_calloc(allocator, num, size); if (!mem) { - aws_raise_error(AWS_ERROR_OOM); + AWS_FATAL_ASSERT( + mem && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " + "allocator implementation"); } return mem; } @@ -150,8 +154,9 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { /* Otherwise, emulate calloc */ void *mem = allocator->mem_acquire(allocator, required_bytes); if (!mem) { - aws_raise_error(AWS_ERROR_OOM); - return NULL; + AWS_FATAL_ASSERT( + mem && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " + "allocator implementation"); } memset(mem, 0, required_bytes); AWS_POSTCONDITION(mem != NULL); @@ -186,8 +191,9 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { allocation = aws_mem_acquire(allocator, total_size); if (!allocation) { - aws_raise_error(AWS_ERROR_OOM); - goto cleanup; + AWS_FATAL_ASSERT( + allocation && "memory allocation failed. If you don't want to exit when this happens, handle OOM in " + "your allocator implementation"); } uint8_t *current_ptr = allocation; @@ -204,7 +210,6 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { } } -cleanup: va_end(args_allocs); return allocation; } @@ -235,7 +240,9 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, if (allocator->mem_realloc) { void *newptr = allocator->mem_realloc(allocator, *ptr, oldsize, newsize); if (!newptr) { - return aws_raise_error(AWS_ERROR_OOM); + AWS_FATAL_ASSERT( + newptr && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " + "allocator implementation"); } *ptr = newptr; return AWS_OP_SUCCESS; @@ -248,7 +255,9 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, void *newptr = allocator->mem_acquire(allocator, newsize); if (!newptr) { - return aws_raise_error(AWS_ERROR_OOM); + AWS_FATAL_ASSERT( + newptr && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " + "allocator implementation"); } memcpy(newptr, *ptr, oldsize); @@ -274,10 +283,6 @@ static void *s_cf_allocator_allocate(CFIndex alloc_size, CFOptionFlags hint, voi void *mem = aws_mem_acquire(allocator, (size_t)alloc_size + sizeof(size_t)); - if (!mem) { - return NULL; - } - size_t allocation_size = (size_t)alloc_size + sizeof(size_t); memcpy(mem, &allocation_size, sizeof(size_t)); return (void *)((uint8_t *)mem + sizeof(size_t)); @@ -301,9 +306,7 @@ static void *s_cf_allocator_reallocate(void *ptr, CFIndex new_size, CFOptionFlag size_t original_size = 0; memcpy(&original_size, original_allocation, sizeof(size_t)); - if (aws_mem_realloc(allocator, &original_allocation, original_size, (size_t)new_size)) { - return NULL; - } + aws_mem_realloc(allocator, &original_allocation, original_size, (size_t)new_size); size_t new_allocation_size = (size_t)new_size; memcpy(original_allocation, &new_allocation_size, sizeof(size_t)); @@ -347,9 +350,7 @@ CFAllocatorRef aws_wrapped_cf_allocator_new(struct aws_allocator *allocator) { cf_allocator = CFAllocatorCreate(NULL, &context); - if (!cf_allocator) { - aws_raise_error(AWS_ERROR_OOM); - } + AWS_FATAL_ASSERT(cf_allocator && "creation of cf allocator failed!"); return cf_allocator; } diff --git a/source/file.c b/source/file.c new file mode 100644 index 000000000..3569d63dc --- /dev/null +++ b/source/file.c @@ -0,0 +1,117 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include + +static void s_ref_dropped_callback(void *dir_ent) { + struct aws_directory_entry *entry = dir_ent; + aws_directory_entry_destroy_internal(entry); + + while (!aws_linked_list_empty(&entry->children)) { + struct aws_linked_list_node *child = aws_linked_list_pop_front(&entry->children); + struct aws_directory_entry *child_entry = AWS_CONTAINER_OF(child, struct aws_directory_entry, node); + aws_directory_entry_release(child_entry); + } + + aws_string_destroy(entry->path); + aws_string_destroy(entry->relative_path); + aws_mem_release(entry->allocator, entry); +} + +struct aws_directory_entry *aws_directory_entry_open_base( + struct aws_allocator *allocator, + struct aws_byte_cursor path, + struct aws_byte_cursor relative_path) { + struct aws_directory_entry *entry = aws_mem_calloc(allocator, 1, sizeof(struct aws_directory_entry)); + + struct aws_byte_cursor trimmed_path = aws_byte_cursor_trim_pred(&path, aws_isspace); + struct aws_byte_cursor trimmed_relative_path = aws_byte_cursor_trim_pred(&relative_path, aws_isspace); + + if (trimmed_path.len && trimmed_path.ptr[trimmed_path.len - 1] == AWS_PATH_DELIM) { + trimmed_path.len -= 1; + } + + entry->path = aws_string_new_from_cursor(allocator, &trimmed_path); + + if (trimmed_relative_path.len && trimmed_relative_path.ptr[trimmed_relative_path.len - 1] == AWS_PATH_DELIM) { + trimmed_relative_path.len -= 1; + } + + entry->relative_path = aws_string_new_from_cursor(allocator, &trimmed_relative_path); + + entry->allocator = allocator; + aws_ref_count_init(&entry->ref_count, entry, s_ref_dropped_callback); + aws_linked_list_init(&entry->children); + + return entry; +} + +struct aws_directory_entry *aws_directory_entry_open( + struct aws_allocator *allocator, + struct aws_byte_cursor path, + struct aws_byte_cursor relative_path) { + struct aws_directory_entry *entry = aws_directory_entry_open_base(allocator, path, relative_path); + + if (aws_directory_entry_open_internal(entry)) { + aws_string_destroy(entry->path); + aws_string_destroy(entry->relative_path); + aws_mem_release(entry->allocator, entry); + return NULL; + } + + return entry; +} + +void aws_directory_entry_acquire(struct aws_directory_entry *entry) { + aws_ref_count_acquire(&entry->ref_count); +} + +void aws_directory_entry_release(struct aws_directory_entry *entry) { + aws_ref_count_release(&entry->ref_count); +} + +struct aws_directory_entry *aws_directory_entry_get_next_sibling(struct aws_directory_entry *entry) { + if (aws_linked_list_node_next_is_valid(&entry->node)) { + struct aws_linked_list_node *sibling = aws_linked_list_next(&entry->node); + return AWS_CONTAINER_OF(sibling, struct aws_directory_entry, node); + } + + return NULL; +} + +struct aws_directory_entry *aws_directory_entry_get_prev_sibling(struct aws_directory_entry *entry) { + if (aws_linked_list_node_prev_is_valid(&entry->node)) { + struct aws_linked_list_node *sibling = aws_linked_list_prev(&entry->node); + return AWS_CONTAINER_OF(sibling, struct aws_directory_entry, node); + } + + return NULL; +} + +struct aws_directory_entry *aws_directory_entry_parent(struct aws_directory_entry *entry) { + return entry->parent; +} + +struct aws_directory_entry *aws_directory_entry_descend(struct aws_directory_entry *entry) { + if (entry->file_type != AWS_FILE_TYPE_DIRECTORY && entry->file_type != AWS_FILE_TYPE_SYM_LINK) { + aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + return NULL; + } + + if (aws_linked_list_empty(&entry->children)) { + if (aws_directory_entry_open_internal(entry)) { + return NULL; + } + } + + if (!aws_linked_list_empty(&entry->children)) { + struct aws_linked_list_node *first_child = aws_linked_list_front(&entry->children); + return AWS_CONTAINER_OF(first_child, struct aws_directory_entry, node); + } + + aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + return NULL; +} diff --git a/source/posix/file.c b/source/posix/file.c index a99d95b84..b6d35e1d1 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -4,8 +4,85 @@ */ #include +#include + #include +#include +#include + FILE *aws_fopen(const char *file_path, const char *mode) { return fopen(file_path, mode); } + +int aws_directory_entry_open_internal(struct aws_directory_entry *entry) { + if (entry->impl) { + return AWS_OP_SUCCESS; + } + + entry->impl = opendir(aws_string_c_str(entry->path)); + + if (!entry->impl) { + return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + } + + entry->file_type = AWS_FILE_TYPE_DIRECTORY; + struct dirent *dirent = NULL; + + while ((dirent = readdir(entry->impl)) != NULL) { + struct aws_byte_cursor name_component = aws_byte_cursor_from_array(dirent->d_name, dirent->d_namlen); + + if (aws_byte_cursor_eq_c_str(&name_component, "..") || aws_byte_cursor_eq_c_str(&name_component, ".")) { + continue; + } + + struct aws_byte_buf full_path; + struct aws_byte_cursor path_component = aws_byte_cursor_from_string(entry->path); + aws_byte_buf_init_copy_from_cursor(&full_path, entry->allocator, path_component); + aws_byte_buf_append_byte_dynamic(&full_path, AWS_PATH_DELIM); + aws_byte_buf_append(&full_path, &name_component); + + struct aws_byte_buf relative_path; + struct aws_byte_cursor relative_path_component = aws_byte_cursor_from_string(entry->relative_path); + AWS_ZERO_STRUCT(relative_path); + aws_byte_buf_init_copy_from_cursor(&relative_path, entry->allocator, relative_path_component); + aws_byte_buf_append_byte_dynamic(&relative_path, AWS_PATH_DELIM); + aws_byte_buf_append(&relative_path, &name_component); + + struct aws_byte_cursor path_cur = aws_byte_cursor_from_buf(&full_path); + struct aws_byte_cursor relative_path_cur = aws_byte_cursor_from_buf(&relative_path); + + struct aws_directory_entry *new_entry = + aws_directory_entry_open_base(entry->allocator, path_cur, relative_path_cur); + /* open base set the ref count to one. */ + aws_linked_list_push_back(&entry->children, &new_entry->node); + new_entry->parent = entry; + aws_byte_buf_clean_up(&full_path); + aws_byte_buf_clean_up(&relative_path); + + struct stat dir_info; + if (!lstat(aws_string_c_str(new_entry->path), &dir_info)) { + if (S_ISDIR(dir_info.st_mode)) { + new_entry->file_type = AWS_FILE_TYPE_DIRECTORY; + } else if (S_ISLNK(dir_info.st_mode)) { + new_entry->file_type = AWS_FILE_TYPE_SYM_LINK; + } else if (S_ISREG(dir_info.st_mode)) { + new_entry->file_type = AWS_FILE_TYPE_FILE; + } else { + new_entry->file_type = AWS_FILE_TYPE_NONE; + AWS_ASSERT("Unknown file type encountered"); + } + + new_entry->file_size = dir_info.st_size; + } + } + + return AWS_OP_SUCCESS; +} + +void aws_directory_entry_destroy_internal(struct aws_directory_entry *entry) { + if (entry->impl) { + closedir(entry->impl); + entry->impl = NULL; + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 50d244501..910c0db08 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -456,6 +456,7 @@ add_test_case(test_scheduler_cancellation_for_pending_scheduled_task) add_test_case(aws_fopen_non_ascii_read_existing_file_test) add_test_case(aws_fopen_non_ascii_test) add_test_case(aws_fopen_ascii_test) +add_test_case(directory_traversal_test) add_test_case(promise_test_wait_forever) add_test_case(promise_test_wait_for_a_bit) diff --git a/tests/file_test.c b/tests/file_test.c index df6d6bc70..eb8f4c8b9 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -2,11 +2,14 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ -#include "fcntl.h" #include +#include #include +#include + + static int s_aws_fopen_test_helper(char *file_path, char *content) { char read_result[100]; AWS_ZERO_ARRAY(read_result); @@ -75,3 +78,48 @@ static int s_aws_fopen_ascii_test_fn(struct aws_allocator *allocator, void *ctx) } AWS_TEST_CASE(aws_fopen_ascii_test, s_aws_fopen_ascii_test_fn) + +static int s_directory_traversal_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_byte_cursor path = aws_byte_cursor_from_c_str("dir_traversal_test"); + + struct aws_directory_entry *root = aws_directory_entry_open(allocator, path, path); + ASSERT_NOT_NULL(root); + ASSERT_INT_EQUALS(AWS_FILE_TYPE_DIRECTORY, root->file_type); + + struct aws_directory_entry *first_child = aws_directory_entry_descend(root); + ASSERT_NOT_NULL(first_child); + + struct aws_directory_entry *child_txt = NULL; + /* this stuff isn't deterministic order wise so we gotta handle both orders. */ + if (first_child->file_type == AWS_FILE_TYPE_FILE) { + ASSERT_SUCCESS(s_aws_fopen_test_helper( + (char *)aws_string_c_str(first_child->relative_path), "dir_traversal_test->root_child.txt")); + + struct aws_directory_entry *next_child = aws_directory_entry_get_next_sibling(first_child); + ASSERT_NOT_NULL(next_child); + ASSERT_INT_EQUALS(AWS_FILE_TYPE_DIRECTORY, next_child->file_type); + + child_txt = aws_directory_entry_descend(next_child); + } else { + child_txt = aws_directory_entry_descend(first_child); + + struct aws_directory_entry *next_child = aws_directory_entry_get_next_sibling(first_child); + ASSERT_NOT_NULL(next_child); + ASSERT_INT_EQUALS(AWS_FILE_TYPE_FILE, next_child->file_type); + ASSERT_SUCCESS(s_aws_fopen_test_helper( + (char *)aws_string_c_str(next_child->relative_path), "dir_traversal_test->root_child.txt")); + } + + ASSERT_NOT_NULL(child_txt); + ASSERT_INT_EQUALS(AWS_FILE_TYPE_FILE, child_txt->file_type); + ASSERT_SUCCESS(s_aws_fopen_test_helper((char *)aws_string_c_str(child_txt->relative_path), + "dir_traversal_test->first_child_dir->child.txt")); + + + aws_directory_entry_release(root); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_traversal_test, s_directory_traversal_test_fn) diff --git a/tests/resources/dir_traversal_test/first_child_dir/child.txt b/tests/resources/dir_traversal_test/first_child_dir/child.txt new file mode 100644 index 000000000..6c696819e --- /dev/null +++ b/tests/resources/dir_traversal_test/first_child_dir/child.txt @@ -0,0 +1 @@ +dir_traversal_test->first_child_dir->child.txt \ No newline at end of file diff --git a/tests/resources/dir_traversal_test/root_child.txt b/tests/resources/dir_traversal_test/root_child.txt new file mode 100644 index 000000000..aa24fb521 --- /dev/null +++ b/tests/resources/dir_traversal_test/root_child.txt @@ -0,0 +1 @@ +dir_traversal_test->root_child.txt \ No newline at end of file From 1dc5cd41dbb9e77d0be50acf3718f866332f644c Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 12 Aug 2021 14:53:31 -0700 Subject: [PATCH 02/34] Reworked api. File system ops are now just done on strings free form. The directory traversal is now a depth-first post-order traversal. --- include/aws/common/error.h | 3 +- include/aws/common/file.h | 132 +++++++++---------- source/allocator.c | 26 +--- source/common.c | 8 ++ source/file.c | 117 ----------------- source/posix/file.c | 192 +++++++++++++++++++++------- tests/CMakeLists.txt | 8 ++ tests/file_test.c | 255 ++++++++++++++++++++++++++++++++----- 8 files changed, 454 insertions(+), 287 deletions(-) delete mode 100644 source/file.c diff --git a/include/aws/common/error.h b/include/aws/common/error.h index e4528953f..722acfa49 100644 --- a/include/aws/common/error.h +++ b/include/aws/common/error.h @@ -191,7 +191,8 @@ enum aws_common_error { AWS_ERROR_C_STRING_BUFFER_NOT_NULL_TERMINATED, AWS_ERROR_STRING_MATCH_NOT_FOUND, AWS_ERROR_DIVIDE_BY_ZERO, - + AWS_ERROR_OPERATION_INTERUPTED, + AWS_ERROR_DIRECTORY_NOT_EMPTY, AWS_ERROR_END_COMMON_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_COMMON_PACKAGE_ID) }; diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 389485224..1f59b087c 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -10,111 +10,101 @@ #include #ifdef _WIN32 -static const char AWS_PATH_DELIM = '\\'; +# define AWS_PATH_DELIM '\\' +# define AWS_PATH_DELIM_STR "\\" #else -static const char AWS_PATH_DELIM = '/'; +# define AWS_PATH_DELIM '/' +# define AWS_PATH_DELIM_STR "/" #endif +struct aws_string; + enum aws_file_type { - AWS_FILE_TYPE_NONE, - AWS_FILE_TYPE_FILE, - AWS_FILE_TYPE_SYM_LINK, - AWS_FILE_TYPE_DIRECTORY, + AWS_FILE_TYPE_FILE = 1, + AWS_FILE_TYPE_SYM_LINK = 2, + AWS_FILE_TYPE_DIRECTORY = 4, }; struct aws_directory_entry { - struct aws_allocator *allocator; - struct aws_string *path; - struct aws_string *relative_path; - enum aws_file_type file_type; + /** + * Absolute path to the entry from the current process root. + */ + struct aws_byte_cursor path; + /** + * Path to the entry relative to the current working directory. + */ + struct aws_byte_cursor relative_path; + /** + * Bit-field of enum aws_file_type + */ + int file_type; + /** + * Size of the file on disk. + */ int64_t file_size; - struct aws_linked_list children; - struct aws_linked_list_node node; - struct aws_directory_entry *parent; - struct aws_ref_count ref_count; - void *impl; }; /** - * Platform implementation of opening a file and loading its immediate children (not recursive). - * return AWS_OP_SUCCESS if successful. - */ -int aws_directory_entry_open_internal(struct aws_directory_entry *entry); - -/** - * Destroy any resources allocated in aws_directory_entry_open_internal. - */ -void aws_directory_entry_destroy_internal(struct aws_directory_entry *entry); - -/** - * General function for allocating and setting up a directory entry. Use for allocating and setting up the - * platform agnostic portions of aws_directory_entry. + * Invoked during calls to aws_directory_traverse() as an entry is encountered. entry will contain + * the parsed directory entry info. * - * returns an instance of aws_directory_entry on success and NULL on failure. + * Return true to continue the traversal, or alternatively, if you have a reason to abort the traversal, return false. */ -struct aws_directory_entry *aws_directory_entry_open_base( - struct aws_allocator *allocator, - struct aws_byte_cursor path, - struct aws_byte_cursor relative_path); +typedef bool(aws_on_directory_entry)(const struct aws_directory_entry *entry, void *user_data); AWS_EXTERN_C_BEGIN -/** - * To support non-ascii file path across platform. - * For windows, _wfopen will be invoked under the hood. For other platforms, same as calling fopen - * Functionality is the same as fopen. - * On error, errno will be set, and NULL will be returned. Same as fopen. - */ AWS_COMMON_API FILE *aws_fopen(const char *file_path, const char *mode); /** - * Opens a directory at path. It's ref-count will be initialized to 1 on successful return. - * You must call aws_directory_entry_release() when finished or the resource will leak. + * Creates a directory if it doesn't currently exist. If the directory already exists, it's ignored and assumed + * successful. * - * This function allocates resources for children in the directory, but only with a depth of one. To get to the - * next level, you call aws_directory_entry_descend() which will repeat this process. + * Returns AWS_OP_SUCCESS on success. Otherwise, check aws_last_error(). */ -AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_open( - struct aws_allocator *allocator, - struct aws_byte_cursor path, - struct aws_byte_cursor relative_path); - +AWS_COMMON_API int aws_directory_create(struct aws_string *dir_path); /** - * Holds a reference to the entry. Call this before seating an instance of aws_directory_entry(). + * Returns true if the directory currently exists. Otherwise, it returns false. */ -AWS_COMMON_API void aws_directory_entry_acquire(struct aws_directory_entry *entry); - -/** - * Releases reference to entry. When the last reference is dropped, the resource will be freed. - */ -AWS_COMMON_API void aws_directory_entry_release(struct aws_directory_entry *entry); - +AWS_COMMON_API bool aws_directory_exists(struct aws_string *dir_path); /** - * Get's the next directory entry at the current directory depth level as entry. Returns NULL when there are no more - * sibling entries. + * Deletes a directory. If the directory is not empty, this will fail unless the recursive parameter is set to true. + * If recursive is true then the entire directory and all of its contents will be deleted. If it is set to false, + * the directory will be deleted only if it is empty. Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, + * aws_last_error() will contain the error that occurred. */ -AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_get_next_sibling(struct aws_directory_entry *entry); - +AWS_COMMON_API int aws_directory_delete(struct aws_string *dir_path, struct aws_allocator *allocator, bool recursive); /** - * Get's the previous directory entry at the current directory depth level as entry. Returns NULL when there are no more - * sibling entries. + * Deletes a file. Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, + * aws_last_error() will contain the error that occurred. */ -AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_get_prev_sibling(struct aws_directory_entry *entry); +AWS_COMMON_API int aws_file_delete(struct aws_string *file_path); /** - * Descends into the next depth level for entry. Returns the first child in entry. Returns NULL if entry is not a - * directory or it's empty. The return value's reference count is owned by entry. You do not need to acquire unless - * you'll be seating the return value beyond the scope of the root entry. + * Moves directory at from to to. + * Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, + * aws_last_error() will contain the error that occurred. */ -AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_descend(struct aws_directory_entry *entry); +AWS_COMMON_API int aws_directory_or_file_move(struct aws_string *from, struct aws_string *to); /** - * Returns the parent for the current entry unless the entry is the initial root directory from - * aws_directory_entry_open(). In that case it returns NULL. Also note, you do not need to acquire or release a - * reference to the returned value unless you plan on seating it beyond the scope of the root entry. + * Traverse a directory starting at path. + * + * If you want the traversal to recursive the entire directory, pass recursive as true. Passing false for this parameter + * will only iterate the contents of the directory, but will not descend into any directories it encounters. + * + * If recursive is set to true, the traversal is performed post-order, depth-first + * (for practical reasons such as deleting a directory that contains subdirectories or files). + * + * returns AWS_OP_SUCCESS(0) on success. */ -AWS_COMMON_API struct aws_directory_entry *aws_directory_entry_parent(struct aws_directory_entry *entry); +AWS_COMMON_API int aws_directory_traverse( + struct aws_allocator *allocator, + struct aws_string *path, + bool recursive, + aws_on_directory_entry *on_entry, + void *user_data); AWS_EXTERN_C_END diff --git a/source/allocator.c b/source/allocator.c index 8437a5b0e..502ea974c 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -119,9 +119,7 @@ void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { void *mem = allocator->mem_acquire(allocator, size); if (!mem) { - AWS_FATAL_ASSERT( - mem && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " - "allocator implementation"); + AWS_FATAL_ASSERT(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); } return mem; } @@ -137,16 +135,14 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { */ size_t required_bytes; if (aws_mul_size_checked(num, size, &required_bytes)) { - AWS_FATAL_ASSERT("illegal size for calloc()"); + AWS_FATAL_ASSERT("calloc computed size > SIZE_MAX"); } /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { void *mem = allocator->mem_calloc(allocator, num, size); if (!mem) { - AWS_FATAL_ASSERT( - mem && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " - "allocator implementation"); + AWS_FATAL_ASSERT(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); } return mem; } @@ -154,9 +150,7 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { /* Otherwise, emulate calloc */ void *mem = allocator->mem_acquire(allocator, required_bytes); if (!mem) { - AWS_FATAL_ASSERT( - mem && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " - "allocator implementation"); + AWS_FATAL_ASSERT(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); } memset(mem, 0, required_bytes); AWS_POSTCONDITION(mem != NULL); @@ -191,9 +185,7 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { allocation = aws_mem_acquire(allocator, total_size); if (!allocation) { - AWS_FATAL_ASSERT( - allocation && "memory allocation failed. If you don't want to exit when this happens, handle OOM in " - "your allocator implementation"); + AWS_FATAL_ASSERT(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); } uint8_t *current_ptr = allocation; @@ -240,9 +232,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, if (allocator->mem_realloc) { void *newptr = allocator->mem_realloc(allocator, *ptr, oldsize, newsize); if (!newptr) { - AWS_FATAL_ASSERT( - newptr && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " - "allocator implementation"); + AWS_FATAL_ASSERT(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); } *ptr = newptr; return AWS_OP_SUCCESS; @@ -255,9 +245,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, void *newptr = allocator->mem_acquire(allocator, newsize); if (!newptr) { - AWS_FATAL_ASSERT( - newptr && "memory allocation failed. If you don't want to exit when this happens, handle OOM in your " - "allocator implementation"); + AWS_FATAL_ASSERT(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); } memcpy(newptr, *ptr, oldsize); diff --git a/source/common.c b/source/common.c index 15864c0d8..55d685276 100644 --- a/source/common.c +++ b/source/common.c @@ -235,6 +235,14 @@ static struct aws_error_info errors[] = { AWS_DEFINE_ERROR_INFO_COMMON( AWS_ERROR_DIVIDE_BY_ZERO, "Attempt to divide a number by zero."), + AWS_DEFINE_ERROR_INFO_COMMON( + AWS_ERROR_OPERATION_INTERUPTED, + "The operation was interrupted." + ), + AWS_DEFINE_ERROR_INFO_COMMON( + AWS_ERROR_DIRECTORY_NOT_EMPTY, + "An operation on a directory was attempted which is not allowed when the directory is not empty." + ), }; /* clang-format on */ diff --git a/source/file.c b/source/file.c deleted file mode 100644 index 3569d63dc..000000000 --- a/source/file.c +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -#include -#include - -static void s_ref_dropped_callback(void *dir_ent) { - struct aws_directory_entry *entry = dir_ent; - aws_directory_entry_destroy_internal(entry); - - while (!aws_linked_list_empty(&entry->children)) { - struct aws_linked_list_node *child = aws_linked_list_pop_front(&entry->children); - struct aws_directory_entry *child_entry = AWS_CONTAINER_OF(child, struct aws_directory_entry, node); - aws_directory_entry_release(child_entry); - } - - aws_string_destroy(entry->path); - aws_string_destroy(entry->relative_path); - aws_mem_release(entry->allocator, entry); -} - -struct aws_directory_entry *aws_directory_entry_open_base( - struct aws_allocator *allocator, - struct aws_byte_cursor path, - struct aws_byte_cursor relative_path) { - struct aws_directory_entry *entry = aws_mem_calloc(allocator, 1, sizeof(struct aws_directory_entry)); - - struct aws_byte_cursor trimmed_path = aws_byte_cursor_trim_pred(&path, aws_isspace); - struct aws_byte_cursor trimmed_relative_path = aws_byte_cursor_trim_pred(&relative_path, aws_isspace); - - if (trimmed_path.len && trimmed_path.ptr[trimmed_path.len - 1] == AWS_PATH_DELIM) { - trimmed_path.len -= 1; - } - - entry->path = aws_string_new_from_cursor(allocator, &trimmed_path); - - if (trimmed_relative_path.len && trimmed_relative_path.ptr[trimmed_relative_path.len - 1] == AWS_PATH_DELIM) { - trimmed_relative_path.len -= 1; - } - - entry->relative_path = aws_string_new_from_cursor(allocator, &trimmed_relative_path); - - entry->allocator = allocator; - aws_ref_count_init(&entry->ref_count, entry, s_ref_dropped_callback); - aws_linked_list_init(&entry->children); - - return entry; -} - -struct aws_directory_entry *aws_directory_entry_open( - struct aws_allocator *allocator, - struct aws_byte_cursor path, - struct aws_byte_cursor relative_path) { - struct aws_directory_entry *entry = aws_directory_entry_open_base(allocator, path, relative_path); - - if (aws_directory_entry_open_internal(entry)) { - aws_string_destroy(entry->path); - aws_string_destroy(entry->relative_path); - aws_mem_release(entry->allocator, entry); - return NULL; - } - - return entry; -} - -void aws_directory_entry_acquire(struct aws_directory_entry *entry) { - aws_ref_count_acquire(&entry->ref_count); -} - -void aws_directory_entry_release(struct aws_directory_entry *entry) { - aws_ref_count_release(&entry->ref_count); -} - -struct aws_directory_entry *aws_directory_entry_get_next_sibling(struct aws_directory_entry *entry) { - if (aws_linked_list_node_next_is_valid(&entry->node)) { - struct aws_linked_list_node *sibling = aws_linked_list_next(&entry->node); - return AWS_CONTAINER_OF(sibling, struct aws_directory_entry, node); - } - - return NULL; -} - -struct aws_directory_entry *aws_directory_entry_get_prev_sibling(struct aws_directory_entry *entry) { - if (aws_linked_list_node_prev_is_valid(&entry->node)) { - struct aws_linked_list_node *sibling = aws_linked_list_prev(&entry->node); - return AWS_CONTAINER_OF(sibling, struct aws_directory_entry, node); - } - - return NULL; -} - -struct aws_directory_entry *aws_directory_entry_parent(struct aws_directory_entry *entry) { - return entry->parent; -} - -struct aws_directory_entry *aws_directory_entry_descend(struct aws_directory_entry *entry) { - if (entry->file_type != AWS_FILE_TYPE_DIRECTORY && entry->file_type != AWS_FILE_TYPE_SYM_LINK) { - aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); - return NULL; - } - - if (aws_linked_list_empty(&entry->children)) { - if (aws_directory_entry_open_internal(entry)) { - return NULL; - } - } - - if (!aws_linked_list_empty(&entry->children)) { - struct aws_linked_list_node *first_child = aws_linked_list_front(&entry->children); - return AWS_CONTAINER_OF(first_child, struct aws_directory_entry, node); - } - - aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); - return NULL; -} diff --git a/source/posix/file.c b/source/posix/file.c index b6d35e1d1..243e68561 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -9,80 +9,184 @@ #include #include +#include #include +#include FILE *aws_fopen(const char *file_path, const char *mode) { return fopen(file_path, mode); } -int aws_directory_entry_open_internal(struct aws_directory_entry *entry) { - if (entry->impl) { +static int s_parse_and_raise_error(int errno_cpy) { + if (errno_cpy == 0) { return AWS_OP_SUCCESS; } - entry->impl = opendir(aws_string_c_str(entry->path)); - - if (!entry->impl) { + if (errno_cpy == ENOENT || errno_cpy == ENOTDIR) { return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); } - entry->file_type = AWS_FILE_TYPE_DIRECTORY; + if (errno_cpy == EMFILE || errno_cpy == ENFILE) { + return aws_raise_error(AWS_ERROR_MAX_FDS_EXCEEDED); + } + + if (errno_cpy == EACCES) { + return aws_raise_error(AWS_ERROR_NO_PERMISSION); + } + + if (errno_cpy == ENOTEMPTY) { + return aws_raise_error(AWS_ERROR_DIRECTORY_NOT_EMPTY); + } + + return aws_raise_error(AWS_ERROR_UNKNOWN); +} + +int aws_directory_create(struct aws_string *path) { + int mkdir_ret = mkdir(aws_string_c_str(path), S_IRWXU | S_IRWXG | S_IRWXO); + + /** nobody cares if it already existed. */ + if (mkdir_ret != 0 && errno != EEXIST) { + return s_parse_and_raise_error(errno); + } + + return AWS_OP_SUCCESS; +} + +bool aws_directory_exists(struct aws_string *dir_path) { + struct stat dir_info; + if (lstat(aws_string_c_str(dir_path), &dir_info) == 0 && S_ISDIR(dir_info.st_mode)) { + return true; + } + + return false; +} + +static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, void *user_data) { + struct aws_allocator *allocator = user_data; + + struct aws_string *path_str = aws_string_new_from_cursor(allocator, &entry->relative_path); + int ret_val = AWS_OP_SUCCESS; + + if (entry->file_type & AWS_FILE_TYPE_FILE) { + ret_val = aws_file_delete(path_str); + } + + if (entry->file_type & AWS_FILE_TYPE_DIRECTORY) { + ret_val = aws_directory_delete(path_str, allocator, false); + } + + aws_string_destroy(path_str); + return ret_val == AWS_OP_SUCCESS; +} + +int aws_directory_delete(struct aws_string *dir_path, struct aws_allocator *allocator, bool recursive) { + int ret_val = AWS_OP_SUCCESS; + + if (recursive) { + ret_val = aws_directory_traverse(allocator, dir_path, true, s_delete_file_or_directory, allocator); + } + + if (ret_val) { + return AWS_OP_ERR; + } + + int error_code = rmdir(aws_string_c_str(dir_path)); + + return error_code == 0 ? AWS_OP_SUCCESS : s_parse_and_raise_error(errno); +} + +int aws_directory_or_file_move(struct aws_string *from, struct aws_string *to) { + int error_code = rename(aws_string_c_str(from), aws_string_c_str(to)); + + return error_code == 0 ? AWS_OP_SUCCESS : s_parse_and_raise_error(errno); +} + +int aws_file_delete(struct aws_string *file_path) { + int error_code = unlink(aws_string_c_str(file_path)); + return error_code == 0 ? AWS_OP_SUCCESS : s_parse_and_raise_error(errno); +} + +int aws_directory_traverse( + struct aws_allocator *allocator, + struct aws_string *path, + bool recursive, + aws_on_directory_entry *on_entry, + void *user_data +) { + DIR *dir = opendir(aws_string_c_str(path)); + + if (!dir) { + return s_parse_and_raise_error(errno); + } + + struct aws_byte_cursor current_path = aws_byte_cursor_from_string(path); + if (current_path.ptr[current_path.len - 1] == AWS_PATH_DELIM) { + current_path.len -= 1; + } + struct dirent *dirent = NULL; + int ret_val = AWS_ERROR_SUCCESS; - while ((dirent = readdir(entry->impl)) != NULL) { + errno = 0; + while ((dirent = readdir(dir)) != NULL) { struct aws_byte_cursor name_component = aws_byte_cursor_from_array(dirent->d_name, dirent->d_namlen); if (aws_byte_cursor_eq_c_str(&name_component, "..") || aws_byte_cursor_eq_c_str(&name_component, ".")) { continue; } - struct aws_byte_buf full_path; - struct aws_byte_cursor path_component = aws_byte_cursor_from_string(entry->path); - aws_byte_buf_init_copy_from_cursor(&full_path, entry->allocator, path_component); - aws_byte_buf_append_byte_dynamic(&full_path, AWS_PATH_DELIM); - aws_byte_buf_append(&full_path, &name_component); - struct aws_byte_buf relative_path; - struct aws_byte_cursor relative_path_component = aws_byte_cursor_from_string(entry->relative_path); - AWS_ZERO_STRUCT(relative_path); - aws_byte_buf_init_copy_from_cursor(&relative_path, entry->allocator, relative_path_component); + aws_byte_buf_init_copy_from_cursor(&relative_path, allocator, current_path); aws_byte_buf_append_byte_dynamic(&relative_path, AWS_PATH_DELIM); - aws_byte_buf_append(&relative_path, &name_component); - - struct aws_byte_cursor path_cur = aws_byte_cursor_from_buf(&full_path); - struct aws_byte_cursor relative_path_cur = aws_byte_cursor_from_buf(&relative_path); + aws_byte_buf_append_dynamic(&relative_path, &name_component); + aws_byte_buf_append_byte_dynamic(&relative_path, 0); + relative_path.len -= 1; - struct aws_directory_entry *new_entry = - aws_directory_entry_open_base(entry->allocator, path_cur, relative_path_cur); - /* open base set the ref count to one. */ - aws_linked_list_push_back(&entry->children, &new_entry->node); - new_entry->parent = entry; - aws_byte_buf_clean_up(&full_path); - aws_byte_buf_clean_up(&relative_path); + struct aws_directory_entry entry; + AWS_ZERO_STRUCT(entry); struct stat dir_info; - if (!lstat(aws_string_c_str(new_entry->path), &dir_info)) { + if (!lstat((const char *)relative_path.buffer, &dir_info)) { if (S_ISDIR(dir_info.st_mode)) { - new_entry->file_type = AWS_FILE_TYPE_DIRECTORY; - } else if (S_ISLNK(dir_info.st_mode)) { - new_entry->file_type = AWS_FILE_TYPE_SYM_LINK; - } else if (S_ISREG(dir_info.st_mode)) { - new_entry->file_type = AWS_FILE_TYPE_FILE; - } else { - new_entry->file_type = AWS_FILE_TYPE_NONE; + entry.file_type |= AWS_FILE_TYPE_DIRECTORY; + } if (S_ISLNK(dir_info.st_mode)) { + entry.file_type |= AWS_FILE_TYPE_SYM_LINK; + } if (S_ISREG(dir_info.st_mode)) { + entry.file_type |= AWS_FILE_TYPE_FILE; + entry.file_size = dir_info.st_size; + } + + if (!entry.file_type){ AWS_ASSERT("Unknown file type encountered"); } - new_entry->file_size = dir_info.st_size; - } - } + entry.relative_path = aws_byte_cursor_from_buf(&relative_path); + const char *full_path = realpath((const char *)relative_path.buffer, NULL); - return AWS_OP_SUCCESS; -} + if (full_path) { + entry.path = aws_byte_cursor_from_c_str(full_path); + } -void aws_directory_entry_destroy_internal(struct aws_directory_entry *entry) { - if (entry->impl) { - closedir(entry->impl); - entry->impl = NULL; + if (recursive && entry.file_type & AWS_FILE_TYPE_DIRECTORY) { \ + struct aws_string *rel_path_str = aws_string_new_from_cursor(allocator, &entry.relative_path); + ret_val = aws_directory_traverse(allocator, rel_path_str, recursive, on_entry, user_data); + aws_string_destroy(rel_path_str); + } + + if (!on_entry(&entry, user_data)) { + aws_byte_buf_clean_up(&relative_path); + ret_val = aws_raise_error(AWS_ERROR_OPERATION_INTERUPTED); + break; + } + + aws_byte_buf_clean_up(&relative_path); + + if (ret_val) { + break; + } + } } + + closedir(dir); + return ret_val; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 910c0db08..e4fdb24f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -457,6 +457,14 @@ add_test_case(aws_fopen_non_ascii_read_existing_file_test) add_test_case(aws_fopen_non_ascii_test) add_test_case(aws_fopen_ascii_test) add_test_case(directory_traversal_test) +add_test_case(directory_traversal_stop_traversal) +add_test_case(directory_traversal_on_file_test) +add_test_case(directory_existence_test) +add_test_case(directory_creation_deletion_test) +add_test_case(directory_non_empty_deletion_fails_test) +add_test_case(directory_non_empty_deletion_recursively_succeeds_test) +add_test_case(directory_move_succeeds_test) +add_test_case(directory_move_src_non_existent_test) add_test_case(promise_test_wait_forever) add_test_case(promise_test_wait_for_a_bit) diff --git a/tests/file_test.c b/tests/file_test.c index eb8f4c8b9..736ff9db8 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -9,7 +9,6 @@ #include - static int s_aws_fopen_test_helper(char *file_path, char *content) { char read_result[100]; AWS_ZERO_ARRAY(read_result); @@ -36,6 +35,19 @@ static int s_aws_fopen_test_helper(char *file_path, char *content) { return AWS_OP_SUCCESS; } +static int s_aws_fopen_content_matches(char *file_path, char *content) { + char read_result[100]; + AWS_ZERO_ARRAY(read_result); + FILE *file = aws_fopen(file_path, "rb"); + ASSERT_NOT_NULL(file); + size_t read_len = fread(read_result, sizeof(char), strlen(content), file); + ASSERT_UINT_EQUALS(strlen(content), read_len); + fclose(file); + ASSERT_SUCCESS(strcmp(content, read_result)); + + return AWS_OP_SUCCESS; +} + static int s_aws_fopen_non_ascii_read_existing_file_test_fn(struct aws_allocator *allocator, void *ctx) { (void)allocator; (void)ctx; @@ -79,47 +91,220 @@ static int s_aws_fopen_ascii_test_fn(struct aws_allocator *allocator, void *ctx) AWS_TEST_CASE(aws_fopen_ascii_test, s_aws_fopen_ascii_test_fn) +struct directory_traversal_test_data { + bool child_dir_verified; + bool child_file_verified; + bool root_file_verified; +}; + +static const char *s_first_child_dir_path = "dir_traversal_test" AWS_PATH_DELIM_STR "first_child_dir"; + +static const char *s_first_child_file_path = + "dir_traversal_test" AWS_PATH_DELIM_STR "first_child_dir" AWS_PATH_DELIM_STR "child.txt"; + +static const char *s_root_child_path = "dir_traversal_test" AWS_PATH_DELIM_STR "root_child.txt"; + +bool s_on_directory_entry(const struct aws_directory_entry *entry, void *user_data) { + struct directory_traversal_test_data *test_data = user_data; + + if (aws_byte_cursor_eq_c_str(&entry->relative_path, s_root_child_path)) { + test_data->root_file_verified = + entry->file_type & AWS_FILE_TYPE_FILE && entry->file_size && + s_aws_fopen_content_matches((char *)entry->relative_path.ptr, "dir_traversal_test->root_child.txt") == + AWS_OP_SUCCESS; + return true; + } + + if (aws_byte_cursor_eq_c_str(&entry->relative_path, s_first_child_file_path)) { + test_data->child_file_verified = + entry->file_type & AWS_FILE_TYPE_FILE && entry->file_size && + s_aws_fopen_content_matches( + (char *)entry->relative_path.ptr, "dir_traversal_test->first_child_dir->child.txt") == AWS_OP_SUCCESS; + return true; + } + + if (aws_byte_cursor_eq_c_str(&entry->relative_path, s_first_child_dir_path)) { + test_data->child_dir_verified = entry->file_type & AWS_FILE_TYPE_DIRECTORY; + return true; + } + + return false; +} + static int s_directory_traversal_test_fn(struct aws_allocator *allocator, void *ctx) { (void)ctx; - struct aws_byte_cursor path = aws_byte_cursor_from_c_str("dir_traversal_test"); - - struct aws_directory_entry *root = aws_directory_entry_open(allocator, path, path); - ASSERT_NOT_NULL(root); - ASSERT_INT_EQUALS(AWS_FILE_TYPE_DIRECTORY, root->file_type); - - struct aws_directory_entry *first_child = aws_directory_entry_descend(root); - ASSERT_NOT_NULL(first_child); - - struct aws_directory_entry *child_txt = NULL; - /* this stuff isn't deterministic order wise so we gotta handle both orders. */ - if (first_child->file_type == AWS_FILE_TYPE_FILE) { - ASSERT_SUCCESS(s_aws_fopen_test_helper( - (char *)aws_string_c_str(first_child->relative_path), "dir_traversal_test->root_child.txt")); - - struct aws_directory_entry *next_child = aws_directory_entry_get_next_sibling(first_child); - ASSERT_NOT_NULL(next_child); - ASSERT_INT_EQUALS(AWS_FILE_TYPE_DIRECTORY, next_child->file_type); - - child_txt = aws_directory_entry_descend(next_child); - } else { - child_txt = aws_directory_entry_descend(first_child); - - struct aws_directory_entry *next_child = aws_directory_entry_get_next_sibling(first_child); - ASSERT_NOT_NULL(next_child); - ASSERT_INT_EQUALS(AWS_FILE_TYPE_FILE, next_child->file_type); - ASSERT_SUCCESS(s_aws_fopen_test_helper( - (char *)aws_string_c_str(next_child->relative_path), "dir_traversal_test->root_child.txt")); - } + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test"); + struct directory_traversal_test_data test_data; + AWS_ZERO_STRUCT(test_data); - ASSERT_NOT_NULL(child_txt); - ASSERT_INT_EQUALS(AWS_FILE_TYPE_FILE, child_txt->file_type); - ASSERT_SUCCESS(s_aws_fopen_test_helper((char *)aws_string_c_str(child_txt->relative_path), - "dir_traversal_test->first_child_dir->child.txt")); + ASSERT_SUCCESS(aws_directory_traverse(allocator, path, true, s_on_directory_entry, &test_data)); + ASSERT_TRUE(test_data.child_dir_verified); + ASSERT_TRUE(test_data.root_file_verified); + ASSERT_TRUE(test_data.child_file_verified); + AWS_ZERO_STRUCT(test_data); + ASSERT_SUCCESS(aws_directory_traverse(allocator, path, false, s_on_directory_entry, &test_data)); + ASSERT_TRUE(test_data.child_dir_verified); + ASSERT_TRUE(test_data.root_file_verified); + ASSERT_FALSE(test_data.child_file_verified); - aws_directory_entry_release(root); + aws_string_destroy(path); return AWS_OP_SUCCESS; } AWS_TEST_CASE(directory_traversal_test, s_directory_traversal_test_fn) + +struct directory_traversal_abort_test_data { + int times_called; +}; + +bool directory_traversal_abort_test_data(const struct aws_directory_entry *entry, void *user_data) { + struct directory_traversal_abort_test_data *test_data = user_data; + test_data->times_called += 1; + + return false; +} + +static int s_directory_traversal_stop_traversal_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test"); + struct directory_traversal_abort_test_data test_data; + AWS_ZERO_STRUCT(test_data); + + ASSERT_ERROR( + AWS_ERROR_OPERATION_INTERUPTED, + aws_directory_traverse(allocator, path, true, directory_traversal_abort_test_data, &test_data)); + ASSERT_INT_EQUALS(1, test_data.times_called); + + aws_string_destroy(path); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_traversal_stop_traversal, s_directory_traversal_stop_traversal_fn) + +static int s_directory_traversal_on_file_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test/root_child.txt"); + struct directory_traversal_test_data test_data; + AWS_ZERO_STRUCT(test_data); + + ASSERT_ERROR( + AWS_ERROR_FILE_INVALID_PATH, aws_directory_traverse(allocator, path, true, s_on_directory_entry, &test_data)); + + aws_string_destroy(path); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_traversal_on_file_test, s_directory_traversal_on_file_test_fn) + +static int s_directory_existence_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test"); + ASSERT_TRUE(aws_directory_exists(path)); + aws_string_destroy(path); + + path = aws_string_new_from_c_str(allocator, "dir_traversal_test_blah"); + ASSERT_FALSE(aws_directory_exists(path)); + aws_string_destroy(path); + + path = aws_string_new_from_c_str(allocator, "dir_traversal_test/root_child.txt"); + ASSERT_FALSE(aws_directory_exists(path)); + aws_string_destroy(path); + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_existence_test, s_directory_existence_test_fn) + +static int s_directory_creation_deletion_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "temp_dir"); + ASSERT_SUCCESS(aws_directory_create(path)); + + /* should be idempotent */ + ASSERT_SUCCESS(aws_directory_create(path)); + + ASSERT_TRUE(aws_directory_exists(path)); + ASSERT_SUCCESS(aws_directory_delete(path, allocator, false)); + ASSERT_FALSE(aws_directory_exists(path)); + + aws_string_destroy(path); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_creation_deletion_test, s_directory_creation_deletion_test_fn) + +static int s_directory_non_empty_deletion_fails_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test"); + ASSERT_TRUE(aws_directory_exists(path)); + ASSERT_ERROR(AWS_ERROR_DIRECTORY_NOT_EMPTY, aws_directory_delete(path, allocator, false)); + ASSERT_TRUE(aws_directory_exists(path)); + + aws_string_destroy(path); + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_non_empty_deletion_fails_test, s_directory_non_empty_deletion_fails_test_fn) + +static int s_directory_non_empty_deletion_recursively_succeeds_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "non_empty_dir_del_test_dir_1"); + ASSERT_SUCCESS(aws_directory_create(path)); + + const char *nested_dir = "test_dir_1" AWS_PATH_DELIM_STR "test_dir_2"; + struct aws_string *nested_dir_path = aws_string_new_from_c_str(allocator, nested_dir); + ASSERT_SUCCESS(aws_directory_create(nested_dir_path)); + + const char *nested_file = "test_dir_1" AWS_PATH_DELIM_STR "test_dir_2" AWS_PATH_DELIM_STR "nested_file.txt"; + + FILE *nested_file_ptr = aws_fopen(nested_file, "w"); + ASSERT_NOT_NULL(nested_file_ptr); + fclose(nested_file_ptr); + + ASSERT_SUCCESS(aws_directory_delete(path, allocator, true)); + ASSERT_FALSE(aws_directory_exists(path)); + + aws_string_destroy(nested_dir_path); + aws_string_destroy(path); + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE( + directory_non_empty_deletion_recursively_succeeds_test, + s_directory_non_empty_deletion_recursively_succeeds_test_fn) + +static int s_directory_move_succeeds_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "directory_move_succeeds_test_dir_1"); + ASSERT_SUCCESS(aws_directory_create(path)); + + struct aws_string *to_path = aws_string_new_from_c_str(allocator, "directory_move_succeeds_test_dir_2"); + ASSERT_SUCCESS(aws_directory_or_file_move(path, to_path)); + + ASSERT_FALSE(aws_directory_exists(path)); + ASSERT_TRUE(aws_directory_exists(to_path)); + + aws_string_destroy(to_path); + aws_string_destroy(path); + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_move_succeeds_test, s_directory_move_succeeds_test_fn) + +static int s_directory_move_src_non_existent_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "directory_move_src_non_existent_test_dir_1"); + + struct aws_string *to_path = aws_string_new_from_c_str(allocator, "directory_move_src_non_existent_test_dir_2"); + ASSERT_ERROR(AWS_ERROR_FILE_INVALID_PATH, aws_directory_or_file_move(path, to_path)); + + aws_string_destroy(to_path); + aws_string_destroy(path); + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_move_src_non_existent_test, s_directory_move_src_non_existent_test_fn) From 2b732b49d5f73af100cd4b0c828979e3b11131c8 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Fri, 13 Aug 2021 12:36:48 -0700 Subject: [PATCH 03/34] Cleaned up allocator api, added some documentation on the NULL return change. Fixed affected tests or deleted them. --- include/aws/common/allocator.h | 23 ++- include/aws/testing/aws_test_allocators.h | 93 ------------- source/posix/file.c | 15 +- source/ring_buffer.c | 61 -------- tests/CMakeLists.txt | 7 - tests/calloc_test.c | 16 +-- tests/file_test.c | 5 +- tests/realloc_test.c | 65 --------- tests/ring_buffer_test.c | 37 ----- tests/task_scheduler_test.c | 162 ---------------------- tests/timebomb_test.c | 41 ------ 11 files changed, 30 insertions(+), 495 deletions(-) delete mode 100644 include/aws/testing/aws_test_allocators.h delete mode 100644 tests/timebomb_test.c diff --git a/include/aws/common/allocator.h b/include/aws/common/allocator.h index ba4d9d5c1..8ff19879b 100644 --- a/include/aws/common/allocator.h +++ b/include/aws/common/allocator.h @@ -52,14 +52,20 @@ void aws_wrapped_cf_allocator_destroy(CFAllocatorRef allocator); #endif /** - * Returns at least `size` of memory ready for usage or returns NULL on failure. + * Returns at least `size` of memory ready for usage. In versions v0.6.8 and prior, this function was allowed to return + * NULL. In later versions, if allocator->mem_acquire() returns NULL, this function will assert and exit. To handle + * conditions where OOM is not a fatal error, allocator->mem_acquire() is responsible for finding/reclaiming/running a + * GC etc...before returning. */ AWS_COMMON_API void *aws_mem_acquire(struct aws_allocator *allocator, size_t size); /** * Allocates a block of memory for an array of num elements, each of them size bytes long, and initializes all its bits - * to zero. Returns null on failure. + * to zero. In versions v0.6.8 and prior, this function was allowed to return NULL. + * In later versions, if allocator->mem_calloc() returns NULL, this function will assert and exit. To handle + * conditions where OOM is not a fatal error, allocator->mem_calloc() is responsible for finding/reclaiming/running a + * GC etc...before returning. */ AWS_COMMON_API void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size); @@ -72,6 +78,11 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size); * in the same contiguous block of memory. * * Returns a pointer to the allocation. + * + * In versions v0.6.8 and prior, this function was allowed to return + * NULL. In later versions, if allocator->mem_acquire() returns NULL, this function will assert and exit. To handle + * conditions where OOM is not a fatal error, allocator->mem_acquire() is responsible for finding/reclaiming/running a + * GC etc...before returning. */ AWS_COMMON_API void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...); @@ -83,13 +94,15 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...); AWS_COMMON_API void aws_mem_release(struct aws_allocator *allocator, void *ptr); -/* +/** * Attempts to adjust the size of the pointed-to memory buffer from oldsize to * newsize. The pointer (*ptr) may be changed if the memory needs to be * reallocated. * - * If reallocation fails, *ptr is unchanged, and this method raises an - * AWS_ERROR_OOM error. + * In versions v0.6.8 and prior, this function was allowed to return + * NULL. In later versions, if allocator->mem_realloc() returns NULL, this function will assert and exit. To handle + * conditions where OOM is not a fatal error, allocator->mem_realloc() is responsible for finding/reclaiming/running a + * GC etc...before returning. */ AWS_COMMON_API int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, size_t newsize); diff --git a/include/aws/testing/aws_test_allocators.h b/include/aws/testing/aws_test_allocators.h deleted file mode 100644 index d30b7f26d..000000000 --- a/include/aws/testing/aws_test_allocators.h +++ /dev/null @@ -1,93 +0,0 @@ -#ifndef AWS_TESTING_AWS_TEST_ALLOCATORS_H -#define AWS_TESTING_AWS_TEST_ALLOCATORS_H -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -#include - -/** \file - * Alternate allocators for use in testing. - */ - -/** - * Timebomb allocator fakes running out of memory after then Nth allocation. - * Once this allocator starts failing, it never succeeds, even if memory is released. - * Wraps an existing allocator. - */ -struct aws_timebomb_impl { - size_t fail_after_n_allocations; - size_t allocation_tally; - struct aws_mutex mutex; - struct aws_allocator *wrapped_allocator; -}; - -static void *s_timebomb_mem_acquire(struct aws_allocator *timebomb_alloc, size_t size) { - struct aws_timebomb_impl *timebomb_impl = (struct aws_timebomb_impl *)timebomb_alloc->impl; - void *ptr = NULL; - - aws_mutex_lock(&timebomb_impl->mutex); - - if (timebomb_impl->allocation_tally < timebomb_impl->fail_after_n_allocations) { - timebomb_impl->allocation_tally++; - ptr = timebomb_impl->wrapped_allocator->mem_acquire(timebomb_impl->wrapped_allocator, size); - } - - aws_mutex_unlock(&timebomb_impl->mutex); - - return ptr; -} - -static void s_timebomb_mem_release(struct aws_allocator *timebomb_alloc, void *ptr) { - struct aws_timebomb_impl *timebomb_impl = (struct aws_timebomb_impl *)timebomb_alloc->impl; - - aws_mutex_lock(&timebomb_impl->mutex); - timebomb_impl->wrapped_allocator->mem_release(timebomb_impl->wrapped_allocator, ptr); - aws_mutex_unlock(&timebomb_impl->mutex); -} - -static int aws_timebomb_allocator_init( - struct aws_allocator *timebomb_allocator, - struct aws_allocator *wrapped_allocator, - size_t fail_after_n_allocations) { - - AWS_ZERO_STRUCT(*timebomb_allocator); - - struct aws_timebomb_impl *timebomb_impl = - (struct aws_timebomb_impl *)aws_mem_calloc(wrapped_allocator, 1, sizeof(struct aws_timebomb_impl)); - ASSERT_NOT_NULL(timebomb_impl); - - timebomb_allocator->mem_acquire = s_timebomb_mem_acquire; - timebomb_allocator->mem_release = s_timebomb_mem_release; - /* Not defining calloc/realloc, all allocation will be piped through the one mem_acquire fn */ - - timebomb_allocator->impl = timebomb_impl; - timebomb_impl->wrapped_allocator = wrapped_allocator; - timebomb_impl->fail_after_n_allocations = fail_after_n_allocations; - ASSERT_SUCCESS(aws_mutex_init(&timebomb_impl->mutex)); - - return AWS_OP_SUCCESS; -} - -static void aws_timebomb_allocator_clean_up(struct aws_allocator *timebomb_alloc) { - struct aws_timebomb_impl *timebomb_impl = (struct aws_timebomb_impl *)timebomb_alloc->impl; - if (timebomb_impl) { - aws_mutex_clean_up(&timebomb_impl->mutex); - aws_mem_release(timebomb_impl->wrapped_allocator, timebomb_impl); - } - AWS_ZERO_STRUCT(*timebomb_alloc); -} - -static void aws_timebomb_allocator_reset_countdown( - struct aws_allocator *timebomb_alloc, - size_t fail_after_n_allocations) { - - struct aws_timebomb_impl *timebomb_impl = (struct aws_timebomb_impl *)timebomb_alloc->impl; - aws_mutex_lock(&timebomb_impl->mutex); - timebomb_impl->allocation_tally = 0; - timebomb_impl->fail_after_n_allocations = fail_after_n_allocations; - aws_mutex_unlock(&timebomb_impl->mutex); -} - -#endif /* AWS_TESTING_AWS_TEST_ALLOCATORS_H */ diff --git a/source/posix/file.c b/source/posix/file.c index 243e68561..bd49aa295 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -68,7 +68,7 @@ static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, int ret_val = AWS_OP_SUCCESS; if (entry->file_type & AWS_FILE_TYPE_FILE) { - ret_val = aws_file_delete(path_str); + ret_val = aws_file_delete(path_str); } if (entry->file_type & AWS_FILE_TYPE_DIRECTORY) { @@ -111,8 +111,7 @@ int aws_directory_traverse( struct aws_string *path, bool recursive, aws_on_directory_entry *on_entry, - void *user_data -) { + void *user_data) { DIR *dir = opendir(aws_string_c_str(path)); if (!dir) { @@ -149,14 +148,16 @@ int aws_directory_traverse( if (!lstat((const char *)relative_path.buffer, &dir_info)) { if (S_ISDIR(dir_info.st_mode)) { entry.file_type |= AWS_FILE_TYPE_DIRECTORY; - } if (S_ISLNK(dir_info.st_mode)) { + } + if (S_ISLNK(dir_info.st_mode)) { entry.file_type |= AWS_FILE_TYPE_SYM_LINK; - } if (S_ISREG(dir_info.st_mode)) { + } + if (S_ISREG(dir_info.st_mode)) { entry.file_type |= AWS_FILE_TYPE_FILE; entry.file_size = dir_info.st_size; } - if (!entry.file_type){ + if (!entry.file_type) { AWS_ASSERT("Unknown file type encountered"); } @@ -167,7 +168,7 @@ int aws_directory_traverse( entry.path = aws_byte_cursor_from_c_str(full_path); } - if (recursive && entry.file_type & AWS_FILE_TYPE_DIRECTORY) { \ + if (recursive && entry.file_type & AWS_FILE_TYPE_DIRECTORY) { struct aws_string *rel_path_str = aws_string_new_from_cursor(allocator, &entry.relative_path); ret_val = aws_directory_traverse(allocator, rel_path_str, recursive, on_entry, user_data); aws_string_destroy(rel_path_str); diff --git a/source/ring_buffer.c b/source/ring_buffer.c index 086d6ec35..bcc8ffaad 100644 --- a/source/ring_buffer.c +++ b/source/ring_buffer.c @@ -259,64 +259,3 @@ bool aws_ring_buffer_buf_belongs_to_pool(const struct aws_ring_buffer *ring_buff AWS_POSTCONDITION(aws_byte_buf_is_valid(buf)); return rval; } - -/* Ring buffer allocator implementation */ -static void *s_ring_buffer_mem_acquire(struct aws_allocator *allocator, size_t size) { - struct aws_ring_buffer *buffer = allocator->impl; - struct aws_byte_buf buf; - AWS_ZERO_STRUCT(buf); - /* allocate extra space for the size */ - if (aws_ring_buffer_acquire(buffer, size + sizeof(size_t), &buf)) { - return NULL; - } - /* store the size ahead of the allocation */ - *((size_t *)buf.buffer) = buf.capacity; - return buf.buffer + sizeof(size_t); -} - -static void s_ring_buffer_mem_release(struct aws_allocator *allocator, void *ptr) { - /* back up to where the size is stored */ - const void *addr = ((uint8_t *)ptr - sizeof(size_t)); - const size_t size = *((size_t *)addr); - - struct aws_byte_buf buf = aws_byte_buf_from_array(addr, size); - buf.allocator = allocator; - - struct aws_ring_buffer *buffer = allocator->impl; - aws_ring_buffer_release(buffer, &buf); -} - -static void *s_ring_buffer_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { - void *mem = s_ring_buffer_mem_acquire(allocator, num * size); - if (!mem) { - return NULL; - } - memset(mem, 0, num * size); - return mem; -} - -static void *s_ring_buffer_mem_realloc(struct aws_allocator *allocator, void *ptr, size_t old_size, size_t new_size) { - (void)allocator; - (void)ptr; - (void)old_size; - (void)new_size; - AWS_FATAL_ASSERT(!"ring_buffer_allocator does not support realloc, as it breaks allocation ordering"); - return NULL; -} - -int aws_ring_buffer_allocator_init(struct aws_allocator *allocator, struct aws_ring_buffer *ring_buffer) { - if (allocator == NULL || ring_buffer == NULL) { - return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); - } - - allocator->impl = ring_buffer; - allocator->mem_acquire = s_ring_buffer_mem_acquire; - allocator->mem_release = s_ring_buffer_mem_release; - allocator->mem_calloc = s_ring_buffer_mem_calloc; - allocator->mem_realloc = s_ring_buffer_mem_realloc; - return AWS_OP_SUCCESS; -} - -void aws_ring_buffer_allocator_clean_up(struct aws_allocator *allocator) { - AWS_ZERO_STRUCT(*allocator); -} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e4fdb24f7..b2fbd1521 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -153,10 +153,8 @@ add_test_case(scheduler_pops_task_late_test) add_test_case(scheduler_has_tasks_test) add_test_case(scheduler_reentrant_safe) add_test_case(scheduler_cleanup_reentrants) -add_test_case(scheduler_oom_still_works) add_test_case(scheduler_schedule_cancellation) add_test_case(scheduler_cleanup_idempotent) -add_test_case(scheduler_oom_during_init) add_test_case(scheduler_task_delete_on_run) add_test_case(test_hash_table_create_find) @@ -292,9 +290,7 @@ add_test_case(test_platform_build_os) add_test_case(test_sanity_check_numa_discovery) add_test_case(test_realloc_fallback) -add_test_case(test_realloc_fallback_oom) add_test_case(test_realloc_passthrough) -add_test_case(test_realloc_passthrough_oom) add_test_case(test_cf_allocator_wrapper) add_test_case(test_acquire_many) add_test_case(test_alloc_nothing) @@ -316,8 +312,6 @@ add_test_case(test_calloc_fallback_from_given) add_test_case(test_calloc_from_default_allocator) add_test_case(test_calloc_from_given_allocator) -add_test_case(timebomb_allocator) - add_test_case(rw_lock_aquire_release_test) add_test_case(rw_lock_is_actually_rw_lock_test) add_test_case(rw_lock_many_readers_test) @@ -381,7 +375,6 @@ add_test_case(ring_buffer_acquire_up_to_test) add_test_case(ring_buffer_acquire_tail_always_chases_head_test) add_test_case(ring_buffer_acquire_multi_threaded_test) add_test_case(ring_buffer_acquire_up_to_multi_threaded_test) -add_test_case(ring_buffer_allocator_test) add_test_case(string_to_log_level_success_test) add_test_case(string_to_log_level_failure_test) diff --git a/tests/calloc_test.c b/tests/calloc_test.c index 8d0da0ab9..d53ffb627 100644 --- a/tests/calloc_test.c +++ b/tests/calloc_test.c @@ -29,21 +29,7 @@ static int s_test_calloc_on_given_allocator(struct aws_allocator *allocator, boo ASSERT_TRUE((intptr_t)allocator->impl == 8); } aws_mem_release(allocator, p); - /* Check that calloc handles overflow securely, by returning null - * Choose values such that [small_val == (small_val)*(large_val)(mod 2**SIZE_BITS)] - */ - for (size_t small_bits = 1; small_bits < 9; ++small_bits) { - size_t large_bits = SIZE_BITS - small_bits; - size_t small_val = (size_t)1 << small_bits; - size_t large_val = ((size_t)1 << large_bits) + 1; - ASSERT_TRUE(small_val * large_val == small_val); - ASSERT_NULL(aws_mem_calloc(allocator, small_val, large_val)); - if (using_calloc_stub_impl) { - /* Calloc should never even be called if overflow could occur */ - ASSERT_TRUE((intptr_t)allocator->impl == 0); - } - } - return 0; + return AWS_OP_SUCCESS; } AWS_TEST_CASE(test_calloc_override, s_test_calloc_override_fn) diff --git a/tests/file_test.c b/tests/file_test.c index 736ff9db8..d44201f91 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -255,11 +255,12 @@ static int s_directory_non_empty_deletion_recursively_succeeds_test_fn(struct aw struct aws_string *path = aws_string_new_from_c_str(allocator, "non_empty_dir_del_test_dir_1"); ASSERT_SUCCESS(aws_directory_create(path)); - const char *nested_dir = "test_dir_1" AWS_PATH_DELIM_STR "test_dir_2"; + const char *nested_dir = "non_empty_dir_del_test_dir_1" AWS_PATH_DELIM_STR "test_dir_2"; struct aws_string *nested_dir_path = aws_string_new_from_c_str(allocator, nested_dir); ASSERT_SUCCESS(aws_directory_create(nested_dir_path)); - const char *nested_file = "test_dir_1" AWS_PATH_DELIM_STR "test_dir_2" AWS_PATH_DELIM_STR "nested_file.txt"; + const char *nested_file = + "non_empty_dir_del_test_dir_1" AWS_PATH_DELIM_STR "test_dir_2" AWS_PATH_DELIM_STR "nested_file.txt"; FILE *nested_file_ptr = aws_fopen(nested_file, "w"); ASSERT_NOT_NULL(nested_file_ptr); diff --git a/tests/realloc_test.c b/tests/realloc_test.c index 7c1c4279c..7b6f6a682 100644 --- a/tests/realloc_test.c +++ b/tests/realloc_test.c @@ -68,20 +68,6 @@ static void *s_test_realloc(struct aws_allocator *allocator, void *ptr, size_t o return buf + 16; } -static void *s_test_malloc_failing(struct aws_allocator *allocator, size_t size) { - (void)allocator; - (void)size; - return NULL; -} - -static void *s_test_realloc_failing(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { - (void)allocator; - (void)ptr; - (void)oldsize; - (void)newsize; - return NULL; -} - static const uint8_t TEST_PATTERN[32] = {0xa5, 0x41, 0xcb, 0xe7, 0x00, 0x19, 0xd9, 0xf3, 0x60, 0x4a, 0x2b, 0x68, 0x55, 0x46, 0xb7, 0xe0, 0x74, 0x91, 0x2a, 0xbe, 0x5e, 0x41, 0x06, 0x39, 0x02, 0x02, 0xf6, 0x79, 0x1c, 0x4a, 0x08, 0xa9}; @@ -115,57 +101,6 @@ static int s_test_realloc_fallback_fn(struct aws_allocator *allocator, void *ctx return 0; } -AWS_TEST_CASE(test_realloc_fallback_oom, s_test_realloc_fallback_oom_fn) -static int s_test_realloc_fallback_oom_fn(struct aws_allocator *allocator, void *ctx) { - (void)allocator; - (void)ctx; - - struct aws_allocator test_allocator = { - .mem_acquire = s_test_alloc_acquire, - .mem_release = s_test_alloc_release, - .mem_realloc = NULL, - }; - - s_call_ct_malloc = s_call_ct_free = s_call_ct_realloc = 0; - void *buf = aws_mem_acquire(&test_allocator, 32); - void *oldbuf = buf; - - test_allocator.mem_acquire = s_test_malloc_failing; - - ASSERT_ERROR(AWS_ERROR_OOM, aws_mem_realloc(&test_allocator, &buf, 32, 64)); - ASSERT_INT_EQUALS(s_call_ct_free, 0); - ASSERT_PTR_EQUALS(buf, oldbuf); - - aws_mem_release(&test_allocator, buf); - - return 0; -} - -AWS_TEST_CASE(test_realloc_passthrough_oom, s_test_realloc_passthrough_oom_fn) -static int s_test_realloc_passthrough_oom_fn(struct aws_allocator *allocator, void *ctx) { - (void)allocator; - (void)ctx; - - struct aws_allocator test_allocator = { - .mem_acquire = s_test_alloc_acquire, - .mem_release = s_test_alloc_release, - .mem_realloc = s_test_realloc_failing, - }; - - s_call_ct_malloc = s_call_ct_free = s_call_ct_realloc = 0; - - void *buf = aws_mem_acquire(&test_allocator, 32); - void *oldbuf = buf; - memcpy(buf, TEST_PATTERN, 32); - - ASSERT_ERROR(AWS_ERROR_OOM, aws_mem_realloc(&test_allocator, &buf, 32, 64)); - ASSERT_PTR_EQUALS(buf, oldbuf); - - aws_mem_release(&test_allocator, buf); - - return 0; -} - AWS_TEST_CASE(test_realloc_passthrough, s_test_realloc_passthrough_fn) static int s_test_realloc_passthrough_fn(struct aws_allocator *allocator, void *ctx) { (void)allocator; diff --git a/tests/ring_buffer_test.c b/tests/ring_buffer_test.c index 9c2ccf725..0d5514292 100644 --- a/tests/ring_buffer_test.c +++ b/tests/ring_buffer_test.c @@ -390,40 +390,3 @@ static int s_test_acquire_up_to_multi_threaded(struct aws_allocator *allocator, } AWS_TEST_CASE(ring_buffer_acquire_up_to_multi_threaded_test, s_test_acquire_up_to_multi_threaded) - -#define RING_BUFFER_ALLOCATOR_CAPACITY (16 * 1024) - -static int s_test_ring_buffer_allocator(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_ring_buffer ring_buffer; - struct aws_allocator rb_allocator; - ASSERT_SUCCESS(aws_ring_buffer_init(&ring_buffer, allocator, RING_BUFFER_ALLOCATOR_CAPACITY)); - ASSERT_SUCCESS(aws_ring_buffer_allocator_init(&rb_allocator, &ring_buffer)); - - for (int cycles = 0; cycles < 10; ++cycles) { - size_t total_size = 0; - const size_t chunk_size = (cycles + 1) * 16; - int chunk_count = 0; - AWS_VARIABLE_LENGTH_ARRAY(void *, chunks, RING_BUFFER_ALLOCATOR_CAPACITY / (16 + sizeof(size_t))); - while ((total_size + chunk_size) <= RING_BUFFER_ALLOCATOR_CAPACITY) { - void *chunk = aws_mem_calloc(&rb_allocator, 1, chunk_size); - if (chunk == NULL) { - ASSERT_TRUE(chunk_count > 0); - break; - } - chunks[chunk_count++] = chunk; - total_size += chunk_size; - } - - for (int idx = 0; idx < chunk_count; ++idx) { - aws_mem_release(&rb_allocator, chunks[idx]); - } - } - - aws_ring_buffer_allocator_clean_up(&rb_allocator); - aws_ring_buffer_clean_up(&ring_buffer); - - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(ring_buffer_allocator_test, s_test_ring_buffer_allocator); diff --git a/tests/task_scheduler_test.c b/tests/task_scheduler_test.c index 8cd53af67..49705f48a 100644 --- a/tests/task_scheduler_test.c +++ b/tests/task_scheduler_test.c @@ -288,152 +288,6 @@ static int s_test_scheduler_cleanup_reentrants(struct aws_allocator *allocator, return AWS_OP_SUCCESS; } -/* Allocator that only works N times. Not at all thread safe. */ -struct oom_allocator_impl { - struct aws_allocator *alloc; /* normal underlying allocator */ - size_t num_allocations; - size_t num_allocations_limit; - size_t num_allocations_rejected; -}; - -static void *s_oom_allocator_acquire(struct aws_allocator *allocator, size_t size) { - struct oom_allocator_impl *impl = allocator->impl; - void *mem = NULL; - - if (impl->num_allocations < impl->num_allocations_limit) { - mem = aws_mem_acquire(impl->alloc, size); - if (mem) { - impl->num_allocations++; - } - } else { - impl->num_allocations_rejected++; - } - - return mem; -} - -static void s_oom_allocator_release(struct aws_allocator *allocator, void *ptr) { - struct oom_allocator_impl *impl = allocator->impl; - aws_mem_release(impl->alloc, ptr); -} - -static struct aws_allocator *s_oom_allocator_new(struct aws_allocator *normal_allocator, size_t num_allocations_limit) { - struct oom_allocator_impl *impl = aws_mem_acquire(normal_allocator, sizeof(struct oom_allocator_impl)); - AWS_ZERO_STRUCT(*impl); - impl->alloc = normal_allocator; - impl->num_allocations_limit = num_allocations_limit; - - struct aws_allocator *oom_allocator = aws_mem_acquire(normal_allocator, sizeof(struct aws_allocator)); - AWS_ZERO_STRUCT(*oom_allocator); - oom_allocator->mem_acquire = s_oom_allocator_acquire; - oom_allocator->mem_release = s_oom_allocator_release; - oom_allocator->impl = impl; - - return oom_allocator; -} - -static void s_oom_allocator_destroy(struct aws_allocator *oom_allocator) { - struct oom_allocator_impl *impl = oom_allocator->impl; - aws_mem_release(impl->alloc, oom_allocator); - aws_mem_release(impl->alloc, impl); -} - -static void s_oom_task_fn(struct aws_task *task, void *arg, enum aws_task_status status) { - (void)status; - struct aws_linked_list *done_list = arg; - aws_linked_list_push_back(done_list, &task->node); -} - -static int s_test_scheduler_oom_still_works(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - - /* Create allocator for scheduler that limits how many allocations it can make. - * Note that timed_queue is an array-list under the hood, so it only grabs memory at init and resize time */ - struct aws_allocator *oom_allocator = s_oom_allocator_new(allocator, 3); /* let timed_queue resize a few times */ - ASSERT_NOT_NULL(oom_allocator); - struct oom_allocator_impl *oom_impl = oom_allocator->impl; - - struct aws_task_scheduler scheduler; - ASSERT_SUCCESS(aws_task_scheduler_init(&scheduler, oom_allocator)); - - /* Pass this to each task so it can insert itself when it's done */ - struct aws_linked_list done_tasks; - aws_linked_list_init(&done_tasks); - - /* Create a bunch of tasks with random times, more tasks than the scheduler can fit in the timed_queue */ - size_t timed_queue_count = 0; - size_t timed_list_count = 0; - uint64_t highest_timestamp = 0; - do { - struct aws_task *task = aws_mem_acquire(allocator, sizeof(struct aws_task)); - ASSERT_NOT_NULL(task); - aws_task_init(task, s_oom_task_fn, &done_tasks, "scheduler_oom_still_works1"); - - size_t prev_rejects = oom_impl->num_allocations_rejected; - - /* add 1 to random time just so no future-tasks have same timestamp as now-tasks */ - uint64_t timestamp = (uint64_t)rand() + 1; - if (timestamp > highest_timestamp) { - highest_timestamp = timestamp; - } - - aws_task_scheduler_schedule_future(&scheduler, task, timestamp); - - /* If scheduling causes a rejected allocation, then task was put on timed_list */ - if (prev_rejects < oom_impl->num_allocations_rejected) { - ++timed_list_count; - } else { - ++timed_queue_count; - } - - /* Keep going until there are twice as many tasks in timed_queue as in timed_list. - * We do this exact ratio so that, when running tasks, at first the scheduler needs to choose between the two, - * but eventually it's just picking from timed_queue. */ - } while (timed_list_count * 2 < timed_queue_count); - - /* Schedule some now-tasks as well */ - size_t now_count; - for (now_count = 0; now_count < 10; ++now_count) { - struct aws_task *task = aws_mem_acquire(allocator, sizeof(struct aws_task)); - ASSERT_NOT_NULL(task); - aws_task_init(task, s_oom_task_fn, &done_tasks, "scheduler_oom_still_works2"); - - aws_task_scheduler_schedule_now(&scheduler, task); - } - - /* Run all tasks and clean up scheduler. - * Run it in a few steps, just to stress the edge-cases */ - const uint64_t num_run_steps = 4; - for (size_t run_i = 0; run_i < num_run_steps; ++run_i) { - uint64_t timestamp = (highest_timestamp / num_run_steps) * run_i; - aws_task_scheduler_run_all(&scheduler, timestamp); - } - aws_task_scheduler_run_all(&scheduler, UINT64_MAX); /* Run whatever's left */ - - aws_task_scheduler_clean_up(&scheduler); - - /* Check that tasks ran in proper order */ - uint64_t done_task_count = 0; - uint64_t prev_task_done_time = 0; - while (!aws_linked_list_empty(&done_tasks)) { - struct aws_task *task = AWS_CONTAINER_OF(aws_linked_list_pop_front(&done_tasks), struct aws_task, node); - ASSERT_TRUE( - prev_task_done_time <= task->timestamp, - "Tasks ran in wrong order: %llu before %llu", - prev_task_done_time, - task->timestamp); - aws_mem_release(allocator, task); - - done_task_count++; - } - - size_t scheduled_task_count = now_count + timed_queue_count + timed_list_count; - ASSERT_UINT_EQUALS(scheduled_task_count, done_task_count); - - s_oom_allocator_destroy(oom_allocator); - return AWS_OP_SUCCESS; -} - struct task_cancelling_task_data { struct aws_task_scheduler *scheduler; struct aws_task *task_to_cancel; @@ -532,20 +386,6 @@ static int s_test_scheduler_cleanup_idempotent(struct aws_allocator *allocator, return 0; } -static int s_test_scheduler_oom_during_init(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - - struct aws_allocator *oom_allocator = s_oom_allocator_new(allocator, 0); - ASSERT_NOT_NULL(oom_allocator); - - struct aws_task_scheduler scheduler; - ASSERT_ERROR(AWS_ERROR_OOM, aws_task_scheduler_init(&scheduler, oom_allocator)); - aws_task_scheduler_clean_up(&scheduler); - - s_oom_allocator_destroy(oom_allocator); - return 0; -} - static void s_delete_myself_fn(struct aws_task *task, void *arg, enum aws_task_status status) { (void)status; @@ -582,8 +422,6 @@ AWS_TEST_CASE(scheduler_has_tasks_test, s_test_scheduler_has_tasks); AWS_TEST_CASE(scheduler_reentrant_safe, s_test_scheduler_reentrant_safe); AWS_TEST_CASE(scheduler_cleanup_cancellation, s_test_scheduler_cleanup_cancellation); AWS_TEST_CASE(scheduler_cleanup_reentrants, s_test_scheduler_cleanup_reentrants); -AWS_TEST_CASE(scheduler_oom_still_works, s_test_scheduler_oom_still_works); AWS_TEST_CASE(scheduler_schedule_cancellation, s_test_scheduler_schedule_cancellation); AWS_TEST_CASE(scheduler_cleanup_idempotent, s_test_scheduler_cleanup_idempotent); -AWS_TEST_CASE(scheduler_oom_during_init, s_test_scheduler_oom_during_init); AWS_TEST_CASE(scheduler_task_delete_on_run, s_test_scheduler_task_delete_on_run); diff --git a/tests/timebomb_test.c b/tests/timebomb_test.c deleted file mode 100644 index 0ef362f20..000000000 --- a/tests/timebomb_test.c +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -#include - -static int s_test_timebomb_allocator(struct aws_allocator *allocator, void *ctx) { - (void)ctx; - struct aws_allocator timebomb; - ASSERT_SUCCESS(aws_timebomb_allocator_init(&timebomb, allocator, 2)); - - /* Should have two successful allocations, then failures. */ - void *one = aws_mem_acquire(&timebomb, 1); - ASSERT_NOT_NULL(one); - - void *two = aws_mem_calloc(&timebomb, 1, 1); - ASSERT_NOT_NULL(two); - - ASSERT_NULL(aws_mem_acquire(&timebomb, 1)); - ASSERT_NULL(aws_mem_acquire(&timebomb, 1)); - - /* Releasing memory should not stop the allocations from failing. */ - aws_mem_release(&timebomb, one); - ASSERT_NULL(aws_mem_acquire(&timebomb, 1)); - - /* Reset should allow allocations to succeed again (until bomb goes off). */ - aws_timebomb_allocator_reset_countdown(&timebomb, 1); - one = aws_mem_acquire(&timebomb, 1); - ASSERT_NOT_NULL(one); - - ASSERT_NULL(aws_mem_acquire(&timebomb, 1)); - - /* Clean up */ - aws_mem_release(&timebomb, one); - aws_mem_release(&timebomb, two); - aws_timebomb_allocator_clean_up(&timebomb); - return AWS_OP_SUCCESS; -} - -AWS_TEST_CASE(timebomb_allocator, s_test_timebomb_allocator); From 38067247c8a3ec240c8f29e598935ef28623a81d Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Fri, 13 Aug 2021 12:50:31 -0700 Subject: [PATCH 04/34] actually be posix compliant. --- source/allocator_sba.c | 2 ++ source/posix/file.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/allocator_sba.c b/source/allocator_sba.c index 48a163ec8..db3840521 100644 --- a/source/allocator_sba.c +++ b/source/allocator_sba.c @@ -187,12 +187,14 @@ static void s_sba_clean_up(struct small_block_allocator *sba) { void *page_addr = NULL; aws_array_list_get_at(&bin->active_pages, &page_addr, page_idx); struct page_header *page = s_page_base(page_addr); + (void)page; AWS_ASSERT(page->alloc_count == 0 && "Memory still allocated in aws_sba_allocator (bin)"); s_aligned_free(page_addr); } if (bin->page_cursor) { void *page_addr = s_page_base(bin->page_cursor); struct page_header *page = page_addr; + (void)page; AWS_ASSERT(page->alloc_count == 0 && "Memory still allocated in aws_sba_allocator (page)"); s_aligned_free(page_addr); } diff --git a/source/posix/file.c b/source/posix/file.c index bd49aa295..bd4eee99c 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -128,7 +128,9 @@ int aws_directory_traverse( errno = 0; while ((dirent = readdir(dir)) != NULL) { - struct aws_byte_cursor name_component = aws_byte_cursor_from_array(dirent->d_name, dirent->d_namlen); + /* note: dirent->name_len is only defined on the BSDs, but not linux. It's not in the + * required posix spec. So we use dirent->d_name as a c string here. */ + struct aws_byte_cursor name_component = aws_byte_cursor_from_c_str(dirent->d_name); if (aws_byte_cursor_eq_c_str(&name_component, "..") || aws_byte_cursor_eq_c_str(&name_component, ".")) { continue; From e35a908173cc41b50493968cd01d948e37d79557 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Fri, 13 Aug 2021 13:08:39 -0700 Subject: [PATCH 05/34] fix memory leak from realpath() --- source/posix/file.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/source/posix/file.c b/source/posix/file.c index bd4eee99c..5483b3e74 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -127,7 +127,7 @@ int aws_directory_traverse( int ret_val = AWS_ERROR_SUCCESS; errno = 0; - while ((dirent = readdir(dir)) != NULL) { + while (!ret_val && (dirent = readdir(dir)) != NULL) { /* note: dirent->name_len is only defined on the BSDs, but not linux. It's not in the * required posix spec. So we use dirent->d_name as a c string here. */ struct aws_byte_cursor name_component = aws_byte_cursor_from_c_str(dirent->d_name); @@ -176,17 +176,27 @@ int aws_directory_traverse( aws_string_destroy(rel_path_str); } + /* post order traversal, if a node below us ended the traversal, don't call the visitor again. */ + if (ret_val && aws_last_error() == AWS_ERROR_OPERATION_INTERUPTED) { + goto cleanup; + } + if (!on_entry(&entry, user_data)) { - aws_byte_buf_clean_up(&relative_path); ret_val = aws_raise_error(AWS_ERROR_OPERATION_INTERUPTED); - break; + goto cleanup; } - aws_byte_buf_clean_up(&relative_path); - if (ret_val) { - break; + goto cleanup; } + + cleanup: + /* per https://man7.org/linux/man-pages/man3/realpath.3.html, realpath must be freed, if NULL was passed + * to the second argument. */ + if (full_path) { + free((void *)full_path); + } + aws_byte_buf_clean_up(&relative_path); } } From 21dbc8281fa4f2cebf4942f0567e5dc98eb2654c Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Tue, 17 Aug 2021 12:09:09 -0700 Subject: [PATCH 06/34] address PR comments. Added some comments, removed allocator from directory_delete. --- include/aws/common/file.h | 7 ++++--- source/posix/file.c | 26 +++++++++++++++++++++----- tests/file_test.c | 6 +++--- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 1f59b087c..591cf11f5 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -72,12 +72,13 @@ AWS_COMMON_API bool aws_directory_exists(struct aws_string *dir_path); * Deletes a directory. If the directory is not empty, this will fail unless the recursive parameter is set to true. * If recursive is true then the entire directory and all of its contents will be deleted. If it is set to false, * the directory will be deleted only if it is empty. Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, - * aws_last_error() will contain the error that occurred. + * aws_last_error() will contain the error that occurred. If the directory doesn't exist, AWS_OP_SUCCESS is still + * returned. */ -AWS_COMMON_API int aws_directory_delete(struct aws_string *dir_path, struct aws_allocator *allocator, bool recursive); +AWS_COMMON_API int aws_directory_delete(struct aws_string *dir_path, bool recursive); /** * Deletes a file. Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, - * aws_last_error() will contain the error that occurred. + * aws_last_error() will contain the error that occurred. If the file doesn't exist, AWS_OP_SUCCESS is still returned. */ AWS_COMMON_API int aws_file_delete(struct aws_string *file_path); diff --git a/source/posix/file.c b/source/posix/file.c index 5483b3e74..b22fd4d81 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -62,7 +62,9 @@ bool aws_directory_exists(struct aws_string *dir_path) { } static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, void *user_data) { - struct aws_allocator *allocator = user_data; + (void)user_data; + + struct aws_allocator *allocator = aws_default_allocator(); struct aws_string *path_str = aws_string_new_from_cursor(allocator, &entry->relative_path); int ret_val = AWS_OP_SUCCESS; @@ -72,18 +74,27 @@ static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, } if (entry->file_type & AWS_FILE_TYPE_DIRECTORY) { - ret_val = aws_directory_delete(path_str, allocator, false); + ret_val = aws_directory_delete(path_str, false); } aws_string_destroy(path_str); return ret_val == AWS_OP_SUCCESS; } -int aws_directory_delete(struct aws_string *dir_path, struct aws_allocator *allocator, bool recursive) { +int aws_directory_delete(struct aws_string *dir_path, bool recursive) { + if (!aws_directory_exists(dir_path)) { + return AWS_OP_SUCCESS; + } + int ret_val = AWS_OP_SUCCESS; if (recursive) { - ret_val = aws_directory_traverse(allocator, dir_path, true, s_delete_file_or_directory, allocator); + ret_val = aws_directory_traverse(aws_default_allocator(), dir_path, true, s_delete_file_or_directory, NULL); + } + + if (ret_val && aws_last_error() == AWS_ERROR_FILE_INVALID_PATH) { + aws_reset_error(); + return AWS_OP_SUCCESS; } if (ret_val) { @@ -103,7 +114,12 @@ int aws_directory_or_file_move(struct aws_string *from, struct aws_string *to) { int aws_file_delete(struct aws_string *file_path) { int error_code = unlink(aws_string_c_str(file_path)); - return error_code == 0 ? AWS_OP_SUCCESS : s_parse_and_raise_error(errno); + + if (!error_code || errno == ENOENT) { + return AWS_OP_SUCCESS; + } + + return s_parse_and_raise_error(errno); } int aws_directory_traverse( diff --git a/tests/file_test.c b/tests/file_test.c index d44201f91..ac67fdc65 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -227,7 +227,7 @@ static int s_directory_creation_deletion_test_fn(struct aws_allocator *allocator ASSERT_SUCCESS(aws_directory_create(path)); ASSERT_TRUE(aws_directory_exists(path)); - ASSERT_SUCCESS(aws_directory_delete(path, allocator, false)); + ASSERT_SUCCESS(aws_directory_delete(path, false)); ASSERT_FALSE(aws_directory_exists(path)); aws_string_destroy(path); @@ -241,7 +241,7 @@ static int s_directory_non_empty_deletion_fails_test_fn(struct aws_allocator *al (void)ctx; struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test"); ASSERT_TRUE(aws_directory_exists(path)); - ASSERT_ERROR(AWS_ERROR_DIRECTORY_NOT_EMPTY, aws_directory_delete(path, allocator, false)); + ASSERT_ERROR(AWS_ERROR_DIRECTORY_NOT_EMPTY, aws_directory_delete(path, false)); ASSERT_TRUE(aws_directory_exists(path)); aws_string_destroy(path); @@ -266,7 +266,7 @@ static int s_directory_non_empty_deletion_recursively_succeeds_test_fn(struct aw ASSERT_NOT_NULL(nested_file_ptr); fclose(nested_file_ptr); - ASSERT_SUCCESS(aws_directory_delete(path, allocator, true)); + ASSERT_SUCCESS(aws_directory_delete(path, true)); ASSERT_FALSE(aws_directory_exists(path)); aws_string_destroy(nested_dir_path); From 24614b0c67b273b80032f07b184c3f6053bd36c7 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Tue, 17 Aug 2021 13:17:01 -0700 Subject: [PATCH 07/34] cleaned up wchar conversion apis, moved them to aws_string. Deprecated the missed const char * mess from aws_fopen(). --- include/aws/common/file.h | 8 +++++ include/aws/common/string.h | 28 ++++++++++++++++- source/file.c | 18 +++++++++++ source/posix/file.c | 4 +-- source/string.c | 63 +++++++++++++++++++++++++++++++++++++ source/windows/file.c | 27 +++++----------- 6 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 source/file.c diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 591cf11f5..10f9f4e9b 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -55,8 +55,16 @@ typedef bool(aws_on_directory_entry)(const struct aws_directory_entry *entry, vo AWS_EXTERN_C_BEGIN AWS_COMMON_API +/** + * Don't use this. It never should have been added in the first place. It's now deprecated. + */ FILE *aws_fopen(const char *file_path, const char *mode); +/** + * Opens file at file_path using mode. Returns the FILE pointer if successful. + */ +FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode); + /** * Creates a directory if it doesn't currently exist. If the directory already exists, it's ignored and assumed * successful. diff --git a/include/aws/common/string.h b/include/aws/common/string.h index 58eba5baf..49e252ffc 100644 --- a/include/aws/common/string.h +++ b/include/aws/common/string.h @@ -40,6 +40,8 @@ #endif struct aws_string { struct aws_allocator *const allocator; + /* size in bytes of `bytes` minus any null terminator. + * NOTE: This is not the number of characters in the string. */ const size_t len; /* give this a storage specifier for C++ purposes. It will likely be larger after init. */ const uint8_t bytes[1]; @@ -50,6 +52,30 @@ struct aws_string { AWS_EXTERN_C_BEGIN +#ifdef _WIN32 +/** + * For windows only. Converts `to_convert` to a windows whcar format (UTF-16) for use with windows OS interop. + * + * Note: `to_convert` is assumed to be UTF-8 or ASCII. + * + * returns NULL on failure. + */ +AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_str( + struct aws_allocator *allocator, + const struct aws_string *to_convert); + +/** + * Returns a wchar_t * pointer for use with windows OS interop. + */ +AWS_COMMON_API const wchar_t *aws_string_wchar_c_str(const struct aws_string *str); + +/** + * Returns the number of characters in the wchar string. NOTE: This is not the length in bytes or the buffer size. + */ +AWS_COMMON_API size_t aws_string_wchar_num_chars(const struct aws_string *str); + +#endif /* _WIN32 */ + /** * Returns true if bytes of string are the same, false otherwise. */ @@ -212,7 +238,7 @@ struct aws_byte_cursor aws_byte_cursor_from_string(const struct aws_string *src) AWS_COMMON_API struct aws_string *aws_string_clone_or_reuse(struct aws_allocator *allocator, const struct aws_string *str); -/* Computes the length of a c string in bytes assuming the character set is either ASCII or UTF-8. If no NULL character +/** Computes the length of a c string in bytes assuming the character set is either ASCII or UTF-8. If no NULL character * is found within max_read_len of str, AWS_ERROR_C_STRING_BUFFER_NOT_NULL_TERMINATED is raised. Otherwise, str_len * will contain the string length minus the NULL character, and AWS_OP_SUCCESS will be returned. */ AWS_COMMON_API diff --git a/source/file.c b/source/file.c new file mode 100644 index 000000000..261767f4e --- /dev/null +++ b/source/file.c @@ -0,0 +1,18 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include + +FILE *aws_fopen(const char *file_path, const char *mode) { + struct aws_string *file_path_str = aws_string_new_from_c_str(aws_default_allocator(), file_path); + struct aws_string *mode_str = aws_string_new_from_c_str(aws_default_allocator(), mode); + + FILE *file = aws_fopen_safe(file_path_str, mode_str); + aws_string_destroy(mode_str); + aws_string_destroy(file_path_str); + + return file; +} diff --git a/source/posix/file.c b/source/posix/file.c index b22fd4d81..ac250391f 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -13,8 +13,8 @@ #include #include -FILE *aws_fopen(const char *file_path, const char *mode) { - return fopen(file_path, mode); +FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode) { + return fopen(aws_string_c_str(file_path), aws_string_c_str(mode)); } static int s_parse_and_raise_error(int errno_cpy) { diff --git a/source/string.c b/source/string.c index d1abf0dbf..8663a6074 100644 --- a/source/string.c +++ b/source/string.c @@ -4,6 +4,69 @@ */ #include +#ifdef _WIN32 +# include + +struct aws_string *aws_string_convert_to_wchar_str( + struct aws_allocator *allocator, + const struct aws_string *to_convert) { + /* if a length is passed for the to_convert string, converted size does not include the null terminator, + * which is a good thing. */ + int converted_size = MultiByteToWideChar(CP_UTF8, 0, aws_string_c_str(to_convert), to_convert->len, NULL, 0); + + if (!converted_size) { + aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + return NULL; + } + + size_t str_len_size; + size_t malloc_size; + + /* double the size because the return value above is # of characters, not bytes size. */ + if (aws_mul_size_checked(sizeof(wchar_t), converted_size, &str_len_size)) { + return NULL; + } + + /* UTF-16, the NULL terminator is two bytes. */ + if (aws_add_size_checked(sizeof(struct aws_string) + 2, str_len_size, &malloc_size)) { + return NULL; + } + + struct aws_string *str = aws_mem_acquire(allocator, malloc_size); + if (!str) { + return NULL; + } + + /* Fields are declared const, so we need to copy them in like this */ + *(struct aws_allocator **)(&str->allocator) = allocator; + *(size_t *)(&str->len) = str_len_size; + + int converted_res = MultiByteToWideChar( + CP_UTF8, 0, aws_string_c_str(to_convert), to_convert->len, (wchar_t *)str->bytes, converted_size); + /* windows had its chance to do its thing, no take backsies. */ + AWS_FATAL_ASSERT(converted_res > 0); + + /* remember.... NULL term is TWO bytes for UTF-16. */ + *(uint8_t *)&str->bytes[str_len_size] = 0; + *(uint8_t *)&str->bytes[str_len_size + 1] = 0; + return str; +} + +const wchar_t *aws_string_wchar_c_str(const struct aws_string *str) { + return (wchar_t *)str->bytes; +} + +size_t aws_string_wchar_num_chars(const struct aws_string *str) { + if (str->len == 0) { + return 0; + } + + AWS_FATAL_ASSERT(str->len % 2 == 0); + return str->len / sizeof(whchar_t); +} + +#endif /* _WIN32 */ + struct aws_string *aws_string_new_from_c_str(struct aws_allocator *allocator, const char *c_str) { AWS_PRECONDITION(allocator && c_str); return aws_string_new_from_array(allocator, (const uint8_t *)c_str, strlen(c_str)); diff --git a/source/windows/file.c b/source/windows/file.c index e24bd7b7b..1199b7647 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -8,26 +8,15 @@ #include #include -FILE *aws_fopen(const char *file_path, const char *mode) { +FILE *aws_fopen_safe(const aws_string *file_path, const aws_string *mode) { - wchar_t w_file_path[1000]; + struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); + struct aws_string *w_mode = aws_string_convert_to_wchar_str(aws_default_allocator(), mode); + + FILE *file = _wfopen_s(&file, aws_string_wchar_c_str(w_file_path), aws_string_wchar_c_str(w_mode)); + /* actually handle the error correctly here. */ + aws_string_destroy(w_mode); + aws_string_destroy(w_file_path); - /* the default encoding is utf-8 or ascii */ - if (!MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, file_path, -1, w_file_path, AWS_ARRAY_SIZE(w_file_path))) { - /* When error happens, we need to set errno to invalid argument, since the function will set the Windows - * specific error that we don't handle */ - errno = EINVAL; - return NULL; - } - wchar_t w_mode[10]; - if (!MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, mode, -1, w_mode, AWS_ARRAY_SIZE(w_mode))) { - errno = EINVAL; - return NULL; - } - FILE *file; - if (_wfopen_s(&file, w_file_path, w_mode)) { - /* errno will be set */ - return NULL; - } return file; } From e40d40f0e11ea009cd621dd82818b40883e0d028 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Tue, 17 Aug 2021 13:27:47 -0700 Subject: [PATCH 08/34] be more const correct. --- include/aws/common/file.h | 12 ++++++------ source/posix/file.c | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 10f9f4e9b..4d258d7fc 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -71,11 +71,11 @@ FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string * * Returns AWS_OP_SUCCESS on success. Otherwise, check aws_last_error(). */ -AWS_COMMON_API int aws_directory_create(struct aws_string *dir_path); +AWS_COMMON_API int aws_directory_create(const struct aws_string *dir_path); /** * Returns true if the directory currently exists. Otherwise, it returns false. */ -AWS_COMMON_API bool aws_directory_exists(struct aws_string *dir_path); +AWS_COMMON_API bool aws_directory_exists(const struct aws_string *dir_path); /** * Deletes a directory. If the directory is not empty, this will fail unless the recursive parameter is set to true. * If recursive is true then the entire directory and all of its contents will be deleted. If it is set to false, @@ -83,19 +83,19 @@ AWS_COMMON_API bool aws_directory_exists(struct aws_string *dir_path); * aws_last_error() will contain the error that occurred. If the directory doesn't exist, AWS_OP_SUCCESS is still * returned. */ -AWS_COMMON_API int aws_directory_delete(struct aws_string *dir_path, bool recursive); +AWS_COMMON_API int aws_directory_delete(const struct aws_string *dir_path, bool recursive); /** * Deletes a file. Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, * aws_last_error() will contain the error that occurred. If the file doesn't exist, AWS_OP_SUCCESS is still returned. */ -AWS_COMMON_API int aws_file_delete(struct aws_string *file_path); +AWS_COMMON_API int aws_file_delete(const struct aws_string *file_path); /** * Moves directory at from to to. * Returns AWS_OP_SUCCESS if the operation was successful. Otherwise, * aws_last_error() will contain the error that occurred. */ -AWS_COMMON_API int aws_directory_or_file_move(struct aws_string *from, struct aws_string *to); +AWS_COMMON_API int aws_directory_or_file_move(const struct aws_string *from, const struct aws_string *to); /** * Traverse a directory starting at path. @@ -110,7 +110,7 @@ AWS_COMMON_API int aws_directory_or_file_move(struct aws_string *from, struct aw */ AWS_COMMON_API int aws_directory_traverse( struct aws_allocator *allocator, - struct aws_string *path, + const struct aws_string *path, bool recursive, aws_on_directory_entry *on_entry, void *user_data); diff --git a/source/posix/file.c b/source/posix/file.c index ac250391f..0a11f247a 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -41,8 +41,8 @@ static int s_parse_and_raise_error(int errno_cpy) { return aws_raise_error(AWS_ERROR_UNKNOWN); } -int aws_directory_create(struct aws_string *path) { - int mkdir_ret = mkdir(aws_string_c_str(path), S_IRWXU | S_IRWXG | S_IRWXO); +int aws_directory_create(const struct aws_string *dir_path) { + int mkdir_ret = mkdir(aws_string_c_str(dir_path), S_IRWXU | S_IRWXG | S_IRWXO); /** nobody cares if it already existed. */ if (mkdir_ret != 0 && errno != EEXIST) { @@ -52,7 +52,7 @@ int aws_directory_create(struct aws_string *path) { return AWS_OP_SUCCESS; } -bool aws_directory_exists(struct aws_string *dir_path) { +bool aws_directory_exists(const struct aws_string *dir_path) { struct stat dir_info; if (lstat(aws_string_c_str(dir_path), &dir_info) == 0 && S_ISDIR(dir_info.st_mode)) { return true; @@ -81,7 +81,7 @@ static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, return ret_val == AWS_OP_SUCCESS; } -int aws_directory_delete(struct aws_string *dir_path, bool recursive) { +int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { if (!aws_directory_exists(dir_path)) { return AWS_OP_SUCCESS; } @@ -106,13 +106,13 @@ int aws_directory_delete(struct aws_string *dir_path, bool recursive) { return error_code == 0 ? AWS_OP_SUCCESS : s_parse_and_raise_error(errno); } -int aws_directory_or_file_move(struct aws_string *from, struct aws_string *to) { +int aws_directory_or_file_move(const struct aws_string *from, const struct aws_string *to) { int error_code = rename(aws_string_c_str(from), aws_string_c_str(to)); return error_code == 0 ? AWS_OP_SUCCESS : s_parse_and_raise_error(errno); } -int aws_file_delete(struct aws_string *file_path) { +int aws_file_delete(const struct aws_string *file_path) { int error_code = unlink(aws_string_c_str(file_path)); if (!error_code || errno == ENOENT) { @@ -124,7 +124,7 @@ int aws_file_delete(struct aws_string *file_path) { int aws_directory_traverse( struct aws_allocator *allocator, - struct aws_string *path, + const struct aws_string *path, bool recursive, aws_on_directory_entry *on_entry, void *user_data) { From b0a6206855cb58ac0de75b23ccba5e833a4a70d2 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 18 Aug 2021 16:47:16 -0700 Subject: [PATCH 09/34] Early windows work. Sending to CI. --- include/aws/common/string.h | 16 ++ source/string.c | 56 ++++++- source/windows/file.c | 289 +++++++++++++++++++++++++++++++++++- 3 files changed, 358 insertions(+), 3 deletions(-) diff --git a/include/aws/common/string.h b/include/aws/common/string.h index 49e252ffc..7fffdccf5 100644 --- a/include/aws/common/string.h +++ b/include/aws/common/string.h @@ -64,6 +64,22 @@ AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_str( struct aws_allocator *allocator, const struct aws_string *to_convert); +AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( + struct aws_allocator *allocator, + const struct aws_byte_cursor *to_convert); + +AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_str( + struct aws_allocator *allocator, + const struct aws_string *to_convert); + +AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_byte_cursor( + struct aws_allocator *allocator, + const struct aws_byte_cursor *to_convert); + +AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_c_str( + struct aws_allocator *allocator, + const wchar_t *to_convert); + /** * Returns a wchar_t * pointer for use with windows OS interop. */ diff --git a/source/string.c b/source/string.c index 8663a6074..cd68b2796 100644 --- a/source/string.c +++ b/source/string.c @@ -10,9 +10,16 @@ struct aws_string *aws_string_convert_to_wchar_str( struct aws_allocator *allocator, const struct aws_string *to_convert) { + struct aws_byte_cursor convert_cur = aws_byte_cursor_from_string(to_convert); + return aws_string_convert_to_wchar_from_byte_cursor(allocator, &convert_cur); +} + +struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( + struct aws_allocator *allocator, + const struct aws_byte_cursor *to_convert) { /* if a length is passed for the to_convert string, converted size does not include the null terminator, * which is a good thing. */ - int converted_size = MultiByteToWideChar(CP_UTF8, 0, aws_string_c_str(to_convert), to_convert->len, NULL, 0); + int converted_size = MultiByteToWideChar(CP_UTF8, 0, (const char *)to_convert->ptr, (int)to_convert->len, NULL, 0); if (!converted_size) { aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); @@ -42,7 +49,7 @@ struct aws_string *aws_string_convert_to_wchar_str( *(size_t *)(&str->len) = str_len_size; int converted_res = MultiByteToWideChar( - CP_UTF8, 0, aws_string_c_str(to_convert), to_convert->len, (wchar_t *)str->bytes, converted_size); + CP_UTF8, 0, (const char *)to_convert->ptr, (int)to_convert->len, (wchar_t *)str->bytes, converted_size); /* windows had its chance to do its thing, no take backsies. */ AWS_FATAL_ASSERT(converted_res > 0); @@ -52,6 +59,51 @@ struct aws_string *aws_string_convert_to_wchar_str( return str; } +static struct aws_string *s_convert_from_wchar( + struct aws_allocator *allocator, + const wchar_t *to_convert, + int len_chars) { + + int bytes_size = WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, NULL, 0, NULL, NULL); + + if (!bytes_size) { + aws_raise_error(AWS_ERROR_INVALID_ARGUMENT); + return NULL; + } + + size_t malloc_size; + + if (aws_add_size_checked(sizeof(struct aws_string) + 1, bytes_size, &malloc_size)) { + return NULL; + } + + struct aws_string *str = aws_mem_acquire(allocator, malloc_size); + if (!str) { + return NULL; + } + + /* Fields are declared const, so we need to copy them in like this */ + *(struct aws_allocator **)(&str->allocator) = allocator; + *(size_t *)(&str->len) = bytes_size; + + int converted_res = WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, str->bytes, (int)str->len, NULL, NULL); + /* windows had its chance to do its thing, no take backsies. */ + AWS_FATAL_ASSERT(converted_res > 0); + + *(uint8_t *)&str->bytes[str->len] = 0; + return str; +} + +struct aws_string *aws_string_convert_from_wchar_str( + struct aws_allocator *allocator, + const struct aws_string *to_convert) { + return s_convert_from_wchar( + allocator, aws_string_wchar_c_str(to_convert), (int)aws_string_wchar_num_chars(to_convert)); +} +struct aws_string *aws_string_convert_from_wchar_c_str(struct aws_allocator *allocator, const wchar_t *to_convert) { + return s_convert_from_wchar(allocator, to_convert, -1); +} + const wchar_t *aws_string_wchar_c_str(const struct aws_string *str) { return (wchar_t *)str->bytes; } diff --git a/source/windows/file.c b/source/windows/file.c index 1199b7647..e283f091b 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -4,11 +4,12 @@ */ #include +#include #include #include #include -FILE *aws_fopen_safe(const aws_string *file_path, const aws_string *mode) { +FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode) { struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); struct aws_string *w_mode = aws_string_convert_to_wchar_str(aws_default_allocator(), mode); @@ -20,3 +21,289 @@ FILE *aws_fopen_safe(const aws_string *file_path, const aws_string *mode) { return file; } + +struct aws_string *s_to_long_path(const struct aws_string *path) { + wchar_t prefix[] = L"\\\\?\\"; + + struct aws_byte_buf new_path; + aws_byte_buf_init(&new_path, aws_default_allocator(), sizeof(prefix) + path->len + 2); + + struct aws_byte_cursor prefix_cur = aws_byte_cursor_from_array((uint8_t *)prefix, sizeof(prefix)); + aws_byte_buf_append_dynamic(&new_path, &prefix_cur); + + struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(path), path->len); + aws_byte_buf_append_dynamic(&new_path, &path_cur); + + struct aws_string *long_path = aws_string_new_from_buf(aws_default_allocator(), &new_path); + aws_byte_buf_clean_up(&new_path); + + return new_path; +} + +int aws_directory_create(const struct aws_string *dir_path) { + struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); + struct aws_string *long_dir_path = s_to_long_path(w_dir_path); + aws_string_destroy(w_dir_path); + + int create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); + aws_string_destroy(long_dir_path); + + if (create_dir_res == ERROR_PATH_NOT_FOUND) { + return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + } + + return AWS_OP_SUCCESS; +} + +bool aws_directory_exists(const struct aws_string *dir_path) { + struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); + struct aws_string *long_dir_path = s_to_long_path(w_dir_path); + aws_string_destroy(w_dir_path); + + DWORD attributes = GetFileAttributes(aws_string_wchar_c_str(long_dir_path)); + aws_string_destroy(long_dir_path); + + return (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)); +} + +static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, void *user_data) { + (void)user_data; + + struct aws_allocator *allocator = aws_default_allocator(); + + struct aws_string *path_str = aws_string_new_from_cursor(allocator, &entry->relative_path); + int ret_val = AWS_OP_SUCCESS; + + if (entry->file_type & AWS_FILE_TYPE_FILE) { + ret_val = aws_file_delete(path_str); + } + + if (entry->file_type & AWS_FILE_TYPE_DIRECTORY) { + ret_val = aws_directory_delete(path_str, false); + } + + aws_string_destroy(path_str); + return ret_val == AWS_OP_SUCCESS; +} + +int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { + if (!aws_directory_exists(dir_path)) { + return AWS_OP_SUCCESS; + } + + if (recursive) { + ret_val = aws_directory_traverse(aws_default_allocator(), dir_path, true, s_delete_file_or_directory, NULL); + } + + if (ret_val && aws_last_error() == AWS_ERROR_FILE_INVALID_PATH) { + aws_reset_error(); + return AWS_OP_SUCCESS; + } + + if (ret_val) { + return AWS_OP_ERR; + } + + struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); + struct aws_string *long_dir_path = s_to_long_path(w_dir_path); + aws_string_destroy(w_dir_path); + + BOOL remove_dir_res = RemoveDirectoryW(aws_string_wchar_c_str(long_dir_path)); + aws_string_destroy(long_dir_path); + + if (!remove_dir_res) { + int error = GetLastError(); + if (error == ERROR_DIR_NOT_EMPTY) { + return aws_raise_error(AWS_ERROR_DIRECTORY_NOT_EMPTY); + } + + return aws_raise_error(AWS_ERROR_UNKNOWN); + } + + return AWS_OP_SUCCESS; +} + +int aws_file_delete(const struct aws_string *file_path) { + struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); + struct aws_string *long_file_path = s_to_long_path(w_file_path); + aws_string_destroy(w_file_path); + + BOOL remove_file_res = DeleteFileW(aws_string_wchar_c_str(long_file_path)); + aws_string_destroy(long_file_path); + + if (!remove_file_res) { + int error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND) { + return AWS_OP_SUCCESS; + } + + if (error == ERROR_ACCESS_DENIED) { + return aws_raise_error(AWS_ERROR_NO_PERMISSION); + } + + return aws_raise_error(AWS_ERROR_UNKNOWN); + } + + return AWS_OP_SUCCESS; +} + +int aws_directory_or_file_move(const struct aws_string *from, const struct aws_string *to) { + struct aws_string *w_from_path = aws_string_convert_to_wchar_str(aws_default_allocator(), from); + struct aws_string *long_from_path = s_to_long_path(w_from_path); + aws_string_destroy(w_from_path); + + struct aws_string *w_to_path = aws_string_convert_to_wchar_str(aws_default_allocator(), to); + struct aws_string *long_to_path = s_to_long_path(w_to_path); + aws_string_destroy(w_to_path); + + BOOL move_res = MoveFileW(aws_string_wchar_c_str(long_from_path), aws_string_wchar_c_str(long_to_path)); + aws_string_destroy(long_from_path); + aws_string_destroy(long_to_path); + + if (!move_res) { + int error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND) { + return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + } + + if (error == ERROR_ACCESS_DENIED) { + return aws_raise_error(AWS_ERROR_NO_PERMISSION); + } + + return aws_raise_error(AWS_ERROR_UNKNOWN); + } + + return AWS_OP_SUCCESS; +} + +int aws_directory_traverse( + struct aws_allocator *allocator, + const struct aws_string *path, + bool recursive, + aws_on_directory_entry *on_entry, + void *user_data) { + struct aws_string *w_path = aws_string_convert_to_wchar_str(allocator, path); + struct aws_string *long_path = s_to_long_path(w_path); + aws_string_destroy(w_path); + + WIN32_FIND_DATAW ffd; + HANDLE find_handle = FindFirstFileW(aws_string_wchar_c_str(long_path), &ffd); + + if (find_handle == INVALID_HANDLE_VALUE) { + aws_string_destroy(long_path); + + int error = GetLastError(); + + if (error == ERROR_FILE_NOT_FOUND) { + return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + } + + return aws_raise_error(AWS_ERROR_UNKNOWN); + } + + FindClose(find_handle); + /* create search path string */ + struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(long_path), long_path->len); + struct aws_byte_buf search_buf; + aws_byte_buf_init_copy_from_cursor(&search_buf, allocator, &path_cur); + aws_string_destroy(long_path); + wchar_t search_wchar_pattern[] = L"\\*"; + struct aws_byte_cursor search_char = + aws_byte_cursor_from_array((uint8_t *)search_wchar_pattern, sizeof(search_wchar_pattern)); + + aws_byte_buf_append_dynamic(&search_buf, &search_char); + /* it's already converted to wide string */ + struct aws_string *search_string = aws_string_new_from_buf(allocator, &search_buf); + search_buf.len -= sizeof(search_wchar_pattern); + + find_handle = FindFirstFileW(aws_string_wchar_c_str(search_string), &ffd); + aws_string_destroy(search_string); + aws_byte_buf_clean_up(&search_buf); + + int ret_val = AWS_OP_SUCCESS; + + while (ret_val == AWS_OP_SUCCESS) { + struct aws_string *name_component_str = aws_string_convert_from_wchar_c_str(ffd.cFileName); + struct aws_byte_cursor name_component = aws_byte_cursor_from_string(name_component_str); + + if (aws_byte_cursor_eq_c_str(&name_component, "..") || aws_byte_cursor_eq_c_str(&name_component, ".")) { + aws_string_destroy(name_component_str); + continue; + } + + struct aws_byte_buf relative_path; + aws_byte_buf_init_copy_from_cursor(&relative_path, allocator, current_path); + aws_byte_buf_append_byte_dynamic(&relative_path, AWS_PATH_DELIM); + aws_byte_buf_append_dynamic(&relative_path, &name_component); + aws_byte_buf_append_byte_dynamic(&relative_path, 0); + relative_path.len -= 1; + + struct aws_byte_cursor relative_path_cur = aws_byte_cursor_from_buf(&relative_path); + struct aws_string *wchar_short_name = + aws_string_convert_to_wchar_from_byte_cursor(allocator, &relative_path_cur); + DWORD path_res = GetFullPathNameW(aws_string_wchar_c_str(wchar_short_name), 0, NULL, NULL); + + AWS_FATAL_ASSERT(path_res > 0); + struct aws_byte_buf full_path_buf; + aws_byte_buf_init(&full_path_buf, allocator, (size_t)path_res * sizeof(wchar_t) + 2); + + full_path_buf.len = path_res + 1; + path_res = GetFullPathNameW( + aws_string_wchar_c_str(wchar_short_name), (DWORD)wchar_short_name->len, full_path_buf.buffer, NULL); + AWS_FATAL_ASSERT(path_res > 0); + aws_string_destroy(wchar_short_name); + + struct aws_string *full_path_name_converted = + aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)full_path_buf.buffer); + aws_byte_buf_clean_up(&full_path_buf); + + struct aws_directory_entry entry; + AWS_ZERO_STRUCT(entry); + entry.relative_path = relative_path_cur; + entry.path = aws_byte_cursor_from_string(full_path_name_converted); + + LARGE_INTEGER file_size; + file_size.HighPart = ffd.nFileSizeHigh; + file_size.LowPart = ffd.nFileSizeLow; + entry.file_size = (int64_t)file_size.QuadPart; + + if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + entry.file_type |= AWS_FILE_TYPE_DIRECTORY; + } else { + entry.file_type |= AWS_FILE_TYPE_FILE; + } + + if (recursive && entry.file_type & AWS_FILE_TYPE_DIRECTORY) { + struct aws_string *rel_path_str = aws_string_new_from_cursor(allocator, &entry.relative_path); + ret_val = aws_directory_traverse(allocator, rel_path_str, recursive, on_entry, user_data); + aws_string_destroy(rel_path_str); + } + + /* post order traversal, if a node below us ended the traversal, don't call the visitor again. */ + if (ret_val && aws_last_error() == AWS_ERROR_OPERATION_INTERUPTED) { + goto cleanup; + } + + if (!on_entry(&entry, user_data)) { + ret_val = aws_raise_error(AWS_ERROR_OPERATION_INTERUPTED); + goto cleanup; + } + + if (ret_val) { + goto cleanup; + } + + cleanup: + aws_string_destroy(full_path_name_converted); + aws_byte_buf_clean_up(&relative_path); + + if (!FindNextFileW(find_handle, &ffd)) { + break; + } + } + + if (find_handle != INVALID_HANDLE_VALUE) { + FindClose(find_handle); + } + return ret_val; +} From 0eb96a92d5db11ff205447740efdfb019450633d Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 19 Aug 2021 12:37:07 -0700 Subject: [PATCH 10/34] windows comiple fixes. --- source/string.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/string.c b/source/string.c index cd68b2796..28a05ed7f 100644 --- a/source/string.c +++ b/source/string.c @@ -86,7 +86,8 @@ static struct aws_string *s_convert_from_wchar( *(struct aws_allocator **)(&str->allocator) = allocator; *(size_t *)(&str->len) = bytes_size; - int converted_res = WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, str->bytes, (int)str->len, NULL, NULL); + int converted_res = + WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, (char *)str->bytes, (int)str->len, NULL, NULL); /* windows had its chance to do its thing, no take backsies. */ AWS_FATAL_ASSERT(converted_res > 0); @@ -114,7 +115,7 @@ size_t aws_string_wchar_num_chars(const struct aws_string *str) { } AWS_FATAL_ASSERT(str->len % 2 == 0); - return str->len / sizeof(whchar_t); + return str->len / sizeof(wchar_t); } #endif /* _WIN32 */ From 869d85de9a1c267ffe526acfa2fdf34fd9538d59 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 19 Aug 2021 12:48:52 -0700 Subject: [PATCH 11/34] Not like that tho --- source/windows/file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/windows/file.c b/source/windows/file.c index ea0bde4ed..87eb7c00a 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -14,8 +14,8 @@ FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); struct aws_string *w_mode = aws_string_convert_to_wchar_str(aws_default_allocator(), mode); - FILE *file = NULL : errno_t error = - _wfopen_s(&file, aws_string_wchar_c_str(w_file_path), aws_string_wchar_c_str(w_mode)); + FILE *file = NULL; + errno_t error = _wfopen_s(&file, aws_string_wchar_c_str(w_file_path), aws_string_wchar_c_str(w_mode)); /* actually handle the error correctly here. */ aws_string_destroy(w_mode); aws_string_destroy(w_file_path); From d1b88e7e6be6338e26143d325768186198e1cdb5 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 19 Aug 2021 12:56:05 -0700 Subject: [PATCH 12/34] More build fixes. --- source/windows/file.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/windows/file.c b/source/windows/file.c index 87eb7c00a..13a24d48a 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -42,7 +42,7 @@ struct aws_string *s_to_long_path(const struct aws_string *path) { struct aws_string *long_path = aws_string_new_from_buf(aws_default_allocator(), &new_path); aws_byte_buf_clean_up(&new_path); - return new_path; + return long_path; } int aws_directory_create(const struct aws_string *dir_path) { @@ -65,10 +65,10 @@ bool aws_directory_exists(const struct aws_string *dir_path) { struct aws_string *long_dir_path = s_to_long_path(w_dir_path); aws_string_destroy(w_dir_path); - DWORD attributes = GetFileAttributes(aws_string_wchar_c_str(long_dir_path)); + DWORD attributes = GetFileAttributesW(aws_string_wchar_c_str(long_dir_path)); aws_string_destroy(long_dir_path); - return (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)); + return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY)); } static bool s_delete_file_or_directory(const struct aws_directory_entry *entry, void *user_data) { @@ -96,6 +96,8 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { return AWS_OP_SUCCESS; } + int ret_val = AWS_OP_SUCCESS; + if (recursive) { ret_val = aws_directory_traverse(aws_default_allocator(), dir_path, true, s_delete_file_or_directory, NULL); } From ab2bebe7201eac6d989764cdcd87697b1f4b1a6a Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 19 Aug 2021 13:02:59 -0700 Subject: [PATCH 13/34] more build fixes. --- source/windows/file.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/windows/file.c b/source/windows/file.c index 13a24d48a..b1e244fdc 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -212,7 +212,7 @@ int aws_directory_traverse( /* create search path string */ struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(long_path), long_path->len); struct aws_byte_buf search_buf; - aws_byte_buf_init_copy_from_cursor(&search_buf, allocator, &path_cur); + aws_byte_buf_init_copy_from_cursor(&search_buf, allocator, path_cur); aws_string_destroy(long_path); wchar_t search_wchar_pattern[] = L"\\*"; struct aws_byte_cursor search_char = @@ -230,7 +230,7 @@ int aws_directory_traverse( int ret_val = AWS_OP_SUCCESS; while (ret_val == AWS_OP_SUCCESS) { - struct aws_string *name_component_str = aws_string_convert_from_wchar_c_str(ffd.cFileName); + struct aws_string *name_component_str = aws_string_convert_from_wchar_c_str(allocator, ffd.cFileName); struct aws_byte_cursor name_component = aws_byte_cursor_from_string(name_component_str); if (aws_byte_cursor_eq_c_str(&name_component, "..") || aws_byte_cursor_eq_c_str(&name_component, ".")) { @@ -239,7 +239,7 @@ int aws_directory_traverse( } struct aws_byte_buf relative_path; - aws_byte_buf_init_copy_from_cursor(&relative_path, allocator, current_path); + aws_byte_buf_init_copy_from_cursor(&relative_path, allocator, path_cur); aws_byte_buf_append_byte_dynamic(&relative_path, AWS_PATH_DELIM); aws_byte_buf_append_dynamic(&relative_path, &name_component); aws_byte_buf_append_byte_dynamic(&relative_path, 0); @@ -256,7 +256,10 @@ int aws_directory_traverse( full_path_buf.len = path_res + 1; path_res = GetFullPathNameW( - aws_string_wchar_c_str(wchar_short_name), (DWORD)wchar_short_name->len, full_path_buf.buffer, NULL); + aws_string_wchar_c_str(wchar_short_name), + (DWORD)wchar_short_name->len, + (wchar_t *)full_path_buf.buffer, + NULL); AWS_FATAL_ASSERT(path_res > 0); aws_string_destroy(wchar_short_name); From f1e6bca9ec8541c8c8a766a8b8c88f96e412f00b Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 19 Aug 2021 13:13:58 -0700 Subject: [PATCH 14/34] github actions are my windows IDE. --- include/aws/common/file.h | 5 ++--- source/allocator.c | 2 +- tests/file_test.c | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 4d258d7fc..84ae6149d 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -54,16 +54,15 @@ typedef bool(aws_on_directory_entry)(const struct aws_directory_entry *entry, vo AWS_EXTERN_C_BEGIN -AWS_COMMON_API /** * Don't use this. It never should have been added in the first place. It's now deprecated. */ -FILE *aws_fopen(const char *file_path, const char *mode); +AWS_COMMON_API FILE *aws_fopen(const char *file_path, const char *mode); /** * Opens file at file_path using mode. Returns the FILE pointer if successful. */ -FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode); +AWS_COMMON_API FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode); /** * Creates a directory if it doesn't currently exist. If the directory already exists, it's ignored and assumed diff --git a/source/allocator.c b/source/allocator.c index 502ea974c..9ce7deea2 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -133,7 +133,7 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { /* Defensive check: never use calloc with size * num that would overflow * https://wiki.sei.cmu.edu/confluence/display/c/MEM07-C.+Ensure+that+the+arguments+to+calloc%28%29%2C+when+multiplied%2C+do+not+wrap */ - size_t required_bytes; + size_t required_bytes = 0; if (aws_mul_size_checked(num, size, &required_bytes)) { AWS_FATAL_ASSERT("calloc computed size > SIZE_MAX"); } diff --git a/tests/file_test.c b/tests/file_test.c index ac67fdc65..e6beef661 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -160,6 +160,7 @@ struct directory_traversal_abort_test_data { }; bool directory_traversal_abort_test_data(const struct aws_directory_entry *entry, void *user_data) { + (void)entry; struct directory_traversal_abort_test_data *test_data = user_data; test_data->times_called += 1; From 713c7d5c7c7158977a2b20bc015ba345002da24e Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 12:06:10 -0700 Subject: [PATCH 15/34] Okay tests pass now on windows. Still need to do a format run from a different machine though. --- include/aws/common/string.h | 28 ++++++ source/string.c | 18 ++-- source/windows/file.c | 169 ++++++++++++++++++++++-------------- 3 files changed, 142 insertions(+), 73 deletions(-) diff --git a/include/aws/common/string.h b/include/aws/common/string.h index 7fffdccf5..4a1a85c7d 100644 --- a/include/aws/common/string.h +++ b/include/aws/common/string.h @@ -64,18 +64,46 @@ AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_str( struct aws_allocator *allocator, const struct aws_string *to_convert); +/** + * For windows only. Converts `to_convert` to a windows whcar format (UTF-16) for use with windows OS interop. + * + * Note: `to_convert` is assumed to be UTF-8 or ASCII. + * + * returns NULL on failure. + */ AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( struct aws_allocator *allocator, const struct aws_byte_cursor *to_convert); +/** + * For windows only. Converts `to_convert` from a windows whcar format (UTF-16) to UTF-8. + * + * Note: `to_convert` is assumed to be wchar already. + * + * returns NULL on failure. + */ AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_str( struct aws_allocator *allocator, const struct aws_string *to_convert); +/** + * For windows only. Converts `to_convert` from a windows whcar format (UTF-16) to UTF-8. + * + * Note: `to_convert` is assumed to be wchar already. + * + * returns NULL on failure. + */ AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_byte_cursor( struct aws_allocator *allocator, const struct aws_byte_cursor *to_convert); +/** + * For windows only. Converts `to_convert` from a windows whcar format (UTF-16) to UTF-8. + * + * Note: `to_convert` is assumed to be wchar already. + * + * returns NULL on failure. + */ AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_c_str( struct aws_allocator *allocator, const wchar_t *to_convert); diff --git a/source/string.c b/source/string.c index 28a05ed7f..cfee6886b 100644 --- a/source/string.c +++ b/source/string.c @@ -26,8 +26,8 @@ struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( return NULL; } - size_t str_len_size; - size_t malloc_size; + size_t str_len_size = 0; + size_t malloc_size = 0; /* double the size because the return value above is # of characters, not bytes size. */ if (aws_mul_size_checked(sizeof(wchar_t), converted_size, &str_len_size)) { @@ -71,9 +71,10 @@ static struct aws_string *s_convert_from_wchar( return NULL; } - size_t malloc_size; + size_t malloc_size = 0; - if (aws_add_size_checked(sizeof(struct aws_string) + 1, bytes_size, &malloc_size)) { + /* bytes_size already contains the space for the null terminator */ + if (aws_add_size_checked(sizeof(struct aws_string), bytes_size, &malloc_size)) { return NULL; } @@ -84,10 +85,10 @@ static struct aws_string *s_convert_from_wchar( /* Fields are declared const, so we need to copy them in like this */ *(struct aws_allocator **)(&str->allocator) = allocator; - *(size_t *)(&str->len) = bytes_size; + *(size_t *)(&str->len) = (size_t)bytes_size - 1; int converted_res = - WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, (char *)str->bytes, (int)str->len, NULL, NULL); + WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, (char *)str->bytes, bytes_size, NULL, NULL); /* windows had its chance to do its thing, no take backsies. */ AWS_FATAL_ASSERT(converted_res > 0); @@ -129,7 +130,7 @@ struct aws_string *aws_string_new_from_array(struct aws_allocator *allocator, co AWS_PRECONDITION(allocator); AWS_PRECONDITION(AWS_MEM_IS_READABLE(bytes, len)); size_t malloc_size; - if (aws_add_size_checked(sizeof(struct aws_string) + 1, len, &malloc_size)) { + if (aws_add_size_checked(sizeof(struct aws_string) + 2, len, &malloc_size)) { return NULL; } struct aws_string *str = aws_mem_acquire(allocator, malloc_size); @@ -143,7 +144,8 @@ struct aws_string *aws_string_new_from_array(struct aws_allocator *allocator, co if (len > 0) { memcpy((void *)str->bytes, bytes, len); } - *(uint8_t *)&str->bytes[len] = '\0'; + *(uint8_t *)&str->bytes[len] = 0; + *(uint8_t *)&str->bytes[len + 1] = 0; AWS_RETURN_WITH_POSTCONDITION(str, aws_string_is_valid(str)); } diff --git a/source/windows/file.c b/source/windows/file.c index b1e244fdc..054e7bd84 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -28,21 +28,27 @@ FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string } struct aws_string *s_to_long_path(const struct aws_string *path) { + wchar_t prefix[] = L"\\\\?\\"; + size_t prefix_size = sizeof(prefix); + if (path->len > MAX_PATH - prefix_size) { + + struct aws_byte_buf new_path; + aws_byte_buf_init(&new_path, aws_default_allocator(), sizeof(prefix) + path->len + 2); - struct aws_byte_buf new_path; - aws_byte_buf_init(&new_path, aws_default_allocator(), sizeof(prefix) + path->len + 2); + struct aws_byte_cursor prefix_cur = aws_byte_cursor_from_array((uint8_t *)prefix, sizeof(prefix) - 2); + aws_byte_buf_append_dynamic(&new_path, &prefix_cur); - struct aws_byte_cursor prefix_cur = aws_byte_cursor_from_array((uint8_t *)prefix, sizeof(prefix)); - aws_byte_buf_append_dynamic(&new_path, &prefix_cur); + struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(path), path->len); + aws_byte_buf_append_dynamic(&new_path, &path_cur); - struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(path), path->len); - aws_byte_buf_append_dynamic(&new_path, &path_cur); + struct aws_string *long_path = aws_string_new_from_buf(aws_default_allocator(), &new_path); + aws_byte_buf_clean_up(&new_path); - struct aws_string *long_path = aws_string_new_from_buf(aws_default_allocator(), &new_path); - aws_byte_buf_clean_up(&new_path); + return long_path; + } - return long_path; + return aws_string_new_from_string(aws_default_allocator(), path); } int aws_directory_create(const struct aws_string *dir_path) { @@ -50,11 +56,15 @@ int aws_directory_create(const struct aws_string *dir_path) { struct aws_string *long_dir_path = s_to_long_path(w_dir_path); aws_string_destroy(w_dir_path); - int create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); + BOOL create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); aws_string_destroy(long_dir_path); - if (create_dir_res == ERROR_PATH_NOT_FOUND) { - return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + int error = GetLastError(); + if (!create_dir_res) { + if (error == ERROR_PATH_NOT_FOUND) { + return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + } + return aws_raise_error(AWS_ERROR_UNKNOWN); } return AWS_OP_SUCCESS; @@ -189,15 +199,23 @@ int aws_directory_traverse( bool recursive, aws_on_directory_entry *on_entry, void *user_data) { - struct aws_string *w_path = aws_string_convert_to_wchar_str(allocator, path); - struct aws_string *long_path = s_to_long_path(w_path); - aws_string_destroy(w_path); + struct aws_string *w_path_wchar = aws_string_convert_to_wchar_str(allocator, path); + struct aws_string *long_path_wchar = s_to_long_path(w_path_wchar); + aws_string_destroy(w_path_wchar); + + /* windows doesn't fail in FindFirstFile if it's not a directory. Do the check here. We don't call the perfectly + good function for this check because the string is already converted to utf-16 and it's trivial to reuse it. */ + DWORD attributes = GetFileAttributesW(aws_string_wchar_c_str(long_path_wchar)); + if (!(attributes & FILE_ATTRIBUTE_DIRECTORY)) { + aws_string_destroy(long_path_wchar); + return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); + } WIN32_FIND_DATAW ffd; - HANDLE find_handle = FindFirstFileW(aws_string_wchar_c_str(long_path), &ffd); + HANDLE find_handle = FindFirstFileW(aws_string_wchar_c_str(long_path_wchar), &ffd); if (find_handle == INVALID_HANDLE_VALUE) { - aws_string_destroy(long_path); + aws_string_destroy(long_path_wchar); int error = GetLastError(); @@ -209,68 +227,90 @@ int aws_directory_traverse( } FindClose(find_handle); + /* create search path string */ - struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(long_path), long_path->len); - struct aws_byte_buf search_buf; - aws_byte_buf_init_copy_from_cursor(&search_buf, allocator, path_cur); - aws_string_destroy(long_path); + struct aws_byte_cursor path_wchar_cur = aws_byte_cursor_from_array(aws_string_bytes(long_path_wchar), long_path_wchar->len); + struct aws_byte_buf search_wchar_buf; + aws_byte_buf_init_copy_from_cursor(&search_wchar_buf, allocator, path_wchar_cur); + wchar_t search_wchar_pattern[] = L"\\*"; - struct aws_byte_cursor search_char = + struct aws_byte_cursor search_char_wchar = aws_byte_cursor_from_array((uint8_t *)search_wchar_pattern, sizeof(search_wchar_pattern)); - aws_byte_buf_append_dynamic(&search_buf, &search_char); + aws_byte_buf_append_dynamic(&search_wchar_buf, &search_char_wchar); /* it's already converted to wide string */ - struct aws_string *search_string = aws_string_new_from_buf(allocator, &search_buf); - search_buf.len -= sizeof(search_wchar_pattern); + struct aws_string *search_wchar_string = aws_string_new_from_buf(allocator, &search_wchar_buf); - find_handle = FindFirstFileW(aws_string_wchar_c_str(search_string), &ffd); - aws_string_destroy(search_string); - aws_byte_buf_clean_up(&search_buf); + find_handle = FindFirstFileW(aws_string_wchar_c_str(search_wchar_string), &ffd); + aws_string_destroy(search_wchar_string); + aws_byte_buf_clean_up(&search_wchar_buf); int ret_val = AWS_OP_SUCCESS; - while (ret_val == AWS_OP_SUCCESS) { - struct aws_string *name_component_str = aws_string_convert_from_wchar_c_str(allocator, ffd.cFileName); - struct aws_byte_cursor name_component = aws_byte_cursor_from_string(name_component_str); - - if (aws_byte_cursor_eq_c_str(&name_component, "..") || aws_byte_cursor_eq_c_str(&name_component, ".")) { - aws_string_destroy(name_component_str); + /* iterate each entry in the directory. Do a bunch of utf-16 conversions. Figure out the paths etc.... + invoke the visitor, and continue recursing if the flag was set. */ + do { + struct aws_string *name_component_multi_char_str = aws_string_convert_from_wchar_c_str(allocator, ffd.cFileName); + struct aws_byte_cursor name_component_multi_char = aws_byte_cursor_from_string(name_component_multi_char_str); + + /* disgard . and .. */ + char *ascend_mark = ".."; + char *cd_mark = "."; + struct aws_byte_cursor ascend_mark_cur = + aws_byte_cursor_from_c_str(ascend_mark); + struct aws_byte_cursor cd_mark_cur = + aws_byte_cursor_from_c_str(cd_mark); + if (aws_byte_cursor_eq(&name_component_multi_char, &ascend_mark_cur) || aws_byte_cursor_eq(&name_component_multi_char, &cd_mark_cur)) { + aws_string_destroy(name_component_multi_char_str); continue; } - struct aws_byte_buf relative_path; - aws_byte_buf_init_copy_from_cursor(&relative_path, allocator, path_cur); - aws_byte_buf_append_byte_dynamic(&relative_path, AWS_PATH_DELIM); - aws_byte_buf_append_dynamic(&relative_path, &name_component); - aws_byte_buf_append_byte_dynamic(&relative_path, 0); - relative_path.len -= 1; + /* get the relative path as utf-16, so we can talk to windows. */ + struct aws_byte_buf relative_path_wchar; + aws_byte_buf_init_copy_from_cursor(&relative_path_wchar, allocator, path_wchar_cur); + + wchar_t unicode_delim[] = L"\\"; + struct aws_byte_cursor delimiter_cur = aws_byte_cursor_from_array((uint8_t *)unicode_delim, sizeof(unicode_delim) - 2); + aws_byte_buf_append_dynamic(&relative_path_wchar, &delimiter_cur); + struct aws_byte_cursor name_str = aws_byte_cursor_from_array(ffd.cFileName, wcsnlen(ffd.cFileName, sizeof(ffd.cFileName)) * sizeof(wchar_t)); + aws_byte_buf_append_dynamic(&relative_path_wchar, &name_str); + aws_byte_buf_append_byte_dynamic(&relative_path_wchar, 0); + aws_byte_buf_append_byte_dynamic(&relative_path_wchar, 0); + + relative_path_wchar.len -= 2; - struct aws_byte_cursor relative_path_cur = aws_byte_cursor_from_buf(&relative_path); - struct aws_string *wchar_short_name = - aws_string_convert_to_wchar_from_byte_cursor(allocator, &relative_path_cur); - DWORD path_res = GetFullPathNameW(aws_string_wchar_c_str(wchar_short_name), 0, NULL, NULL); + /* now get the absolute path from the relative path we just computed. */ + DWORD path_res = GetFullPathNameW((wchar_t *)relative_path_wchar.buffer, 0, NULL, NULL); + wchar_t *test = (wchar_t *)relative_path_wchar.buffer; AWS_FATAL_ASSERT(path_res > 0); - struct aws_byte_buf full_path_buf; - aws_byte_buf_init(&full_path_buf, allocator, (size_t)path_res * sizeof(wchar_t) + 2); + struct aws_byte_buf full_path_wchar_buf; + aws_byte_buf_init(&full_path_wchar_buf, allocator, (size_t)path_res * sizeof(wchar_t) + 2); - full_path_buf.len = path_res + 1; + full_path_wchar_buf.len = full_path_wchar_buf.capacity - 2; path_res = GetFullPathNameW( - aws_string_wchar_c_str(wchar_short_name), - (DWORD)wchar_short_name->len, - (wchar_t *)full_path_buf.buffer, + (wchar_t *)relative_path_wchar.buffer, + (DWORD)path_res + 1, + (wchar_t *)full_path_wchar_buf.buffer, NULL); AWS_FATAL_ASSERT(path_res > 0); - aws_string_destroy(wchar_short_name); + + aws_byte_buf_append_byte_dynamic(&full_path_wchar_buf, 0); + aws_byte_buf_append_byte_dynamic(&full_path_wchar_buf, 0); - struct aws_string *full_path_name_converted = - aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)full_path_buf.buffer); - aws_byte_buf_clean_up(&full_path_buf); + /* now we have the data, convert the utf-16 strings we used to communicate with windows back to + utf-8 for the user to actually consume. */ + struct aws_string *full_path_name_multi_char = + aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)full_path_wchar_buf.buffer); + aws_byte_buf_clean_up(&full_path_wchar_buf); + + (void)test; + struct aws_string *relative_path_multi_char = aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)relative_path_wchar.buffer); struct aws_directory_entry entry; AWS_ZERO_STRUCT(entry); - entry.relative_path = relative_path_cur; - entry.path = aws_byte_cursor_from_string(full_path_name_converted); + entry.relative_path = aws_byte_cursor_from_string(relative_path_multi_char); + entry.path = aws_byte_cursor_from_string(full_path_name_multi_char); LARGE_INTEGER file_size; file_size.HighPart = ffd.nFileSizeHigh; @@ -284,9 +324,7 @@ int aws_directory_traverse( } if (recursive && entry.file_type & AWS_FILE_TYPE_DIRECTORY) { - struct aws_string *rel_path_str = aws_string_new_from_cursor(allocator, &entry.relative_path); - ret_val = aws_directory_traverse(allocator, rel_path_str, recursive, on_entry, user_data); - aws_string_destroy(rel_path_str); + ret_val = aws_directory_traverse(allocator, relative_path_multi_char, recursive, on_entry, user_data); } /* post order traversal, if a node below us ended the traversal, don't call the visitor again. */ @@ -304,16 +342,17 @@ int aws_directory_traverse( } cleanup: - aws_string_destroy(full_path_name_converted); - aws_byte_buf_clean_up(&relative_path); + aws_string_destroy(relative_path_multi_char); + aws_string_destroy(full_path_name_multi_char); + aws_byte_buf_clean_up(&relative_path_wchar); + aws_string_destroy(name_component_multi_char_str); - if (!FindNextFileW(find_handle, &ffd)) { - break; - } - } + } while (ret_val == AWS_OP_SUCCESS && FindNextFileW(find_handle, &ffd)); + aws_string_destroy(long_path_wchar); if (find_handle != INVALID_HANDLE_VALUE) { FindClose(find_handle); } + return ret_val; } From 101ffe6c39cf14088c9be8520829424f7df2a9d0 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 12:07:49 -0700 Subject: [PATCH 16/34] Just formatting. --- source/windows/file.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/source/windows/file.c b/source/windows/file.c index 054e7bd84..b2554f97c 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -30,9 +30,9 @@ FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string struct aws_string *s_to_long_path(const struct aws_string *path) { wchar_t prefix[] = L"\\\\?\\"; - size_t prefix_size = sizeof(prefix); + size_t prefix_size = sizeof(prefix); if (path->len > MAX_PATH - prefix_size) { - + struct aws_byte_buf new_path; aws_byte_buf_init(&new_path, aws_default_allocator(), sizeof(prefix) + path->len + 2); @@ -59,7 +59,7 @@ int aws_directory_create(const struct aws_string *dir_path) { BOOL create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); aws_string_destroy(long_dir_path); - int error = GetLastError(); + int error = GetLastError(); if (!create_dir_res) { if (error == ERROR_PATH_NOT_FOUND) { return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); @@ -229,7 +229,8 @@ int aws_directory_traverse( FindClose(find_handle); /* create search path string */ - struct aws_byte_cursor path_wchar_cur = aws_byte_cursor_from_array(aws_string_bytes(long_path_wchar), long_path_wchar->len); + struct aws_byte_cursor path_wchar_cur = + aws_byte_cursor_from_array(aws_string_bytes(long_path_wchar), long_path_wchar->len); struct aws_byte_buf search_wchar_buf; aws_byte_buf_init_copy_from_cursor(&search_wchar_buf, allocator, path_wchar_cur); @@ -250,17 +251,17 @@ int aws_directory_traverse( /* iterate each entry in the directory. Do a bunch of utf-16 conversions. Figure out the paths etc.... invoke the visitor, and continue recursing if the flag was set. */ do { - struct aws_string *name_component_multi_char_str = aws_string_convert_from_wchar_c_str(allocator, ffd.cFileName); + struct aws_string *name_component_multi_char_str = + aws_string_convert_from_wchar_c_str(allocator, ffd.cFileName); struct aws_byte_cursor name_component_multi_char = aws_byte_cursor_from_string(name_component_multi_char_str); /* disgard . and .. */ char *ascend_mark = ".."; char *cd_mark = "."; - struct aws_byte_cursor ascend_mark_cur = - aws_byte_cursor_from_c_str(ascend_mark); - struct aws_byte_cursor cd_mark_cur = - aws_byte_cursor_from_c_str(cd_mark); - if (aws_byte_cursor_eq(&name_component_multi_char, &ascend_mark_cur) || aws_byte_cursor_eq(&name_component_multi_char, &cd_mark_cur)) { + struct aws_byte_cursor ascend_mark_cur = aws_byte_cursor_from_c_str(ascend_mark); + struct aws_byte_cursor cd_mark_cur = aws_byte_cursor_from_c_str(cd_mark); + if (aws_byte_cursor_eq(&name_component_multi_char, &ascend_mark_cur) || + aws_byte_cursor_eq(&name_component_multi_char, &cd_mark_cur)) { aws_string_destroy(name_component_multi_char_str); continue; } @@ -270,9 +271,11 @@ int aws_directory_traverse( aws_byte_buf_init_copy_from_cursor(&relative_path_wchar, allocator, path_wchar_cur); wchar_t unicode_delim[] = L"\\"; - struct aws_byte_cursor delimiter_cur = aws_byte_cursor_from_array((uint8_t *)unicode_delim, sizeof(unicode_delim) - 2); - aws_byte_buf_append_dynamic(&relative_path_wchar, &delimiter_cur); - struct aws_byte_cursor name_str = aws_byte_cursor_from_array(ffd.cFileName, wcsnlen(ffd.cFileName, sizeof(ffd.cFileName)) * sizeof(wchar_t)); + struct aws_byte_cursor delimiter_cur = + aws_byte_cursor_from_array((uint8_t *)unicode_delim, sizeof(unicode_delim) - 2); + aws_byte_buf_append_dynamic(&relative_path_wchar, &delimiter_cur); + struct aws_byte_cursor name_str = + aws_byte_cursor_from_array(ffd.cFileName, wcsnlen(ffd.cFileName, sizeof(ffd.cFileName)) * sizeof(wchar_t)); aws_byte_buf_append_dynamic(&relative_path_wchar, &name_str); aws_byte_buf_append_byte_dynamic(&relative_path_wchar, 0); aws_byte_buf_append_byte_dynamic(&relative_path_wchar, 0); @@ -289,12 +292,9 @@ int aws_directory_traverse( full_path_wchar_buf.len = full_path_wchar_buf.capacity - 2; path_res = GetFullPathNameW( - (wchar_t *)relative_path_wchar.buffer, - (DWORD)path_res + 1, - (wchar_t *)full_path_wchar_buf.buffer, - NULL); + (wchar_t *)relative_path_wchar.buffer, (DWORD)path_res + 1, (wchar_t *)full_path_wchar_buf.buffer, NULL); AWS_FATAL_ASSERT(path_res > 0); - + aws_byte_buf_append_byte_dynamic(&full_path_wchar_buf, 0); aws_byte_buf_append_byte_dynamic(&full_path_wchar_buf, 0); @@ -305,7 +305,8 @@ int aws_directory_traverse( aws_byte_buf_clean_up(&full_path_wchar_buf); (void)test; - struct aws_string *relative_path_multi_char = aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)relative_path_wchar.buffer); + struct aws_string *relative_path_multi_char = + aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)relative_path_wchar.buffer); struct aws_directory_entry entry; AWS_ZERO_STRUCT(entry); From 72f48a8db01bd862cb9b3abe6d8dc7b082f673a4 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 12:10:25 -0700 Subject: [PATCH 17/34] Remove unneeded headers. --- include/aws/common/file.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 84ae6149d..2c7d60b19 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -6,8 +6,6 @@ */ #include #include -#include -#include #ifdef _WIN32 # define AWS_PATH_DELIM '\\' From 7cf1b2e56ef4eaddf68abd28b0f90594f08d2554 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 12:33:06 -0700 Subject: [PATCH 18/34] Better error handling, deleted some old test tracing markers for debugging. --- source/windows/file.c | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/source/windows/file.c b/source/windows/file.c index b2554f97c..c82735501 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -27,14 +27,14 @@ FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string return file; } -struct aws_string *s_to_long_path(const struct aws_string *path) { +struct aws_string *s_to_long_path(struct aws_allocator *allocator, const struct aws_string *path) { wchar_t prefix[] = L"\\\\?\\"; size_t prefix_size = sizeof(prefix); if (path->len > MAX_PATH - prefix_size) { struct aws_byte_buf new_path; - aws_byte_buf_init(&new_path, aws_default_allocator(), sizeof(prefix) + path->len + 2); + aws_byte_buf_init(&new_path, allocator, sizeof(prefix) + path->len + 2); struct aws_byte_cursor prefix_cur = aws_byte_cursor_from_array((uint8_t *)prefix, sizeof(prefix) - 2); aws_byte_buf_append_dynamic(&new_path, &prefix_cur); @@ -42,18 +42,18 @@ struct aws_string *s_to_long_path(const struct aws_string *path) { struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(path), path->len); aws_byte_buf_append_dynamic(&new_path, &path_cur); - struct aws_string *long_path = aws_string_new_from_buf(aws_default_allocator(), &new_path); + struct aws_string *long_path = aws_string_new_from_buf(allocator, &new_path); aws_byte_buf_clean_up(&new_path); return long_path; } - return aws_string_new_from_string(aws_default_allocator(), path); + return aws_string_new_from_string(allocator, path); } int aws_directory_create(const struct aws_string *dir_path) { struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(w_dir_path); + struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator() , w_dir_path); aws_string_destroy(w_dir_path); BOOL create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); @@ -61,9 +61,18 @@ int aws_directory_create(const struct aws_string *dir_path) { int error = GetLastError(); if (!create_dir_res) { + if (error == ERROR_ALREADY_EXISTS) { + return AWS_OP_SUCCESS; + } + if (error == ERROR_PATH_NOT_FOUND) { return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); } + + if (error == ERROR_ACCESS_DENIED) { + return aws_raise_error(AWS_ERROR_NO_PERMISSION); + } + return aws_raise_error(AWS_ERROR_UNKNOWN); } @@ -72,7 +81,7 @@ int aws_directory_create(const struct aws_string *dir_path) { bool aws_directory_exists(const struct aws_string *dir_path) { struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(w_dir_path); + struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); aws_string_destroy(w_dir_path); DWORD attributes = GetFileAttributesW(aws_string_wchar_c_str(long_dir_path)); @@ -122,7 +131,7 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { } struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(w_dir_path); + struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); aws_string_destroy(w_dir_path); BOOL remove_dir_res = RemoveDirectoryW(aws_string_wchar_c_str(long_dir_path)); @@ -134,6 +143,10 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { return aws_raise_error(AWS_ERROR_DIRECTORY_NOT_EMPTY); } + if (error == ERROR_ACCESS_DENIED) { + return aws_raise_error(AWS_ERROR_NO_PERMISSION); + } + return aws_raise_error(AWS_ERROR_UNKNOWN); } @@ -142,7 +155,7 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { int aws_file_delete(const struct aws_string *file_path) { struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); - struct aws_string *long_file_path = s_to_long_path(w_file_path); + struct aws_string *long_file_path = s_to_long_path(aws_default_allocator(), w_file_path); aws_string_destroy(w_file_path); BOOL remove_file_res = DeleteFileW(aws_string_wchar_c_str(long_file_path)); @@ -166,11 +179,11 @@ int aws_file_delete(const struct aws_string *file_path) { int aws_directory_or_file_move(const struct aws_string *from, const struct aws_string *to) { struct aws_string *w_from_path = aws_string_convert_to_wchar_str(aws_default_allocator(), from); - struct aws_string *long_from_path = s_to_long_path(w_from_path); + struct aws_string *long_from_path = s_to_long_path(aws_default_allocator(), w_from_path); aws_string_destroy(w_from_path); struct aws_string *w_to_path = aws_string_convert_to_wchar_str(aws_default_allocator(), to); - struct aws_string *long_to_path = s_to_long_path(w_to_path); + struct aws_string *long_to_path = s_to_long_path(aws_default_allocator(), w_to_path); aws_string_destroy(w_to_path); BOOL move_res = MoveFileW(aws_string_wchar_c_str(long_from_path), aws_string_wchar_c_str(long_to_path)); @@ -200,7 +213,7 @@ int aws_directory_traverse( aws_on_directory_entry *on_entry, void *user_data) { struct aws_string *w_path_wchar = aws_string_convert_to_wchar_str(allocator, path); - struct aws_string *long_path_wchar = s_to_long_path(w_path_wchar); + struct aws_string *long_path_wchar = s_to_long_path(allocator, w_path_wchar); aws_string_destroy(w_path_wchar); /* windows doesn't fail in FindFirstFile if it's not a directory. Do the check here. We don't call the perfectly @@ -284,7 +297,6 @@ int aws_directory_traverse( /* now get the absolute path from the relative path we just computed. */ DWORD path_res = GetFullPathNameW((wchar_t *)relative_path_wchar.buffer, 0, NULL, NULL); - wchar_t *test = (wchar_t *)relative_path_wchar.buffer; AWS_FATAL_ASSERT(path_res > 0); struct aws_byte_buf full_path_wchar_buf; @@ -304,10 +316,10 @@ int aws_directory_traverse( aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)full_path_wchar_buf.buffer); aws_byte_buf_clean_up(&full_path_wchar_buf); - (void)test; struct aws_string *relative_path_multi_char = aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)relative_path_wchar.buffer); + struct aws_directory_entry entry; AWS_ZERO_STRUCT(entry); entry.relative_path = aws_byte_cursor_from_string(relative_path_multi_char); From 19f1b3a5d48a910486c772d871c0e6e9c56fce5c Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 12:35:08 -0700 Subject: [PATCH 19/34] Formatting --- source/windows/file.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/windows/file.c b/source/windows/file.c index c82735501..7d9b05547 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -53,7 +53,7 @@ struct aws_string *s_to_long_path(struct aws_allocator *allocator, const struct int aws_directory_create(const struct aws_string *dir_path) { struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator() , w_dir_path); + struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); aws_string_destroy(w_dir_path); BOOL create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); @@ -146,7 +146,7 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { if (error == ERROR_ACCESS_DENIED) { return aws_raise_error(AWS_ERROR_NO_PERMISSION); } - + return aws_raise_error(AWS_ERROR_UNKNOWN); } @@ -319,7 +319,6 @@ int aws_directory_traverse( struct aws_string *relative_path_multi_char = aws_string_convert_from_wchar_c_str(allocator, (wchar_t *)relative_path_wchar.buffer); - struct aws_directory_entry entry; AWS_ZERO_STRUCT(entry); entry.relative_path = aws_byte_cursor_from_string(relative_path_multi_char); From 23eb6c1f4a4ec3b34a747cc821700b03b5295bd3 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 16:36:54 -0700 Subject: [PATCH 20/34] hopefully fix the cbmc proofs. --- source/allocator.c | 31 ++++++++++--------------------- source/string.c | 1 + 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index 9ce7deea2..939057094 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -118,9 +118,8 @@ void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { AWS_FATAL_PRECONDITION(size != 0); void *mem = allocator->mem_acquire(allocator, size); - if (!mem) { - AWS_FATAL_ASSERT(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); - } + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + return mem; } @@ -134,24 +133,19 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { * https://wiki.sei.cmu.edu/confluence/display/c/MEM07-C.+Ensure+that+the+arguments+to+calloc%28%29%2C+when+multiplied%2C+do+not+wrap */ size_t required_bytes = 0; - if (aws_mul_size_checked(num, size, &required_bytes)) { - AWS_FATAL_ASSERT("calloc computed size > SIZE_MAX"); - } + AWS_FATAL_POSTCONDITION(!aws_mul_size_checked(num, size, &required_bytes) && "calloc computed size > SIZE_MAX"); /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { void *mem = allocator->mem_calloc(allocator, num, size); - if (!mem) { - AWS_FATAL_ASSERT(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); - } + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } /* Otherwise, emulate calloc */ void *mem = allocator->mem_acquire(allocator, required_bytes); - if (!mem) { - AWS_FATAL_ASSERT(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); - } + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + memset(mem, 0, required_bytes); AWS_POSTCONDITION(mem != NULL); return mem; @@ -184,9 +178,7 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { if (total_size > 0) { allocation = aws_mem_acquire(allocator, total_size); - if (!allocation) { - AWS_FATAL_ASSERT(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); - } + AWS_FATAL_POSTCONDITION(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); uint8_t *current_ptr = allocation; @@ -231,9 +223,8 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, if (allocator->mem_realloc) { void *newptr = allocator->mem_realloc(allocator, *ptr, oldsize, newsize); - if (!newptr) { - AWS_FATAL_ASSERT(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); - } + AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + *ptr = newptr; return AWS_OP_SUCCESS; } @@ -244,9 +235,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, } void *newptr = allocator->mem_acquire(allocator, newsize); - if (!newptr) { - AWS_FATAL_ASSERT(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); - } + AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); memcpy(newptr, *ptr, oldsize); memset((uint8_t *)newptr + oldsize, 0, newsize - oldsize); diff --git a/source/string.c b/source/string.c index cfee6886b..20b4035d4 100644 --- a/source/string.c +++ b/source/string.c @@ -144,6 +144,7 @@ struct aws_string *aws_string_new_from_array(struct aws_allocator *allocator, co if (len > 0) { memcpy((void *)str->bytes, bytes, len); } + /* in case this is a utf-16 string in the array, allow that here. */ *(uint8_t *)&str->bytes[len] = 0; *(uint8_t *)&str->bytes[len + 1] = 0; AWS_RETURN_WITH_POSTCONDITION(str, aws_string_is_valid(str)); From b79a295bb5b0cee09943d76809f0a26cced1fd1c Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 17:12:40 -0700 Subject: [PATCH 21/34] Let us pray. --- source/allocator.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index 939057094..c217775b4 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -51,9 +51,13 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { const size_t alignment = sizeof(void *) * (size > PAGE_SIZE ? 8 : 2); #if !defined(_WIN32) void *result = NULL; - return (posix_memalign(&result, alignment, size)) ? NULL : result; + posix_memalign(&result, alignment, size); + AWS_FATAL_POSTCONDITION(result && "posix_memalign failed to allocate memory"); + return result; #else - return _aligned_malloc(size, alignment); + void *mem = _aligned_malloc(size, alignment); + AWS_FATAL_POSTCONDITION(mem && "_aligned_malloc failed to allocate memory"); + return mem; #endif } @@ -69,34 +73,33 @@ static void s_default_free(struct aws_allocator *allocator, void *ptr) { static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { (void)allocator; (void)oldsize; -#if !defined(_WIN32) - if (newsize == 0 || !ptr) { - free(ptr); - return NULL; - } + AWS_FATAL_PRECONDITION(ptr) + AWS_FATAL_PRECONDITION(newsize) +#if !defined(_WIN32) if (newsize <= oldsize) { return ptr; } /* newsize is > oldsize, need more memory */ void *new_mem = s_default_malloc(allocator, newsize); - if (new_mem) { - memcpy(new_mem, ptr, oldsize); - s_default_free(allocator, ptr); - } + AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in s_default_malloc") + memcpy(new_mem, ptr, oldsize); + s_default_free(allocator, ptr); + return new_mem; #else const size_t alignment = sizeof(void *) * (newsize > PAGE_SIZE ? 8 : 2); - return _aligned_realloc(ptr, newsize, alignment); + void *new_mem = _aligned_realloc(ptr, newsize, alignment); + AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in _aligned_realloc") + return new_mem; #endif } static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) { void *mem = s_default_malloc(allocator, num * size); - if (mem) { - memset(mem, 0, num * size); - } + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in s_default_malloc") + memset(mem, 0, num * size); return mem; } From 8ce6b55e61be759db9f753b4e0e0623bd5dc569a Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 17:16:34 -0700 Subject: [PATCH 22/34] posix_memalign return nonsense. --- source/allocator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index c217775b4..719c54d78 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -51,8 +51,8 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { const size_t alignment = sizeof(void *) * (size > PAGE_SIZE ? 8 : 2); #if !defined(_WIN32) void *result = NULL; - posix_memalign(&result, alignment, size); - AWS_FATAL_POSTCONDITION(result && "posix_memalign failed to allocate memory"); + int ret_val = posix_memalign(&result, alignment, size); + AWS_FATAL_POSTCONDITION(!ret_val && result && "posix_memalign failed to allocate memory"); return result; #else void *mem = _aligned_malloc(size, alignment); From 9b7bd0bc2200a629d041ee2cf25175701ad8e73c Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 17:22:51 -0700 Subject: [PATCH 23/34] God I'm sick of CBMC. --- source/allocator.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index 719c54d78..469699db2 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -73,8 +73,8 @@ static void s_default_free(struct aws_allocator *allocator, void *ptr) { static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { (void)allocator; (void)oldsize; - AWS_FATAL_PRECONDITION(ptr) - AWS_FATAL_PRECONDITION(newsize) + AWS_FATAL_PRECONDITION(ptr); + AWS_FATAL_PRECONDITION(newsize); #if !defined(_WIN32) if (newsize <= oldsize) { @@ -83,7 +83,7 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ /* newsize is > oldsize, need more memory */ void *new_mem = s_default_malloc(allocator, newsize); - AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in s_default_malloc") + AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in s_default_malloc"); memcpy(new_mem, ptr, oldsize); s_default_free(allocator, ptr); @@ -91,14 +91,14 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ #else const size_t alignment = sizeof(void *) * (newsize > PAGE_SIZE ? 8 : 2); void *new_mem = _aligned_realloc(ptr, newsize, alignment); - AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in _aligned_realloc") + AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in _aligned_realloc"); return new_mem; #endif } static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) { void *mem = s_default_malloc(allocator, num * size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in s_default_malloc") + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in s_default_malloc"); memset(mem, 0, num * size); return mem; } From 53a01076b965b83001adf9e4baf9a632e2988a78 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 20:11:17 -0700 Subject: [PATCH 24/34] Try using assumptions instead? --- source/allocator.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index 469699db2..1d04ac5bd 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -52,11 +52,11 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { #if !defined(_WIN32) void *result = NULL; int ret_val = posix_memalign(&result, alignment, size); - AWS_FATAL_POSTCONDITION(!ret_val && result && "posix_memalign failed to allocate memory"); + AWS_ASSUME(!ret_val && result && "posix_memalign failed to allocate memory"); return result; #else void *mem = _aligned_malloc(size, alignment); - AWS_FATAL_POSTCONDITION(mem && "_aligned_malloc failed to allocate memory"); + AWS_ASSUME(mem && "_aligned_malloc failed to allocate memory"); return mem; #endif } @@ -83,7 +83,7 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ /* newsize is > oldsize, need more memory */ void *new_mem = s_default_malloc(allocator, newsize); - AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in s_default_malloc"); + AWS_ASSUME(new_mem && "Unhandled OOM encountered in s_default_malloc"); memcpy(new_mem, ptr, oldsize); s_default_free(allocator, ptr); @@ -91,14 +91,14 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ #else const size_t alignment = sizeof(void *) * (newsize > PAGE_SIZE ? 8 : 2); void *new_mem = _aligned_realloc(ptr, newsize, alignment); - AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in _aligned_realloc"); + AWS_ASSUME(new_mem && "Unhandled OOM encountered in _aligned_realloc"); return new_mem; #endif } static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) { void *mem = s_default_malloc(allocator, num * size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in s_default_malloc"); + AWS_ASSUME(mem && "Unhandled OOM encountered in s_default_malloc"); memset(mem, 0, num * size); return mem; } @@ -121,7 +121,7 @@ void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { AWS_FATAL_PRECONDITION(size != 0); void *mem = allocator->mem_acquire(allocator, size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_ASSUME(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } @@ -136,21 +136,20 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { * https://wiki.sei.cmu.edu/confluence/display/c/MEM07-C.+Ensure+that+the+arguments+to+calloc%28%29%2C+when+multiplied%2C+do+not+wrap */ size_t required_bytes = 0; - AWS_FATAL_POSTCONDITION(!aws_mul_size_checked(num, size, &required_bytes) && "calloc computed size > SIZE_MAX"); + AWS_ASSUME(!aws_mul_size_checked(num, size, &required_bytes) && "calloc computed size > SIZE_MAX"); /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { void *mem = allocator->mem_calloc(allocator, num, size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_ASSUME(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } /* Otherwise, emulate calloc */ void *mem = allocator->mem_acquire(allocator, required_bytes); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_ASSUME(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); memset(mem, 0, required_bytes); - AWS_POSTCONDITION(mem != NULL); return mem; } @@ -181,7 +180,7 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { if (total_size > 0) { allocation = aws_mem_acquire(allocator, total_size); - AWS_FATAL_POSTCONDITION(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_ASSUME(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); uint8_t *current_ptr = allocation; @@ -226,7 +225,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, if (allocator->mem_realloc) { void *newptr = allocator->mem_realloc(allocator, *ptr, oldsize, newsize); - AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_ASSUME(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); *ptr = newptr; return AWS_OP_SUCCESS; @@ -238,7 +237,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, } void *newptr = allocator->mem_acquire(allocator, newsize); - AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_ASSUME(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); memcpy(newptr, *ptr, oldsize); memset((uint8_t *)newptr + oldsize, 0, newsize - oldsize); From dc39a5bc018de1731401b1f2fd906424b05f5364 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 26 Aug 2021 21:07:31 -0700 Subject: [PATCH 25/34] Revert assumptions. --- source/allocator.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index 1d04ac5bd..469699db2 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -52,11 +52,11 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { #if !defined(_WIN32) void *result = NULL; int ret_val = posix_memalign(&result, alignment, size); - AWS_ASSUME(!ret_val && result && "posix_memalign failed to allocate memory"); + AWS_FATAL_POSTCONDITION(!ret_val && result && "posix_memalign failed to allocate memory"); return result; #else void *mem = _aligned_malloc(size, alignment); - AWS_ASSUME(mem && "_aligned_malloc failed to allocate memory"); + AWS_FATAL_POSTCONDITION(mem && "_aligned_malloc failed to allocate memory"); return mem; #endif } @@ -83,7 +83,7 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ /* newsize is > oldsize, need more memory */ void *new_mem = s_default_malloc(allocator, newsize); - AWS_ASSUME(new_mem && "Unhandled OOM encountered in s_default_malloc"); + AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in s_default_malloc"); memcpy(new_mem, ptr, oldsize); s_default_free(allocator, ptr); @@ -91,14 +91,14 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ #else const size_t alignment = sizeof(void *) * (newsize > PAGE_SIZE ? 8 : 2); void *new_mem = _aligned_realloc(ptr, newsize, alignment); - AWS_ASSUME(new_mem && "Unhandled OOM encountered in _aligned_realloc"); + AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in _aligned_realloc"); return new_mem; #endif } static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) { void *mem = s_default_malloc(allocator, num * size); - AWS_ASSUME(mem && "Unhandled OOM encountered in s_default_malloc"); + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in s_default_malloc"); memset(mem, 0, num * size); return mem; } @@ -121,7 +121,7 @@ void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { AWS_FATAL_PRECONDITION(size != 0); void *mem = allocator->mem_acquire(allocator, size); - AWS_ASSUME(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } @@ -136,20 +136,21 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { * https://wiki.sei.cmu.edu/confluence/display/c/MEM07-C.+Ensure+that+the+arguments+to+calloc%28%29%2C+when+multiplied%2C+do+not+wrap */ size_t required_bytes = 0; - AWS_ASSUME(!aws_mul_size_checked(num, size, &required_bytes) && "calloc computed size > SIZE_MAX"); + AWS_FATAL_POSTCONDITION(!aws_mul_size_checked(num, size, &required_bytes) && "calloc computed size > SIZE_MAX"); /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { void *mem = allocator->mem_calloc(allocator, num, size); - AWS_ASSUME(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } /* Otherwise, emulate calloc */ void *mem = allocator->mem_acquire(allocator, required_bytes); - AWS_ASSUME(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); memset(mem, 0, required_bytes); + AWS_POSTCONDITION(mem != NULL); return mem; } @@ -180,7 +181,7 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { if (total_size > 0) { allocation = aws_mem_acquire(allocator, total_size); - AWS_ASSUME(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_FATAL_POSTCONDITION(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); uint8_t *current_ptr = allocation; @@ -225,7 +226,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, if (allocator->mem_realloc) { void *newptr = allocator->mem_realloc(allocator, *ptr, oldsize, newsize); - AWS_ASSUME(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); *ptr = newptr; return AWS_OP_SUCCESS; @@ -237,7 +238,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, } void *newptr = allocator->mem_acquire(allocator, newsize); - AWS_ASSUME(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); memcpy(newptr, *ptr, oldsize); memset((uint8_t *)newptr + oldsize, 0, newsize - oldsize); From d5983d604ecf60d0ca1c98e12f927477c9f3e860 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 30 Aug 2021 11:46:50 -0700 Subject: [PATCH 26/34] Added oom panic macro to aid in formal verification of allocators. --- include/aws/common/assert.h | 18 ++++++++++++++++++ source/allocator.c | 22 +++++++++++----------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/include/aws/common/assert.h b/include/aws/common/assert.h index 420e33ccf..caf0811d6 100644 --- a/include/aws/common/assert.h +++ b/include/aws/common/assert.h @@ -18,6 +18,24 @@ void aws_fatal_assert(const char *cond_str, const char *file, int line) AWS_ATTR AWS_EXTERN_C_END +#if defined(CBMC) +# define AWS_PANIC_OOM(mem, msg) \ + do { \ + if (!(mem)) { \ + fprintf(stderr, "%s: %s, line %d", msg, __FILE__, __LINE__); \ + exit(-1); \ + } \ + } while (0) +#else +# define AWS_PANIC_OOM(mem, msg) \ + do { \ + if (!(mem)) { \ + fprintf(stderr, "%s: %s, line %d", msg, __FILE__, __LINE__); \ + abort(); \ + } \ + } while (0) +#endif /* defined(CBMC) */ + #if defined(CBMC) # define AWS_ASSUME(cond) __CPROVER_assume(cond) #elif defined(_MSC_VER) diff --git a/source/allocator.c b/source/allocator.c index 469699db2..69cdc7b2a 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -51,8 +51,8 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { const size_t alignment = sizeof(void *) * (size > PAGE_SIZE ? 8 : 2); #if !defined(_WIN32) void *result = NULL; - int ret_val = posix_memalign(&result, alignment, size); - AWS_FATAL_POSTCONDITION(!ret_val && result && "posix_memalign failed to allocate memory"); + posix_memalign(&result, alignment, size); + AWS_PANIC_OOM(result, "posix_memalign failed to allocate memory"); return result; #else void *mem = _aligned_malloc(size, alignment); @@ -83,7 +83,7 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ /* newsize is > oldsize, need more memory */ void *new_mem = s_default_malloc(allocator, newsize); - AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in s_default_malloc"); + AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_default_malloc"); memcpy(new_mem, ptr, oldsize); s_default_free(allocator, ptr); @@ -91,14 +91,14 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ #else const size_t alignment = sizeof(void *) * (newsize > PAGE_SIZE ? 8 : 2); void *new_mem = _aligned_realloc(ptr, newsize, alignment); - AWS_FATAL_POSTCONDITION(new_mem && "Unhandled OOM encountered in _aligned_realloc"); + AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in _aligned_realloc"); return new_mem; #endif } static void *s_default_calloc(struct aws_allocator *allocator, size_t num, size_t size) { void *mem = s_default_malloc(allocator, num * size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in s_default_malloc"); + AWS_PANIC_OOM(mem, "Unhandled OOM encountered in s_default_malloc"); memset(mem, 0, num * size); return mem; } @@ -121,7 +121,7 @@ void *aws_mem_acquire(struct aws_allocator *allocator, size_t size) { AWS_FATAL_PRECONDITION(size != 0); void *mem = allocator->mem_acquire(allocator, size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_PANIC_OOM(mem, "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } @@ -141,13 +141,13 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { void *mem = allocator->mem_calloc(allocator, num, size); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_PANIC_OOM(mem, "Unhandled OOM encountered in aws_mem_acquire with allocator"); return mem; } /* Otherwise, emulate calloc */ void *mem = allocator->mem_acquire(allocator, required_bytes); - AWS_FATAL_POSTCONDITION(mem && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_PANIC_OOM(mem, "Unhandled OOM encountered in aws_mem_acquire with allocator"); memset(mem, 0, required_bytes); AWS_POSTCONDITION(mem != NULL); @@ -181,7 +181,7 @@ void *aws_mem_acquire_many(struct aws_allocator *allocator, size_t count, ...) { if (total_size > 0) { allocation = aws_mem_acquire(allocator, total_size); - AWS_FATAL_POSTCONDITION(allocation && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_PANIC_OOM(allocation, "Unhandled OOM encountered in aws_mem_acquire with allocator"); uint8_t *current_ptr = allocation; @@ -226,7 +226,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, if (allocator->mem_realloc) { void *newptr = allocator->mem_realloc(allocator, *ptr, oldsize, newsize); - AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_PANIC_OOM(newptr, "Unhandled OOM encountered in aws_mem_acquire with allocator"); *ptr = newptr; return AWS_OP_SUCCESS; @@ -238,7 +238,7 @@ int aws_mem_realloc(struct aws_allocator *allocator, void **ptr, size_t oldsize, } void *newptr = allocator->mem_acquire(allocator, newsize); - AWS_FATAL_POSTCONDITION(newptr && "Unhandled OOM encountered in aws_mem_acquire with allocator"); + AWS_PANIC_OOM(newptr, "Unhandled OOM encountered in aws_mem_acquire with allocator"); memcpy(newptr, *ptr, oldsize); memset((uint8_t *)newptr + oldsize, 0, newsize - oldsize); From baba6071895bdb8b40a48f90d99d6928e8d093bd Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 30 Aug 2021 12:43:38 -0700 Subject: [PATCH 27/34] Lets try again. --- include/aws/common/assert.h | 2 +- source/allocator.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/aws/common/assert.h b/include/aws/common/assert.h index caf0811d6..e7ce341ce 100644 --- a/include/aws/common/assert.h +++ b/include/aws/common/assert.h @@ -30,7 +30,7 @@ AWS_EXTERN_C_END # define AWS_PANIC_OOM(mem, msg) \ do { \ if (!(mem)) { \ - fprintf(stderr, "%s: %s, line %d", msg, __FILE__, __LINE__); \ + fprintf(stderr, "%s", msg); \ abort(); \ } \ } while (0) diff --git a/source/allocator.c b/source/allocator.c index 69cdc7b2a..90a29e182 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -51,7 +51,8 @@ static void *s_default_malloc(struct aws_allocator *allocator, size_t size) { const size_t alignment = sizeof(void *) * (size > PAGE_SIZE ? 8 : 2); #if !defined(_WIN32) void *result = NULL; - posix_memalign(&result, alignment, size); + int err = posix_memalign(&result, alignment, size); + (void)err; AWS_PANIC_OOM(result, "posix_memalign failed to allocate memory"); return result; #else @@ -73,8 +74,6 @@ static void s_default_free(struct aws_allocator *allocator, void *ptr) { static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { (void)allocator; (void)oldsize; - AWS_FATAL_PRECONDITION(ptr); - AWS_FATAL_PRECONDITION(newsize); #if !defined(_WIN32) if (newsize <= oldsize) { From 45d6ee8d79d7b8734e3f2ca63335a236f4dfe074 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 30 Aug 2021 16:34:25 -0700 Subject: [PATCH 28/34] figure it oot --- source/allocator.c | 11 +++++++---- source/string.c | 10 ++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/source/allocator.c b/source/allocator.c index 90a29e182..a67266247 100644 --- a/source/allocator.c +++ b/source/allocator.c @@ -74,6 +74,7 @@ static void s_default_free(struct aws_allocator *allocator, void *ptr) { static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_t oldsize, size_t newsize) { (void)allocator; (void)oldsize; + AWS_FATAL_PRECONDITION(newsize); #if !defined(_WIN32) if (newsize <= oldsize) { @@ -83,8 +84,11 @@ static void *s_default_realloc(struct aws_allocator *allocator, void *ptr, size_ /* newsize is > oldsize, need more memory */ void *new_mem = s_default_malloc(allocator, newsize); AWS_PANIC_OOM(new_mem, "Unhandled OOM encountered in s_default_malloc"); - memcpy(new_mem, ptr, oldsize); - s_default_free(allocator, ptr); + + if (ptr) { + memcpy(new_mem, ptr, oldsize); + s_default_free(allocator, ptr); + } return new_mem; #else @@ -135,7 +139,7 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { * https://wiki.sei.cmu.edu/confluence/display/c/MEM07-C.+Ensure+that+the+arguments+to+calloc%28%29%2C+when+multiplied%2C+do+not+wrap */ size_t required_bytes = 0; - AWS_FATAL_POSTCONDITION(!aws_mul_size_checked(num, size, &required_bytes) && "calloc computed size > SIZE_MAX"); + AWS_FATAL_POSTCONDITION(!aws_mul_size_checked(num, size, &required_bytes), "calloc computed size > SIZE_MAX"); /* If there is a defined calloc, use it */ if (allocator->mem_calloc) { @@ -149,7 +153,6 @@ void *aws_mem_calloc(struct aws_allocator *allocator, size_t num, size_t size) { AWS_PANIC_OOM(mem, "Unhandled OOM encountered in aws_mem_acquire with allocator"); memset(mem, 0, required_bytes); - AWS_POSTCONDITION(mem != NULL); return mem; } diff --git a/source/string.c b/source/string.c index 20b4035d4..9a5d72a07 100644 --- a/source/string.c +++ b/source/string.c @@ -10,6 +10,8 @@ struct aws_string *aws_string_convert_to_wchar_str( struct aws_allocator *allocator, const struct aws_string *to_convert) { + AWS_PRECONDITION(to_convert); + struct aws_byte_cursor convert_cur = aws_byte_cursor_from_string(to_convert); return aws_string_convert_to_wchar_from_byte_cursor(allocator, &convert_cur); } @@ -17,6 +19,8 @@ struct aws_string *aws_string_convert_to_wchar_str( struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( struct aws_allocator *allocator, const struct aws_byte_cursor *to_convert) { + AWS_PRECONDITION(to_convert); + /* if a length is passed for the to_convert string, converted size does not include the null terminator, * which is a good thing. */ int converted_size = MultiByteToWideChar(CP_UTF8, 0, (const char *)to_convert->ptr, (int)to_convert->len, NULL, 0); @@ -63,6 +67,7 @@ static struct aws_string *s_convert_from_wchar( struct aws_allocator *allocator, const wchar_t *to_convert, int len_chars) { + AWS_FATAL_PRECONDITION(to_convert); int bytes_size = WideCharToMultiByte(CP_UTF8, 0, to_convert, len_chars, NULL, 0, NULL, NULL); @@ -99,6 +104,8 @@ static struct aws_string *s_convert_from_wchar( struct aws_string *aws_string_convert_from_wchar_str( struct aws_allocator *allocator, const struct aws_string *to_convert) { + AWS_FATAL_PRECONDITION(to_convert); + return s_convert_from_wchar( allocator, aws_string_wchar_c_str(to_convert), (int)aws_string_wchar_num_chars(to_convert)); } @@ -107,10 +114,13 @@ struct aws_string *aws_string_convert_from_wchar_c_str(struct aws_allocator *all } const wchar_t *aws_string_wchar_c_str(const struct aws_string *str) { + AWS_PRECONDITION(str); return (wchar_t *)str->bytes; } size_t aws_string_wchar_num_chars(const struct aws_string *str) { + AWS_PRECONDITION(str); + if (str->len == 0) { return 0; } From 7dd6922d356e6404330a06a9dd508ded2cdeb414 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 11 Oct 2021 11:02:54 -0700 Subject: [PATCH 29/34] Ran formatter from reasonably decent toolchain. --- source/file.c | 84 ++++++++++++++++++++++----------------------- source/posix/file.c | 2 +- tests/bus_test.c | 7 ++-- 3 files changed, 47 insertions(+), 46 deletions(-) diff --git a/source/file.c b/source/file.c index 94b7a7ad9..e75d99e55 100644 --- a/source/file.c +++ b/source/file.c @@ -3,10 +3,10 @@ * SPDX-License-Identifier: Apache-2.0. */ -#include -#include #include +#include #include +#include #include @@ -20,54 +20,54 @@ FILE *aws_fopen(const char *file_path, const char *mode) { return file; -int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocator *alloc, const char *filename) { - AWS_ZERO_STRUCT(*out_buf); - FILE *fp = aws_fopen(filename, "rb"); + int aws_byte_buf_init_from_file(struct aws_byte_buf * out_buf, struct aws_allocator * alloc, const char *filename) { + AWS_ZERO_STRUCT(*out_buf); + FILE *fp = aws_fopen(filename, "rb"); - if (fp) { - if (fseek(fp, 0L, SEEK_END)) { - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); - fclose(fp); - return aws_translate_and_raise_io_error(errno); - } + if (fp) { + if (fseek(fp, 0L, SEEK_END)) { + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); + fclose(fp); + return aws_translate_and_raise_io_error(errno); + } - size_t allocation_size = (size_t)ftell(fp) + 1; - /* Tell the user that we allocate here and if success they're responsible for the free. */ - if (aws_byte_buf_init(out_buf, alloc, allocation_size)) { - fclose(fp); - return AWS_OP_ERR; - } + size_t allocation_size = (size_t)ftell(fp) + 1; + /* Tell the user that we allocate here and if success they're responsible for the free. */ + if (aws_byte_buf_init(out_buf, alloc, allocation_size)) { + fclose(fp); + return AWS_OP_ERR; + } + + /* Ensure compatibility with null-terminated APIs, but don't consider + * the null terminator part of the length of the payload */ + out_buf->len = out_buf->capacity - 1; + out_buf->buffer[out_buf->len] = 0; - /* Ensure compatibility with null-terminated APIs, but don't consider - * the null terminator part of the length of the payload */ - out_buf->len = out_buf->capacity - 1; - out_buf->buffer[out_buf->len] = 0; + if (fseek(fp, 0L, SEEK_SET)) { + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); + aws_byte_buf_clean_up(out_buf); + fclose(fp); + return aws_translate_and_raise_io_error(errno); + } - if (fseek(fp, 0L, SEEK_SET)) { - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); - aws_byte_buf_clean_up(out_buf); + size_t read = fread(out_buf->buffer, 1, out_buf->len, fp); fclose(fp); - return aws_translate_and_raise_io_error(errno); - } + if (read < out_buf->len) { + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to read file %s with errno %d", filename, errno); + aws_secure_zero(out_buf->buffer, out_buf->len); + aws_byte_buf_clean_up(out_buf); + return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE); + } - size_t read = fread(out_buf->buffer, 1, out_buf->len, fp); - fclose(fp); - if (read < out_buf->len) { - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to read file %s with errno %d", filename, errno); - aws_secure_zero(out_buf->buffer, out_buf->len); - aws_byte_buf_clean_up(out_buf); - return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE); + return AWS_OP_SUCCESS; } - return AWS_OP_SUCCESS; - } + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to open file %s with errno %d", filename, errno); - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to open file %s with errno %d", filename, errno); - - return aws_translate_and_raise_io_error(errno); -} + return aws_translate_and_raise_io_error(errno); + } -bool aws_is_any_directory_separator(char value) { - return value == '\\' || value == '/'; + bool aws_is_any_directory_separator(char value) { + return value == '\\' || value == '/'; >>>>>>> main -} + } diff --git a/source/posix/file.c b/source/posix/file.c index df9c82dc9..f1f9e0bf6 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -6,10 +6,10 @@ #include #include #include +#include #include #include #include -#include #include FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode) { diff --git a/tests/bus_test.c b/tests/bus_test.c index 2ff549ea8..62a407cca 100644 --- a/tests/bus_test.c +++ b/tests/bus_test.c @@ -255,8 +255,8 @@ static int s_bus_async_test_send_multi_threaded(struct aws_allocator *allocator, /* test sending to a bunch of addresses from many threads */ struct bus_test_ctx thread_ctx = { - .bus = bus, - .allocator = allocator, + .bus = bus, + .allocator = allocator, }; AWS_VARIABLE_LENGTH_ARRAY(struct aws_thread, threads, 8); for (int t = 0; t < AWS_ARRAY_SIZE(threads); ++t) { @@ -423,7 +423,8 @@ static int s_bus_async_test_churn(struct aws_allocator *allocator, void *ctx) { size_t recv_count = aws_atomic_load_int(&s_bus_async_churn_data.recv_count); size_t fail_count = aws_atomic_load_int(&s_bus_async_churn_data.fail_count); size_t send_count = aws_atomic_load_int(&s_bus_async_churn_data.send_count); - AWS_LOGF_INFO(AWS_LS_COMMON_TEST, "BUS CHURN TEST: sent: %zu, recv: %zu, fail: %zu", send_count, recv_count, fail_count); + AWS_LOGF_INFO( + AWS_LS_COMMON_TEST, "BUS CHURN TEST: sent: %zu, recv: %zu, fail: %zu", send_count, recv_count, fail_count); /* Ensure SOME messages made it */ ASSERT_TRUE(send_count > 0); ASSERT_TRUE(recv_count > 0); From c5165a1f46e2e58854740ed5a6f7412554dbecda Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 11 Oct 2021 16:39:06 -0700 Subject: [PATCH 30/34] Tests pass again. aws_wstring is now a thing. --- include/aws/common/file.h | 2 +- include/aws/common/string.h | 51 ++++++++++++++-- source/file.c | 12 ++-- source/posix/file.c | 4 +- source/string.c | 84 +++++++++++++++++++++------ source/windows/file.c | 112 +++++++++++++++++++----------------- tests/file_test.c | 2 + 7 files changed, 182 insertions(+), 85 deletions(-) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index e74c0f51f..4ceb0148e 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -135,7 +135,7 @@ struct aws_string *aws_get_home_directory(struct aws_allocator *allocator); * Returns true if a file or path exists, otherwise, false. */ AWS_COMMON_API -bool aws_path_exists(const char *path); +bool aws_path_exists(const struct aws_string *path); /* * Wrapper for highest-resolution platform-dependent seek implementation. diff --git a/include/aws/common/string.h b/include/aws/common/string.h index 4a1a85c7d..1a3c6838c 100644 --- a/include/aws/common/string.h +++ b/include/aws/common/string.h @@ -46,6 +46,17 @@ struct aws_string { /* give this a storage specifier for C++ purposes. It will likely be larger after init. */ const uint8_t bytes[1]; }; + +#ifdef _WIN32 +struct aws_wstring { + struct aws_allocator* const allocator; + /* number of characters in the string not including the null terminator. */ + const size_t len; + /* give this a storage specifier for C++ purposes. It will likely be larger after init. */ + const wchar_t bytes[1]; +}; +#endif /* _WIN32 */ + #ifdef _MSC_VER # pragma warning(pop) #endif @@ -60,7 +71,7 @@ AWS_EXTERN_C_BEGIN * * returns NULL on failure. */ -AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_str( +AWS_COMMON_API struct aws_wstring * aws_string_convert_to_wstring( struct aws_allocator *allocator, const struct aws_string *to_convert); @@ -71,10 +82,16 @@ AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_str( * * returns NULL on failure. */ -AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( +AWS_COMMON_API struct aws_wstring *aws_string_convert_to_wchar_from_byte_cursor( struct aws_allocator *allocator, const struct aws_byte_cursor *to_convert); +/** + * clean up str. + */ +AWS_COMMON_API +void aws_wstring_destroy(struct aws_wstring* str); + /** * For windows only. Converts `to_convert` from a windows whcar format (UTF-16) to UTF-8. * @@ -84,7 +101,7 @@ AWS_COMMON_API struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( */ AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_str( struct aws_allocator *allocator, - const struct aws_string *to_convert); + const struct aws_wstring *to_convert); /** * For windows only. Converts `to_convert` from a windows whcar format (UTF-16) to UTF-8. @@ -108,15 +125,39 @@ AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_c_str( struct aws_allocator *allocator, const wchar_t *to_convert); +/** + * Create a new wide string from a byte cursor. This assumes that w_str_cur is already in utf-16. + * + * returns NULL on failure. + */ +AWS_COMMON_API struct aws_wstring* aws_wstring_new_from_cursor(struct aws_allocator *allocator, const struct aws_byte_cursor* w_str_cur); + +/** + * Create a new wide string from a utf-16 string enclosing array. The length field is in number of characters not counting the null terminator. + * + * returns NULL on failure. + */ +AWS_COMMON_API struct aws_wstring* aws_wstring_new_from_array(struct aws_allocator *allocator, const wchar_t* w_str, size_t length); + /** * Returns a wchar_t * pointer for use with windows OS interop. */ -AWS_COMMON_API const wchar_t *aws_string_wchar_c_str(const struct aws_string *str); +AWS_COMMON_API const wchar_t *aws_wstring_c_str(const struct aws_wstring *str); /** * Returns the number of characters in the wchar string. NOTE: This is not the length in bytes or the buffer size. */ -AWS_COMMON_API size_t aws_string_wchar_num_chars(const struct aws_string *str); +AWS_COMMON_API size_t aws_wstring_num_chars(const struct aws_wstring *str); + +/** + * Returns the length in bytes for the buffer. + */ +AWS_COMMON_API size_t aws_wstring_size_bytes(const struct aws_wstring* str); + +/** + * Verifies that str is a valid string. Returns true if it's valid and false otherwise. + */ +AWS_COMMON_API bool aws_wstring_is_valid(const struct aws_wstring* str); #endif /* _WIN32 */ diff --git a/source/file.c b/source/file.c index e75d99e55..5f5ff32f6 100644 --- a/source/file.c +++ b/source/file.c @@ -10,17 +10,18 @@ #include -FILE *aws_fopen(const char *file_path, const char *mode) { - struct aws_string *file_path_str = aws_string_new_from_c_str(aws_default_allocator(), file_path); - struct aws_string *mode_str = aws_string_new_from_c_str(aws_default_allocator(), mode); +FILE* aws_fopen(const char* file_path, const char* mode) { + struct aws_string* file_path_str = aws_string_new_from_c_str(aws_default_allocator(), file_path); + struct aws_string* mode_str = aws_string_new_from_c_str(aws_default_allocator(), mode); - FILE *file = aws_fopen_safe(file_path_str, mode_str); + FILE* file = aws_fopen_safe(file_path_str, mode_str); aws_string_destroy(mode_str); aws_string_destroy(file_path_str); return file; +} - int aws_byte_buf_init_from_file(struct aws_byte_buf * out_buf, struct aws_allocator * alloc, const char *filename) { +int aws_byte_buf_init_from_file(struct aws_byte_buf * out_buf, struct aws_allocator * alloc, const char *filename) { AWS_ZERO_STRUCT(*out_buf); FILE *fp = aws_fopen(filename, "rb"); @@ -69,5 +70,4 @@ FILE *aws_fopen(const char *file_path, const char *mode) { bool aws_is_any_directory_separator(char value) { return value == '\\' || value == '/'; ->>>>>>> main } diff --git a/source/posix/file.c b/source/posix/file.c index f1f9e0bf6..7c26ade8c 100644 --- a/source/posix/file.c +++ b/source/posix/file.c @@ -236,9 +236,9 @@ struct aws_string *aws_get_home_directory(struct aws_allocator *allocator) { return NULL; } -bool aws_path_exists(const char *path) { +bool aws_path_exists(const struct aws_string *path) { struct stat buffer; - return stat(path, &buffer) == 0; + return stat(aws_string_c_str(path), &buffer) == 0; } int aws_fseek(FILE *file, int64_t offset, int whence) { diff --git a/source/string.c b/source/string.c index 9a5d72a07..24630ce3e 100644 --- a/source/string.c +++ b/source/string.c @@ -7,7 +7,7 @@ #ifdef _WIN32 # include -struct aws_string *aws_string_convert_to_wchar_str( +struct aws_wstring *aws_string_convert_to_wstring( struct aws_allocator *allocator, const struct aws_string *to_convert) { AWS_PRECONDITION(to_convert); @@ -16,7 +16,7 @@ struct aws_string *aws_string_convert_to_wchar_str( return aws_string_convert_to_wchar_from_byte_cursor(allocator, &convert_cur); } -struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( +struct aws_wstring *aws_string_convert_to_wchar_from_byte_cursor( struct aws_allocator *allocator, const struct aws_byte_cursor *to_convert) { AWS_PRECONDITION(to_convert); @@ -39,30 +39,75 @@ struct aws_string *aws_string_convert_to_wchar_from_byte_cursor( } /* UTF-16, the NULL terminator is two bytes. */ - if (aws_add_size_checked(sizeof(struct aws_string) + 2, str_len_size, &malloc_size)) { + if (aws_add_size_checked(sizeof(struct aws_wstring) + 2, str_len_size, &malloc_size)) { return NULL; } - struct aws_string *str = aws_mem_acquire(allocator, malloc_size); + struct aws_wstring *str = aws_mem_acquire(allocator, malloc_size); if (!str) { return NULL; } /* Fields are declared const, so we need to copy them in like this */ *(struct aws_allocator **)(&str->allocator) = allocator; - *(size_t *)(&str->len) = str_len_size; + *(size_t *)(&str->len) = (size_t)converted_size; int converted_res = MultiByteToWideChar( CP_UTF8, 0, (const char *)to_convert->ptr, (int)to_convert->len, (wchar_t *)str->bytes, converted_size); /* windows had its chance to do its thing, no take backsies. */ AWS_FATAL_ASSERT(converted_res > 0); - /* remember.... NULL term is TWO bytes for UTF-16. */ - *(uint8_t *)&str->bytes[str_len_size] = 0; - *(uint8_t *)&str->bytes[str_len_size + 1] = 0; + *(wchar_t *)&str->bytes[converted_size] = 0; return str; } +struct aws_wstring* aws_wstring_new_from_cursor(struct aws_allocator *allocator, const struct aws_byte_cursor* w_str_cur) { + AWS_PRECONDITION(allocator && aws_byte_cursor_is_valid(w_str_cur)); + return aws_wstring_new_from_array(allocator, (wchar_t *)w_str_cur->ptr, w_str_cur->len / sizeof(wchar_t)); +} + +struct aws_wstring* aws_wstring_new_from_array(struct aws_allocator* allocator, const wchar_t* w_str, size_t len) { + AWS_PRECONDITION(allocator); + AWS_PRECONDITION(AWS_MEM_IS_READABLE(bytes, len)); + + size_t str_byte_len = 0; + size_t malloc_size = 0; + + /* double the size because the return value above is # of characters, not bytes size. */ + if (aws_mul_size_checked(sizeof(wchar_t), len, &str_byte_len)) { + return NULL; + } + + /* UTF-16, the NULL terminator is two bytes. */ + if (aws_add_size_checked(sizeof(struct aws_wstring) + 2, str_byte_len, &malloc_size)) { + return NULL; + } + + struct aws_wstring* str = aws_mem_acquire(allocator, malloc_size); + + /* Fields are declared const, so we need to copy them in like this */ + *(struct aws_allocator**)(&str->allocator) = allocator; + *(size_t*)(&str->len) = len; + if (len > 0) { + memcpy((void*)str->bytes, w_str, str_byte_len); + } + /* in case this is a utf-16 string in the array, allow that here. */ + *(wchar_t *)&str->bytes[len] = 0; + AWS_RETURN_WITH_POSTCONDITION(str, aws_wstring_is_valid(str)); +} + +bool aws_wstring_is_valid(const struct aws_wstring* str) { + return str && AWS_MEM_IS_READABLE(&str->bytes[0], str->len + 1) && str->bytes[str->len] == 0; +} + +void aws_wstring_destroy(struct aws_wstring* str) { + AWS_PRECONDITION(!str || aws_string_is_valid(str)); + if (str && str->allocator) { + aws_mem_release(str->allocator, str); + } +} + + static struct aws_string *s_convert_from_wchar( struct aws_allocator *allocator, const wchar_t *to_convert, @@ -103,30 +148,35 @@ static struct aws_string *s_convert_from_wchar( struct aws_string *aws_string_convert_from_wchar_str( struct aws_allocator *allocator, - const struct aws_string *to_convert) { + const struct aws_wstring *to_convert) { AWS_FATAL_PRECONDITION(to_convert); return s_convert_from_wchar( - allocator, aws_string_wchar_c_str(to_convert), (int)aws_string_wchar_num_chars(to_convert)); + allocator, aws_wstring_c_str(to_convert), (int)aws_wstring_num_chars(to_convert)); } struct aws_string *aws_string_convert_from_wchar_c_str(struct aws_allocator *allocator, const wchar_t *to_convert) { return s_convert_from_wchar(allocator, to_convert, -1); } -const wchar_t *aws_string_wchar_c_str(const struct aws_string *str) { +const wchar_t * aws_wstring_c_str(const struct aws_wstring *str) { AWS_PRECONDITION(str); - return (wchar_t *)str->bytes; + return str->bytes; } -size_t aws_string_wchar_num_chars(const struct aws_string *str) { +size_t aws_wstring_num_chars(const struct aws_wstring *str) { AWS_PRECONDITION(str); if (str->len == 0) { return 0; } - AWS_FATAL_ASSERT(str->len % 2 == 0); - return str->len / sizeof(wchar_t); + return str->len; +} + +size_t aws_wstring_size_bytes(const struct aws_wstring* str) { + AWS_PRECONDITION(str); + + return aws_wstring_num_chars(str) * sizeof(wchar_t); } #endif /* _WIN32 */ @@ -140,7 +190,7 @@ struct aws_string *aws_string_new_from_array(struct aws_allocator *allocator, co AWS_PRECONDITION(allocator); AWS_PRECONDITION(AWS_MEM_IS_READABLE(bytes, len)); size_t malloc_size; - if (aws_add_size_checked(sizeof(struct aws_string) + 2, len, &malloc_size)) { + if (aws_add_size_checked(sizeof(struct aws_string) + 1, len, &malloc_size)) { return NULL; } struct aws_string *str = aws_mem_acquire(allocator, malloc_size); @@ -154,9 +204,7 @@ struct aws_string *aws_string_new_from_array(struct aws_allocator *allocator, co if (len > 0) { memcpy((void *)str->bytes, bytes, len); } - /* in case this is a utf-16 string in the array, allow that here. */ *(uint8_t *)&str->bytes[len] = 0; - *(uint8_t *)&str->bytes[len + 1] = 0; AWS_RETURN_WITH_POSTCONDITION(str, aws_string_is_valid(str)); } diff --git a/source/windows/file.c b/source/windows/file.c index 9f09af6a3..70e7dd76d 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -10,16 +10,18 @@ #include #include #include +#include +#include FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode) { - struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); - struct aws_string *w_mode = aws_string_convert_to_wchar_str(aws_default_allocator(), mode); + struct aws_wstring *w_file_path = aws_string_convert_to_wstring(aws_default_allocator(), file_path); + struct aws_wstring *w_mode = aws_string_convert_to_wstring(aws_default_allocator(), mode); FILE *file = NULL; - errno_t error = _wfopen_s(&file, aws_string_wchar_c_str(w_file_path), aws_string_wchar_c_str(w_mode)); + errno_t error = _wfopen_s(&file, aws_wstring_c_str(w_file_path), aws_wstring_c_str(w_mode)); /* actually handle the error correctly here. */ - aws_string_destroy(w_mode); - aws_string_destroy(w_file_path); + aws_wstring_destroy(w_mode); + aws_wstring_destroy(w_file_path); if (error) { aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); @@ -28,11 +30,11 @@ FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string return file; } -struct aws_string *s_to_long_path(struct aws_allocator *allocator, const struct aws_string *path) { +struct aws_wstring *s_to_long_path(struct aws_allocator *allocator, const struct aws_wstring *path) { wchar_t prefix[] = L"\\\\?\\"; size_t prefix_size = sizeof(prefix); - if (path->len > MAX_PATH - prefix_size) { + if (aws_wstring_num_chars(path) > MAX_PATH - prefix_size) { struct aws_byte_buf new_path; aws_byte_buf_init(&new_path, allocator, sizeof(prefix) + path->len + 2); @@ -40,25 +42,25 @@ struct aws_string *s_to_long_path(struct aws_allocator *allocator, const struct struct aws_byte_cursor prefix_cur = aws_byte_cursor_from_array((uint8_t *)prefix, sizeof(prefix) - 2); aws_byte_buf_append_dynamic(&new_path, &prefix_cur); - struct aws_byte_cursor path_cur = aws_byte_cursor_from_array(aws_string_bytes(path), path->len); + struct aws_byte_cursor path_cur = aws_byte_cursor_from_array((uint8_t *)aws_wstring_c_str(path), path->len); aws_byte_buf_append_dynamic(&new_path, &path_cur); - struct aws_string *long_path = aws_string_new_from_buf(allocator, &new_path); + struct aws_wstring *long_path = aws_wstring_new_from_array(allocator, (wchar_t *)new_path.buffer, new_path.len / sizeof(wchar_t)); aws_byte_buf_clean_up(&new_path); return long_path; } - return aws_string_new_from_string(allocator, path); + return aws_wstring_new_from_array(allocator, aws_wstring_c_str(path), aws_wstring_num_chars(path)); } int aws_directory_create(const struct aws_string *dir_path) { - struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); - aws_string_destroy(w_dir_path); + struct aws_wstring *w_dir_path = aws_string_convert_to_wstring(aws_default_allocator(), dir_path); + struct aws_wstring *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); + aws_wstring_destroy(w_dir_path); - BOOL create_dir_res = CreateDirectoryW(aws_string_wchar_c_str(long_dir_path), NULL); - aws_string_destroy(long_dir_path); + BOOL create_dir_res = CreateDirectoryW(aws_wstring_c_str(long_dir_path), NULL); + aws_wstring_destroy(long_dir_path); int error = GetLastError(); if (!create_dir_res) { @@ -81,12 +83,12 @@ int aws_directory_create(const struct aws_string *dir_path) { } bool aws_directory_exists(const struct aws_string *dir_path) { - struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); - aws_string_destroy(w_dir_path); + struct aws_wstring *w_dir_path = aws_string_convert_to_wstring(aws_default_allocator(), dir_path); + struct aws_wstring *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); + aws_wstring_destroy(w_dir_path); - DWORD attributes = GetFileAttributesW(aws_string_wchar_c_str(long_dir_path)); - aws_string_destroy(long_dir_path); + DWORD attributes = GetFileAttributesW(aws_wstring_c_str(long_dir_path)); + aws_wstring_destroy(long_dir_path); return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY)); } @@ -131,12 +133,12 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { return AWS_OP_ERR; } - struct aws_string *w_dir_path = aws_string_convert_to_wchar_str(aws_default_allocator(), dir_path); - struct aws_string *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); - aws_string_destroy(w_dir_path); + struct aws_wstring *w_dir_path = aws_string_convert_to_wstring(aws_default_allocator(), dir_path); + struct aws_wstring *long_dir_path = s_to_long_path(aws_default_allocator(), w_dir_path); + aws_wstring_destroy(w_dir_path); - BOOL remove_dir_res = RemoveDirectoryW(aws_string_wchar_c_str(long_dir_path)); - aws_string_destroy(long_dir_path); + BOOL remove_dir_res = RemoveDirectoryW(aws_wstring_c_str(long_dir_path)); + aws_wstring_destroy(long_dir_path); if (!remove_dir_res) { int error = GetLastError(); @@ -155,12 +157,12 @@ int aws_directory_delete(const struct aws_string *dir_path, bool recursive) { } int aws_file_delete(const struct aws_string *file_path) { - struct aws_string *w_file_path = aws_string_convert_to_wchar_str(aws_default_allocator(), file_path); - struct aws_string *long_file_path = s_to_long_path(aws_default_allocator(), w_file_path); - aws_string_destroy(w_file_path); + struct aws_wstring *w_file_path = aws_string_convert_to_wstring(aws_default_allocator(), file_path); + struct aws_wstring *long_file_path = s_to_long_path(aws_default_allocator(), w_file_path); + aws_wstring_destroy(w_file_path); - BOOL remove_file_res = DeleteFileW(aws_string_wchar_c_str(long_file_path)); - aws_string_destroy(long_file_path); + BOOL remove_file_res = DeleteFileW(aws_wstring_c_str(long_file_path)); + aws_wstring_destroy(long_file_path); if (!remove_file_res) { int error = GetLastError(); @@ -179,17 +181,17 @@ int aws_file_delete(const struct aws_string *file_path) { } int aws_directory_or_file_move(const struct aws_string *from, const struct aws_string *to) { - struct aws_string *w_from_path = aws_string_convert_to_wchar_str(aws_default_allocator(), from); - struct aws_string *long_from_path = s_to_long_path(aws_default_allocator(), w_from_path); - aws_string_destroy(w_from_path); + struct aws_wstring *w_from_path = aws_string_convert_to_wstring(aws_default_allocator(), from); + struct aws_wstring *long_from_path = s_to_long_path(aws_default_allocator(), w_from_path); + aws_wstring_destroy(w_from_path); - struct aws_string *w_to_path = aws_string_convert_to_wchar_str(aws_default_allocator(), to); - struct aws_string *long_to_path = s_to_long_path(aws_default_allocator(), w_to_path); - aws_string_destroy(w_to_path); + struct aws_wstring *w_to_path = aws_string_convert_to_wstring(aws_default_allocator(), to); + struct aws_wstring *long_to_path = s_to_long_path(aws_default_allocator(), w_to_path); + aws_wstring_destroy(w_to_path); - BOOL move_res = MoveFileW(aws_string_wchar_c_str(long_from_path), aws_string_wchar_c_str(long_to_path)); - aws_string_destroy(long_from_path); - aws_string_destroy(long_to_path); + BOOL move_res = MoveFileW(aws_wstring_c_str(long_from_path), aws_wstring_c_str(long_to_path)); + aws_wstring_destroy(long_from_path); + aws_wstring_destroy(long_to_path); if (!move_res) { int error = GetLastError(); @@ -213,23 +215,23 @@ int aws_directory_traverse( bool recursive, aws_on_directory_entry *on_entry, void *user_data) { - struct aws_string *w_path_wchar = aws_string_convert_to_wchar_str(allocator, path); - struct aws_string *long_path_wchar = s_to_long_path(allocator, w_path_wchar); - aws_string_destroy(w_path_wchar); + struct aws_wstring *w_path_wchar = aws_string_convert_to_wstring(allocator, path); + struct aws_wstring *long_path_wchar = s_to_long_path(allocator, w_path_wchar); + aws_wstring_destroy(w_path_wchar); /* windows doesn't fail in FindFirstFile if it's not a directory. Do the check here. We don't call the perfectly good function for this check because the string is already converted to utf-16 and it's trivial to reuse it. */ - DWORD attributes = GetFileAttributesW(aws_string_wchar_c_str(long_path_wchar)); + DWORD attributes = GetFileAttributesW(aws_wstring_c_str(long_path_wchar)); if (!(attributes & FILE_ATTRIBUTE_DIRECTORY)) { - aws_string_destroy(long_path_wchar); + aws_wstring_destroy(long_path_wchar); return aws_raise_error(AWS_ERROR_FILE_INVALID_PATH); } WIN32_FIND_DATAW ffd; - HANDLE find_handle = FindFirstFileW(aws_string_wchar_c_str(long_path_wchar), &ffd); + HANDLE find_handle = FindFirstFileW(aws_wstring_c_str(long_path_wchar), &ffd); if (find_handle == INVALID_HANDLE_VALUE) { - aws_string_destroy(long_path_wchar); + aws_wstring_destroy(long_path_wchar); int error = GetLastError(); @@ -244,7 +246,7 @@ int aws_directory_traverse( /* create search path string */ struct aws_byte_cursor path_wchar_cur = - aws_byte_cursor_from_array(aws_string_bytes(long_path_wchar), long_path_wchar->len); + aws_byte_cursor_from_array(aws_wstring_c_str(long_path_wchar), aws_wstring_size_bytes(long_path_wchar)); struct aws_byte_buf search_wchar_buf; aws_byte_buf_init_copy_from_cursor(&search_wchar_buf, allocator, path_wchar_cur); @@ -253,11 +255,12 @@ int aws_directory_traverse( aws_byte_cursor_from_array((uint8_t *)search_wchar_pattern, sizeof(search_wchar_pattern)); aws_byte_buf_append_dynamic(&search_wchar_buf, &search_char_wchar); + struct aws_byte_cursor search_wchar_cur = aws_byte_cursor_from_buf(&search_wchar_buf); /* it's already converted to wide string */ - struct aws_string *search_wchar_string = aws_string_new_from_buf(allocator, &search_wchar_buf); + struct aws_wstring* search_wchar_string = aws_wstring_new_from_cursor(allocator, &search_wchar_cur); - find_handle = FindFirstFileW(aws_string_wchar_c_str(search_wchar_string), &ffd); - aws_string_destroy(search_wchar_string); + find_handle = FindFirstFileW(aws_wstring_c_str(search_wchar_string), &ffd); + aws_wstring_destroy(search_wchar_string); aws_byte_buf_clean_up(&search_wchar_buf); int ret_val = AWS_OP_SUCCESS; @@ -362,7 +365,7 @@ int aws_directory_traverse( } while (ret_val == AWS_OP_SUCCESS && FindNextFileW(find_handle, &ffd)); - aws_string_destroy(long_path_wchar); + aws_wstring_destroy(long_path_wchar); if (find_handle != INVALID_HANDLE_VALUE) { FindClose(find_handle); } @@ -435,8 +438,11 @@ struct aws_string *aws_get_home_directory(struct aws_allocator *allocator) { return NULL; } -bool aws_path_exists(const char *path) { - return PathFileExistsA(path) == TRUE; +bool aws_path_exists(const struct aws_string *path) { + struct aws_wstring *wchar_path = aws_string_convert_to_wstring(aws_default_allocator(), path); + bool ret_val = PathFileExistsW(aws_wstring_c_str(wchar_path)) == TRUE; + aws_wstring_destroy(wchar_path); + return ret_val; } int aws_fseek(FILE *file, int64_t offset, int whence) { diff --git a/tests/file_test.c b/tests/file_test.c index 82ecc1aef..fb3a49e14 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -290,6 +290,8 @@ static int s_directory_move_succeeds_test_fn(struct aws_allocator *allocator, vo ASSERT_FALSE(aws_directory_exists(path)); ASSERT_TRUE(aws_directory_exists(to_path)); + ASSERT_SUCCESS(aws_directory_delete(to_path, true)); + aws_string_destroy(to_path); aws_string_destroy(path); return AWS_OP_SUCCESS; From c0346a219dba6feb2bd5c6bf4c1aac8bf9edf07f Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 11 Oct 2021 16:40:34 -0700 Subject: [PATCH 31/34] Ran formatter. --- source/file.c | 88 +++++++++++++++++++++---------------------- source/string.c | 28 +++++++------- source/windows/file.c | 9 +++-- 3 files changed, 63 insertions(+), 62 deletions(-) diff --git a/source/file.c b/source/file.c index 5f5ff32f6..c334aa401 100644 --- a/source/file.c +++ b/source/file.c @@ -10,64 +10,64 @@ #include -FILE* aws_fopen(const char* file_path, const char* mode) { - struct aws_string* file_path_str = aws_string_new_from_c_str(aws_default_allocator(), file_path); - struct aws_string* mode_str = aws_string_new_from_c_str(aws_default_allocator(), mode); +FILE *aws_fopen(const char *file_path, const char *mode) { + struct aws_string *file_path_str = aws_string_new_from_c_str(aws_default_allocator(), file_path); + struct aws_string *mode_str = aws_string_new_from_c_str(aws_default_allocator(), mode); - FILE* file = aws_fopen_safe(file_path_str, mode_str); + FILE *file = aws_fopen_safe(file_path_str, mode_str); aws_string_destroy(mode_str); aws_string_destroy(file_path_str); return file; } -int aws_byte_buf_init_from_file(struct aws_byte_buf * out_buf, struct aws_allocator * alloc, const char *filename) { - AWS_ZERO_STRUCT(*out_buf); - FILE *fp = aws_fopen(filename, "rb"); +int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocator *alloc, const char *filename) { + AWS_ZERO_STRUCT(*out_buf); + FILE *fp = aws_fopen(filename, "rb"); - if (fp) { - if (fseek(fp, 0L, SEEK_END)) { - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); - fclose(fp); - return aws_translate_and_raise_io_error(errno); - } - - size_t allocation_size = (size_t)ftell(fp) + 1; - /* Tell the user that we allocate here and if success they're responsible for the free. */ - if (aws_byte_buf_init(out_buf, alloc, allocation_size)) { - fclose(fp); - return AWS_OP_ERR; - } + if (fp) { + if (fseek(fp, 0L, SEEK_END)) { + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); + fclose(fp); + return aws_translate_and_raise_io_error(errno); + } - /* Ensure compatibility with null-terminated APIs, but don't consider - * the null terminator part of the length of the payload */ - out_buf->len = out_buf->capacity - 1; - out_buf->buffer[out_buf->len] = 0; + size_t allocation_size = (size_t)ftell(fp) + 1; + /* Tell the user that we allocate here and if success they're responsible for the free. */ + if (aws_byte_buf_init(out_buf, alloc, allocation_size)) { + fclose(fp); + return AWS_OP_ERR; + } - if (fseek(fp, 0L, SEEK_SET)) { - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); - aws_byte_buf_clean_up(out_buf); - fclose(fp); - return aws_translate_and_raise_io_error(errno); - } + /* Ensure compatibility with null-terminated APIs, but don't consider + * the null terminator part of the length of the payload */ + out_buf->len = out_buf->capacity - 1; + out_buf->buffer[out_buf->len] = 0; - size_t read = fread(out_buf->buffer, 1, out_buf->len, fp); + if (fseek(fp, 0L, SEEK_SET)) { + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno); + aws_byte_buf_clean_up(out_buf); fclose(fp); - if (read < out_buf->len) { - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to read file %s with errno %d", filename, errno); - aws_secure_zero(out_buf->buffer, out_buf->len); - aws_byte_buf_clean_up(out_buf); - return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE); - } - - return AWS_OP_SUCCESS; + return aws_translate_and_raise_io_error(errno); } - AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to open file %s with errno %d", filename, errno); + size_t read = fread(out_buf->buffer, 1, out_buf->len, fp); + fclose(fp); + if (read < out_buf->len) { + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to read file %s with errno %d", filename, errno); + aws_secure_zero(out_buf->buffer, out_buf->len); + aws_byte_buf_clean_up(out_buf); + return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE); + } - return aws_translate_and_raise_io_error(errno); + return AWS_OP_SUCCESS; } - bool aws_is_any_directory_separator(char value) { - return value == '\\' || value == '/'; - } + AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to open file %s with errno %d", filename, errno); + + return aws_translate_and_raise_io_error(errno); +} + +bool aws_is_any_directory_separator(char value) { + return value == '\\' || value == '/'; +} diff --git a/source/string.c b/source/string.c index 24630ce3e..0bbc53517 100644 --- a/source/string.c +++ b/source/string.c @@ -61,12 +61,14 @@ struct aws_wstring *aws_string_convert_to_wchar_from_byte_cursor( return str; } -struct aws_wstring* aws_wstring_new_from_cursor(struct aws_allocator *allocator, const struct aws_byte_cursor* w_str_cur) { +struct aws_wstring *aws_wstring_new_from_cursor( + struct aws_allocator *allocator, + const struct aws_byte_cursor *w_str_cur) { AWS_PRECONDITION(allocator && aws_byte_cursor_is_valid(w_str_cur)); return aws_wstring_new_from_array(allocator, (wchar_t *)w_str_cur->ptr, w_str_cur->len / sizeof(wchar_t)); } -struct aws_wstring* aws_wstring_new_from_array(struct aws_allocator* allocator, const wchar_t* w_str, size_t len) { +struct aws_wstring *aws_wstring_new_from_array(struct aws_allocator *allocator, const wchar_t *w_str, size_t len) { AWS_PRECONDITION(allocator); AWS_PRECONDITION(AWS_MEM_IS_READABLE(bytes, len)); @@ -82,32 +84,31 @@ struct aws_wstring* aws_wstring_new_from_array(struct aws_allocator* allocator, if (aws_add_size_checked(sizeof(struct aws_wstring) + 2, str_byte_len, &malloc_size)) { return NULL; } - - struct aws_wstring* str = aws_mem_acquire(allocator, malloc_size); + + struct aws_wstring *str = aws_mem_acquire(allocator, malloc_size); /* Fields are declared const, so we need to copy them in like this */ - *(struct aws_allocator**)(&str->allocator) = allocator; - *(size_t*)(&str->len) = len; + *(struct aws_allocator **)(&str->allocator) = allocator; + *(size_t *)(&str->len) = len; if (len > 0) { - memcpy((void*)str->bytes, w_str, str_byte_len); + memcpy((void *)str->bytes, w_str, str_byte_len); } /* in case this is a utf-16 string in the array, allow that here. */ *(wchar_t *)&str->bytes[len] = 0; AWS_RETURN_WITH_POSTCONDITION(str, aws_wstring_is_valid(str)); } -bool aws_wstring_is_valid(const struct aws_wstring* str) { +bool aws_wstring_is_valid(const struct aws_wstring *str) { return str && AWS_MEM_IS_READABLE(&str->bytes[0], str->len + 1) && str->bytes[str->len] == 0; } -void aws_wstring_destroy(struct aws_wstring* str) { +void aws_wstring_destroy(struct aws_wstring *str) { AWS_PRECONDITION(!str || aws_string_is_valid(str)); if (str && str->allocator) { aws_mem_release(str->allocator, str); } } - static struct aws_string *s_convert_from_wchar( struct aws_allocator *allocator, const wchar_t *to_convert, @@ -151,14 +152,13 @@ struct aws_string *aws_string_convert_from_wchar_str( const struct aws_wstring *to_convert) { AWS_FATAL_PRECONDITION(to_convert); - return s_convert_from_wchar( - allocator, aws_wstring_c_str(to_convert), (int)aws_wstring_num_chars(to_convert)); + return s_convert_from_wchar(allocator, aws_wstring_c_str(to_convert), (int)aws_wstring_num_chars(to_convert)); } struct aws_string *aws_string_convert_from_wchar_c_str(struct aws_allocator *allocator, const wchar_t *to_convert) { return s_convert_from_wchar(allocator, to_convert, -1); } -const wchar_t * aws_wstring_c_str(const struct aws_wstring *str) { +const wchar_t *aws_wstring_c_str(const struct aws_wstring *str) { AWS_PRECONDITION(str); return str->bytes; } @@ -173,7 +173,7 @@ size_t aws_wstring_num_chars(const struct aws_wstring *str) { return str->len; } -size_t aws_wstring_size_bytes(const struct aws_wstring* str) { +size_t aws_wstring_size_bytes(const struct aws_wstring *str) { AWS_PRECONDITION(str); return aws_wstring_num_chars(str) * sizeof(wchar_t); diff --git a/source/windows/file.c b/source/windows/file.c index 70e7dd76d..9c74d0e32 100644 --- a/source/windows/file.c +++ b/source/windows/file.c @@ -7,11 +7,11 @@ #include #include +#include #include +#include #include #include -#include -#include FILE *aws_fopen_safe(const struct aws_string *file_path, const struct aws_string *mode) { struct aws_wstring *w_file_path = aws_string_convert_to_wstring(aws_default_allocator(), file_path); @@ -45,7 +45,8 @@ struct aws_wstring *s_to_long_path(struct aws_allocator *allocator, const struct struct aws_byte_cursor path_cur = aws_byte_cursor_from_array((uint8_t *)aws_wstring_c_str(path), path->len); aws_byte_buf_append_dynamic(&new_path, &path_cur); - struct aws_wstring *long_path = aws_wstring_new_from_array(allocator, (wchar_t *)new_path.buffer, new_path.len / sizeof(wchar_t)); + struct aws_wstring *long_path = + aws_wstring_new_from_array(allocator, (wchar_t *)new_path.buffer, new_path.len / sizeof(wchar_t)); aws_byte_buf_clean_up(&new_path); return long_path; @@ -257,7 +258,7 @@ int aws_directory_traverse( aws_byte_buf_append_dynamic(&search_wchar_buf, &search_char_wchar); struct aws_byte_cursor search_wchar_cur = aws_byte_cursor_from_buf(&search_wchar_buf); /* it's already converted to wide string */ - struct aws_wstring* search_wchar_string = aws_wstring_new_from_cursor(allocator, &search_wchar_cur); + struct aws_wstring *search_wchar_string = aws_wstring_new_from_cursor(allocator, &search_wchar_cur); find_handle = FindFirstFileW(aws_wstring_c_str(search_wchar_string), &ffd); aws_wstring_destroy(search_wchar_string); From 6a8251647fb275c8dbb54b41bd72dfbe8c1a2f48 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 11 Oct 2021 16:48:34 -0700 Subject: [PATCH 32/34] addressed remaining PR comments. --- include/aws/common/file.h | 5 +++-- include/aws/common/string.h | 8 ++++---- source/allocator_sba.c | 2 -- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 4ceb0148e..3f86971b2 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -6,9 +6,10 @@ */ #include #include +#include #include -#ifdef _WIN32 +#ifdef AWS_OS_WINDOWS # define AWS_PATH_DELIM '\\' # define AWS_PATH_DELIM_STR "\\" #else @@ -98,7 +99,7 @@ AWS_COMMON_API int aws_directory_or_file_move(const struct aws_string *from, con /** * Traverse a directory starting at path. * - * If you want the traversal to recursive the entire directory, pass recursive as true. Passing false for this parameter + * If you want the traversal to recurse the entire directory, pass recursive as true. Passing false for this parameter * will only iterate the contents of the directory, but will not descend into any directories it encounters. * * If recursive is set to true, the traversal is performed post-order, depth-first diff --git a/include/aws/common/string.h b/include/aws/common/string.h index 1a3c6838c..be7b51266 100644 --- a/include/aws/common/string.h +++ b/include/aws/common/string.h @@ -47,7 +47,7 @@ struct aws_string { const uint8_t bytes[1]; }; -#ifdef _WIN32 +#ifdef AWS_OS_WINDOWS struct aws_wstring { struct aws_allocator* const allocator; /* number of characters in the string not including the null terminator. */ @@ -55,7 +55,7 @@ struct aws_wstring { /* give this a storage specifier for C++ purposes. It will likely be larger after init. */ const wchar_t bytes[1]; }; -#endif /* _WIN32 */ +#endif /* AWS_OS_WINDOWS */ #ifdef _MSC_VER # pragma warning(pop) @@ -63,7 +63,7 @@ struct aws_wstring { AWS_EXTERN_C_BEGIN -#ifdef _WIN32 +#ifdef AWS_OS_WINDOWS /** * For windows only. Converts `to_convert` to a windows whcar format (UTF-16) for use with windows OS interop. * @@ -159,7 +159,7 @@ AWS_COMMON_API size_t aws_wstring_size_bytes(const struct aws_wstring* str); */ AWS_COMMON_API bool aws_wstring_is_valid(const struct aws_wstring* str); -#endif /* _WIN32 */ +#endif /* AWS_OS_WINDOWS */ /** * Returns true if bytes of string are the same, false otherwise. diff --git a/source/allocator_sba.c b/source/allocator_sba.c index eeb518d93..47f080aca 100644 --- a/source/allocator_sba.c +++ b/source/allocator_sba.c @@ -187,14 +187,12 @@ static void s_sba_clean_up(struct small_block_allocator *sba) { void *page_addr = NULL; aws_array_list_get_at(&bin->active_pages, &page_addr, page_idx); struct page_header *page = page_addr; - (void)page; AWS_ASSERT(page->alloc_count == 0 && "Memory still allocated in aws_sba_allocator (bin)"); s_aligned_free(page); } if (bin->page_cursor) { void *page_addr = s_page_base(bin->page_cursor); struct page_header *page = page_addr; - (void)page; AWS_ASSERT(page->alloc_count == 0 && "Memory still allocated in aws_sba_allocator (page)"); s_aligned_free(page); } From d1ad6cc504107bc8e2720cb53755067e7c82d8b4 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Mon, 11 Oct 2021 16:50:23 -0700 Subject: [PATCH 33/34] Formatting. --- include/aws/common/string.h | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/include/aws/common/string.h b/include/aws/common/string.h index be7b51266..c73a24ad4 100644 --- a/include/aws/common/string.h +++ b/include/aws/common/string.h @@ -49,7 +49,7 @@ struct aws_string { #ifdef AWS_OS_WINDOWS struct aws_wstring { - struct aws_allocator* const allocator; + struct aws_allocator *const allocator; /* number of characters in the string not including the null terminator. */ const size_t len; /* give this a storage specifier for C++ purposes. It will likely be larger after init. */ @@ -71,7 +71,7 @@ AWS_EXTERN_C_BEGIN * * returns NULL on failure. */ -AWS_COMMON_API struct aws_wstring * aws_string_convert_to_wstring( +AWS_COMMON_API struct aws_wstring *aws_string_convert_to_wstring( struct aws_allocator *allocator, const struct aws_string *to_convert); @@ -90,7 +90,7 @@ AWS_COMMON_API struct aws_wstring *aws_string_convert_to_wchar_from_byte_cursor( * clean up str. */ AWS_COMMON_API -void aws_wstring_destroy(struct aws_wstring* str); +void aws_wstring_destroy(struct aws_wstring *str); /** * For windows only. Converts `to_convert` from a windows whcar format (UTF-16) to UTF-8. @@ -127,17 +127,23 @@ AWS_COMMON_API struct aws_string *aws_string_convert_from_wchar_c_str( /** * Create a new wide string from a byte cursor. This assumes that w_str_cur is already in utf-16. - * + * * returns NULL on failure. */ -AWS_COMMON_API struct aws_wstring* aws_wstring_new_from_cursor(struct aws_allocator *allocator, const struct aws_byte_cursor* w_str_cur); +AWS_COMMON_API struct aws_wstring *aws_wstring_new_from_cursor( + struct aws_allocator *allocator, + const struct aws_byte_cursor *w_str_cur); /** - * Create a new wide string from a utf-16 string enclosing array. The length field is in number of characters not counting the null terminator. - * + * Create a new wide string from a utf-16 string enclosing array. The length field is in number of characters not + * counting the null terminator. + * * returns NULL on failure. */ -AWS_COMMON_API struct aws_wstring* aws_wstring_new_from_array(struct aws_allocator *allocator, const wchar_t* w_str, size_t length); +AWS_COMMON_API struct aws_wstring *aws_wstring_new_from_array( + struct aws_allocator *allocator, + const wchar_t *w_str, + size_t length); /** * Returns a wchar_t * pointer for use with windows OS interop. @@ -152,12 +158,12 @@ AWS_COMMON_API size_t aws_wstring_num_chars(const struct aws_wstring *str); /** * Returns the length in bytes for the buffer. */ -AWS_COMMON_API size_t aws_wstring_size_bytes(const struct aws_wstring* str); +AWS_COMMON_API size_t aws_wstring_size_bytes(const struct aws_wstring *str); /** * Verifies that str is a valid string. Returns true if it's valid and false otherwise. */ -AWS_COMMON_API bool aws_wstring_is_valid(const struct aws_wstring* str); +AWS_COMMON_API bool aws_wstring_is_valid(const struct aws_wstring *str); #endif /* AWS_OS_WINDOWS */ From e6757bc467c2c9a006156eecfcf55682c7a00187 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Tue, 12 Oct 2021 15:26:45 -0700 Subject: [PATCH 34/34] Added iterator api for file-system API. --- include/aws/common/file.h | 33 +++++++++++++ source/file.c | 98 +++++++++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + tests/file_test.c | 79 +++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+) diff --git a/include/aws/common/file.h b/include/aws/common/file.h index 3f86971b2..4bbc1540d 100644 --- a/include/aws/common/file.h +++ b/include/aws/common/file.h @@ -18,6 +18,7 @@ #endif struct aws_string; +struct aws_directory_iterator; enum aws_file_type { AWS_FILE_TYPE_FILE = 1, @@ -114,6 +115,38 @@ AWS_COMMON_API int aws_directory_traverse( aws_on_directory_entry *on_entry, void *user_data); +/** + * Creates a read-only iterator of a directory starting at path. If path is invalid or there's any other error + * condition, NULL will be returned. Call aws_last_error() for the exact error in that case. + */ +AWS_COMMON_API struct aws_directory_iterator *aws_directory_entry_iterator_new( + struct aws_allocator *allocator, + const struct aws_string *path); + +/** + * Moves the iterator to the next entry. Returns AWS_OP_SUCCESS if another entry is available, or AWS_OP_ERR with + * AWS_ERROR_LIST_EMPTY as the value for aws_last_error() if no more entries are available. + */ +AWS_COMMON_API int aws_directory_entry_iterator_next(struct aws_directory_iterator *iterator); + +/** + * Moves the iterator to the previous entry. Returns AWS_OP_SUCCESS if another entry is available, or AWS_OP_ERR with + * AWS_ERROR_LIST_EMPTY as the value for aws_last_error() if no more entries are available. + */ +AWS_COMMON_API int aws_directory_entry_iterator_previous(struct aws_directory_iterator *iterator); + +/** + * Cleanup and deallocate iterator + */ +AWS_COMMON_API void aws_directory_entry_iterator_destroy(struct aws_directory_iterator *iterator); + +/** + * Gets the aws_directory_entry value for iterator at the current position. Returns NULL if the iterator contains no + * entries. + */ +AWS_COMMON_API const struct aws_directory_entry *aws_directory_entry_iterator_get_value( + const struct aws_directory_iterator *iterator); + /** * Returns true iff the character is a directory separator on ANY supported platform. */ diff --git a/source/file.c b/source/file.c index c334aa401..a64453fd2 100644 --- a/source/file.c +++ b/source/file.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -71,3 +72,100 @@ int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocat bool aws_is_any_directory_separator(char value) { return value == '\\' || value == '/'; } + +struct aws_directory_iterator { + struct aws_linked_list list_data; + struct aws_allocator *allocator; + struct aws_linked_list_node *current_node; +}; + +struct directory_entry_value { + struct aws_directory_entry entry; + struct aws_byte_buf path; + struct aws_byte_buf relative_path; + struct aws_linked_list_node node; +}; + +static bool s_directory_iterator_directory_entry(const struct aws_directory_entry *entry, void *user_data) { + struct aws_directory_iterator *iterator = user_data; + struct directory_entry_value *value = aws_mem_calloc(iterator->allocator, 1, sizeof(struct directory_entry_value)); + + value->entry = *entry; + aws_byte_buf_init_copy_from_cursor(&value->path, iterator->allocator, entry->path); + value->entry.path = aws_byte_cursor_from_buf(&value->path); + aws_byte_buf_init_copy_from_cursor(&value->relative_path, iterator->allocator, entry->relative_path); + value->entry.relative_path = aws_byte_cursor_from_buf(&value->relative_path); + aws_linked_list_push_back(&iterator->list_data, &value->node); + + return true; +} + +struct aws_directory_iterator *aws_directory_entry_iterator_new( + struct aws_allocator *allocator, + const struct aws_string *path) { + struct aws_directory_iterator *iterator = aws_mem_acquire(allocator, sizeof(struct aws_directory_iterator)); + iterator->allocator = allocator; + aws_linked_list_init(&iterator->list_data); + + /* the whole point of this iterator is to avoid recursion, so let's do that by passing recurse as false. */ + if (AWS_OP_SUCCESS == + aws_directory_traverse(allocator, path, false, s_directory_iterator_directory_entry, iterator)) { + if (!aws_linked_list_empty(&iterator->list_data)) { + iterator->current_node = aws_linked_list_front(&iterator->list_data); + } + return iterator; + } + + aws_mem_release(allocator, iterator); + return NULL; +} + +int aws_directory_entry_iterator_next(struct aws_directory_iterator *iterator) { + struct aws_linked_list_node *node = iterator->current_node; + + if (!node || node->next == aws_linked_list_end(&iterator->list_data)) { + return aws_raise_error(AWS_ERROR_LIST_EMPTY); + } + + iterator->current_node = aws_linked_list_next(node); + + return AWS_OP_SUCCESS; +} + +int aws_directory_entry_iterator_previous(struct aws_directory_iterator *iterator) { + struct aws_linked_list_node *node = iterator->current_node; + + if (!node || node == aws_linked_list_begin(&iterator->list_data)) { + return aws_raise_error(AWS_ERROR_LIST_EMPTY); + } + + iterator->current_node = aws_linked_list_prev(node); + + return AWS_OP_SUCCESS; +} + +void aws_directory_entry_iterator_destroy(struct aws_directory_iterator *iterator) { + while (!aws_linked_list_empty(&iterator->list_data)) { + struct aws_linked_list_node *node = aws_linked_list_pop_front(&iterator->list_data); + struct directory_entry_value *value = AWS_CONTAINER_OF(node, struct directory_entry_value, node); + + aws_byte_buf_clean_up(&value->path); + aws_byte_buf_clean_up(&value->relative_path); + + aws_mem_release(iterator->allocator, value); + } + + aws_mem_release(iterator->allocator, iterator); +} + +const struct aws_directory_entry *aws_directory_entry_iterator_get_value( + const struct aws_directory_iterator *iterator) { + struct aws_linked_list_node *node = iterator->current_node; + + if (!iterator->current_node) { + return NULL; + } + + struct directory_entry_value *value = AWS_CONTAINER_OF(node, struct directory_entry_value, node); + return &value->entry; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c4a3718cf..f9f53a75f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -459,6 +459,8 @@ add_test_case(aws_fopen_non_ascii_read_existing_file_test) add_test_case(aws_fopen_non_ascii_test) add_test_case(aws_fopen_ascii_test) add_test_case(directory_traversal_test) +add_test_case(directory_iteration_test) +add_test_case(directory_iteration_non_existent_directory_test) add_test_case(directory_traversal_stop_traversal) add_test_case(directory_traversal_on_file_test) add_test_case(directory_existence_test) diff --git a/tests/file_test.c b/tests/file_test.c index fb3a49e14..719c19bd0 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -155,6 +155,85 @@ static int s_directory_traversal_test_fn(struct aws_allocator *allocator, void * AWS_TEST_CASE(directory_traversal_test, s_directory_traversal_test_fn) +static int s_directory_iteration_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test"); + + struct aws_directory_iterator *iterator = aws_directory_entry_iterator_new(allocator, path); + ASSERT_NOT_NULL(iterator); + const struct aws_directory_entry *first_entry = aws_directory_entry_iterator_get_value(iterator); + ASSERT_NOT_NULL(first_entry); + + bool first_child_dir_found = false; + bool root_file_found = false; + + do { + const struct aws_directory_entry *entry = aws_directory_entry_iterator_get_value(iterator); + if (entry->file_type == AWS_FILE_TYPE_DIRECTORY) { + struct aws_byte_cursor first_child_dir_path_cur = aws_byte_cursor_from_c_str(s_first_child_dir_path); + ASSERT_BIN_ARRAYS_EQUALS( + first_child_dir_path_cur.ptr, + first_child_dir_path_cur.len, + entry->relative_path.ptr, + entry->relative_path.len); + first_child_dir_found = true; + + struct aws_string *next_path = aws_string_new_from_cursor(allocator, &entry->relative_path); + struct aws_directory_iterator *next_iter = aws_directory_entry_iterator_new(allocator, next_path); + aws_string_destroy(next_path); + ASSERT_NOT_NULL(next_iter); + + entry = aws_directory_entry_iterator_get_value(next_iter); + struct aws_byte_cursor first_child_file_path_cur = aws_byte_cursor_from_c_str(s_first_child_file_path); + ASSERT_BIN_ARRAYS_EQUALS( + first_child_file_path_cur.ptr, + first_child_file_path_cur.len, + entry->relative_path.ptr, + entry->relative_path.len); + ASSERT_INT_EQUALS(AWS_FILE_TYPE_FILE, entry->file_type); + + ASSERT_ERROR(AWS_ERROR_LIST_EMPTY, aws_directory_entry_iterator_next(next_iter)); + aws_directory_entry_iterator_destroy(next_iter); + } else { + struct aws_byte_cursor root_child_file_path_cur = aws_byte_cursor_from_c_str(s_root_child_path); + ASSERT_BIN_ARRAYS_EQUALS( + root_child_file_path_cur.ptr, + root_child_file_path_cur.len, + entry->relative_path.ptr, + entry->relative_path.len); + ASSERT_INT_EQUALS(AWS_FILE_TYPE_FILE, entry->file_type); + root_file_found = true; + } + } while (aws_directory_entry_iterator_next(iterator) == AWS_OP_SUCCESS); + + ASSERT_ERROR(AWS_ERROR_LIST_EMPTY, aws_directory_entry_iterator_next(iterator)); + ASSERT_SUCCESS(aws_directory_entry_iterator_previous(iterator)); + ASSERT_PTR_EQUALS(first_entry, aws_directory_entry_iterator_get_value(iterator)); + aws_directory_entry_iterator_destroy(iterator); + aws_string_destroy(path); + + ASSERT_TRUE(root_file_found); + ASSERT_TRUE(first_child_dir_found); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_iteration_test, s_directory_iteration_test_fn) + +static int s_directory_iteration_non_existent_directory_test_fn(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + struct aws_string *path = aws_string_new_from_c_str(allocator, "dir_traversal_test_non_existent"); + + struct aws_directory_iterator *iterator = aws_directory_entry_iterator_new(allocator, path); + ASSERT_NULL(iterator); + ASSERT_INT_EQUALS(aws_last_error(), AWS_ERROR_FILE_INVALID_PATH); + + aws_string_destroy(path); + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(directory_iteration_non_existent_directory_test, s_directory_iteration_non_existent_directory_test_fn) + struct directory_traversal_abort_test_data { int times_called; };