Imported file utils from IO (awslabs#841)

* Imported file utils from IO

* Fixed non-LFS seek

* Fixed aws_file_get_length on windows

* Windows needs Shlwapi

CMakeLists.txt

@@ -72,7 +72,7 @@ if (WIN32)
     endif ()
 
     list(APPEND PLATFORM_DEFINES WINDOWS_KERNEL_LIB=${WINDOWS_KERNEL_LIB})
-    list(APPEND PLATFORM_LIBS BCrypt ${WINDOWS_KERNEL_LIB} Ws2_32)
+    list(APPEND PLATFORM_LIBS BCrypt ${WINDOWS_KERNEL_LIB} Ws2_32 Shlwapi)
 else ()
     file(GLOB AWS_COMMON_OS_HEADERS
         "include/aws/common/posix/*"

include/aws/common/byte_buf.h

@@ -124,6 +124,15 @@ AWS_COMMON_API int aws_byte_buf_init_copy(
     struct aws_allocator *allocator,
     const struct aws_byte_buf *src);
 
+/**
+ * Reads 'filename' into 'out_buf'. If successful, 'out_buf' is allocated and filled with the data;
+ * It is your responsibility to call 'aws_byte_buf_clean_up()' on it. Otherwise, 'out_buf' remains
+ * unused. In the very unfortunate case where some API needs to treat out_buf as a c_string, a null terminator
+ * is appended, but is not included as part of the length field.
+ */
+AWS_COMMON_API
+int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocator *alloc, const char *filename);
+
 /**
  * Evaluates the set of properties that define the shape of all valid aws_byte_buf structures.
  * It is also a cheap check, in the sense it run in constant time (i.e., no loops or recursion).

include/aws/common/error.h

@@ -191,6 +191,7 @@ enum aws_common_error {
     AWS_ERROR_C_STRING_BUFFER_NOT_NULL_TERMINATED,
     AWS_ERROR_STRING_MATCH_NOT_FOUND,
     AWS_ERROR_DIVIDE_BY_ZERO,
+    AWS_ERROR_INVALID_FILE_HANDLE,
 
     AWS_ERROR_END_COMMON_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_COMMON_PACKAGE_ID)
 };

include/aws/common/file.h

@@ -6,6 +6,8 @@
  */
 #include <aws/common/common.h>
 
+#include <stdio.h>
+
 AWS_EXTERN_C_BEGIN
 
 /**
@@ -17,6 +19,52 @@ AWS_EXTERN_C_BEGIN
 AWS_COMMON_API
 FILE *aws_fopen(const char *file_path, const char *mode);
 
+/**
+ * Returns true iff the character is a directory separator on ANY supported platform.
+ */
+AWS_COMMON_API
+bool aws_is_any_directory_separator(char value);
+
+/**
+ * Returns the directory separator used by the local platform
+ */
+AWS_COMMON_API
+char aws_get_platform_directory_separator(void);
+
+/**
+ * Returns the current user's home directory.
+ */
+AWS_COMMON_API
+struct aws_string *aws_get_home_directory(struct aws_allocator *allocator);
+
+/**
+ * Returns true if a file or path exists, otherwise, false.
+ */
+AWS_COMMON_API
+bool aws_path_exists(const char *path);
+
+/*
+ * Wrapper for highest-resolution platform-dependent seek implementation.
+ * Maps to:
+ *
+ *   _fseeki64() on windows
+ *   fseeko() on linux
+ *
+ * whence can either be SEEK_SET or SEEK_END
+ */
+AWS_COMMON_API
+int aws_fseek(FILE *file, int64_t offset, int whence);
+
+/*
+ * Wrapper for os-specific file length query.  We can't use fseek(END, 0)
+ * because support for it is not technically required.
+ *
+ * Unix flavors call fstat, while Windows variants use GetFileSize on a
+ * HANDLE queried from the libc FILE pointer.
+ */
+AWS_COMMON_API
+int aws_file_get_length(FILE *file, int64_t *length);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_COMMON_FILE_H */

include/aws/common/logging.h

@@ -81,6 +81,7 @@ enum aws_common_log_subject {
     AWS_LS_COMMON_THREAD,
     AWS_LS_COMMON_MEMTRACE,
     AWS_LS_COMMON_XML_PARSER,
+    AWS_LS_COMMON_IO,
     AWS_LS_COMMON_BUS,
     AWS_LS_COMMON_TEST,
 

source/common.c

@@ -235,6 +235,9 @@ static struct aws_error_info errors[] = {
     AWS_DEFINE_ERROR_INFO_COMMON(
         AWS_ERROR_DIVIDE_BY_ZERO,
         "Attempt to divide a number by zero."),
+    AWS_DEFINE_ERROR_INFO_COMMON(
+        AWS_ERROR_INVALID_FILE_HANDLE,
+    "Invalid file handle"),
 };
 /* clang-format on */
 
@@ -255,6 +258,9 @@ static struct aws_log_subject_info s_common_log_subject_infos[] = {
     DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_THREAD, "thread", "Subject for logging thread related functions."),
     DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_MEMTRACE, "memtrace", "Output from the aws_mem_trace_dump function"),
     DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_XML_PARSER, "xml-parser", "Subject for xml parser specific logging."),
+    DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_IO, "common-io", "Common IO utilities"),
+    DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_BUS, "bus", "Message bus"),
+    DEFINE_LOG_SUBJECT_INFO(AWS_LS_COMMON_TEST, "test", "Unit/integration testing"),
 };
 
 static struct aws_log_subject_info_list s_common_log_subject_list = {

source/file.c

@@ -0,0 +1,61 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include <aws/common/byte_buf.h>
+#include <aws/common/file.h>
+#include <aws/common/logging.h>
+
+#include <errno.h>
+
+int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocator *alloc, const char *filename) {
+    AWS_ZERO_STRUCT(*out_buf);
+    FILE *fp = aws_fopen(filename, "rb");
+
+    if (fp) {
+        if (fseek(fp, 0L, SEEK_END)) {
+            AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno);
+            fclose(fp);
+            return aws_translate_and_raise_io_error(errno);
+        }
+
+        size_t allocation_size = (size_t)ftell(fp) + 1;
+        /* Tell the user that we allocate here and if success they're responsible for the free. */
+        if (aws_byte_buf_init(out_buf, alloc, allocation_size)) {
+            fclose(fp);
+            return AWS_OP_ERR;
+        }
+
+        /* Ensure compatibility with null-terminated APIs, but don't consider
+         * the null terminator part of the length of the payload */
+        out_buf->len = out_buf->capacity - 1;
+        out_buf->buffer[out_buf->len] = 0;
+
+        if (fseek(fp, 0L, SEEK_SET)) {
+            AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to seek file %s with errno %d", filename, errno);
+            aws_byte_buf_clean_up(out_buf);
+            fclose(fp);
+            return aws_translate_and_raise_io_error(errno);
+        }
+
+        size_t read = fread(out_buf->buffer, 1, out_buf->len, fp);
+        fclose(fp);
+        if (read < out_buf->len) {
+            AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to read file %s with errno %d", filename, errno);
+            aws_secure_zero(out_buf->buffer, out_buf->len);
+            aws_byte_buf_clean_up(out_buf);
+            return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE);
+        }
+
+        return AWS_OP_SUCCESS;
+    }
+
+    AWS_LOGF_ERROR(AWS_LS_COMMON_IO, "static: Failed to open file %s with errno %d", filename, errno);
+
+    return aws_translate_and_raise_io_error(errno);
+}
+
+bool aws_is_any_directory_separator(char value) {
+    return value == '\\' || value == '/';
+}

source/posix/file.c

@@ -3,9 +3,72 @@
  * SPDX-License-Identifier: Apache-2.0.
  */
 
+#include <aws/common/environment.h>
 #include <aws/common/file.h>
+#include <aws/common/string.h>
+#include <errno.h>
 #include <stdio.h>
+#include <sys/stat.h>
 
 FILE *aws_fopen(const char *file_path, const char *mode) {
     return fopen(file_path, mode);
 }
+
+char aws_get_platform_directory_separator(void) {
+    return '/';
+}
+
+AWS_STATIC_STRING_FROM_LITERAL(s_home_env_var, "HOME");
+
+struct aws_string *aws_get_home_directory(struct aws_allocator *allocator) {
+
+    /* ToDo: check getpwuid_r if environment check fails */
+    struct aws_string *home_env_var_value = NULL;
+    if (aws_get_environment_value(allocator, s_home_env_var, &home_env_var_value) == 0 && home_env_var_value != NULL) {
+        return home_env_var_value;
+    }
+
+    return NULL;
+}
+
+bool aws_path_exists(const char *path) {
+    struct stat buffer;
+    return stat(path, &buffer) == 0;
+}
+
+int aws_fseek(FILE *file, int64_t offset, int whence) {
+
+#ifdef AWS_HAVE_POSIX_LARGE_FILE_SUPPORT
+    int result = fseeko(file, offset, whence);
+#else
+    /* must use fseek(), which takes offset as a long */
+    if (offset < LONG_MIN || offset > LONG_MAX) {
+        return aws_raise_error(AWS_ERROR_INVALID_ARGUMENT);
+    }
+    int result = fseek(file, offset, whence);
+#endif /* AWS_HAVE_POSIX_LFS */
+
+    if (result != 0) {
+        return aws_translate_and_raise_io_error(errno);
+    }
+
+    return AWS_OP_SUCCESS;
+}
+
+int aws_file_get_length(FILE *file, int64_t *length) {
+
+    struct stat file_stats;
+
+    int fd = fileno(file);
+    if (fd == -1) {
+        return aws_raise_error(AWS_ERROR_INVALID_FILE_HANDLE);
+    }
+
+    if (fstat(fd, &file_stats)) {
+        return aws_translate_and_raise_io_error(errno);
+    }
+
+    *length = file_stats.st_size;
+
+    return AWS_OP_SUCCESS;
+}

source/windows/file.c

@@ -3,11 +3,17 @@
  * SPDX-License-Identifier: Apache-2.0.
  */
 
+#include <aws/common/environment.h>
 #include <aws/common/file.h>
+#include <aws/common/string.h>
+
 #include <errno.h>
 #include <stdio.h>
 #include <windows.h>
 
+#include <Shlwapi.h>
+#include <io.h>
+
 FILE *aws_fopen(const char *file_path, const char *mode) {
 
     wchar_t w_file_path[1000];
@@ -31,3 +37,106 @@ FILE *aws_fopen(const char *file_path, const char *mode) {
     }
     return file;
 }
+
+char aws_get_platform_directory_separator(void) {
+    return '\\';
+}
+
+AWS_STATIC_STRING_FROM_LITERAL(s_userprofile_env_var, "USERPROFILE");
+AWS_STATIC_STRING_FROM_LITERAL(s_homedrive_env_var, "HOMEDRIVE");
+AWS_STATIC_STRING_FROM_LITERAL(s_homepath_env_var, "HOMEPATH");
+
+AWS_STATIC_STRING_FROM_LITERAL(s_home_env_var, "HOME");
+
+struct aws_string *aws_get_home_directory(struct aws_allocator *allocator) {
+
+    /*
+     * 1. Check HOME
+     */
+    struct aws_string *home_env_var_value = NULL;
+    if (aws_get_environment_value(allocator, s_home_env_var, &home_env_var_value) == 0 && home_env_var_value != NULL) {
+        return home_env_var_value;
+    }
+
+    /*
+     * 2. (Windows) Check USERPROFILE
+     */
+    struct aws_string *userprofile_env_var_value = NULL;
+    if (aws_get_environment_value(allocator, s_userprofile_env_var, &userprofile_env_var_value) == 0 &&
+        userprofile_env_var_value != NULL) {
+        return userprofile_env_var_value;
+    }
+
+    /*
+     * 3. (Windows) Check HOMEDRIVE ++ HOMEPATH
+     */
+    struct aws_string *final_path = NULL;
+    struct aws_string *homedrive_env_var_value = NULL;
+    if (aws_get_environment_value(allocator, s_homedrive_env_var, &homedrive_env_var_value) == 0 &&
+        homedrive_env_var_value != NULL) {
+        struct aws_string *homepath_env_var_value = NULL;
+        if (aws_get_environment_value(allocator, s_homepath_env_var, &homepath_env_var_value) == 0 &&
+            homepath_env_var_value != NULL) {
+            struct aws_byte_buf concatenated_dir;
+            aws_byte_buf_init(
+                &concatenated_dir, allocator, homedrive_env_var_value->len + homepath_env_var_value->len + 1);
+
+            struct aws_byte_cursor drive_cursor = aws_byte_cursor_from_string(homedrive_env_var_value);
+            struct aws_byte_cursor path_cursor = aws_byte_cursor_from_string(homepath_env_var_value);
+
+            aws_byte_buf_append(&concatenated_dir, &drive_cursor);
+            aws_byte_buf_append(&concatenated_dir, &path_cursor);
+
+            final_path = aws_string_new_from_buf(allocator, &concatenated_dir);
+
+            aws_byte_buf_clean_up(&concatenated_dir);
+            aws_string_destroy(homepath_env_var_value);
+        }
+
+        aws_string_destroy(homedrive_env_var_value);
+    }
+
+    if (final_path != NULL) {
+        return final_path;
+    }
+
+    return NULL;
+}
+
+bool aws_path_exists(const char *path) {
+    return PathFileExistsA(path) == TRUE;
+}
+
+int aws_fseek(FILE *file, int64_t offset, int whence) {
+    if (_fseeki64(file, offset, whence)) {
+        return aws_translate_and_raise_io_error(errno);
+    }
+
+    return AWS_OP_SUCCESS;
+}
+
+int aws_file_get_length(FILE *file, int64_t *length) {
+    int fd = _fileno(file);
+    if (fd == -1) {
+        return aws_raise_error(AWS_ERROR_INVALID_FILE_HANDLE);
+    }
+
+    HANDLE os_file = (HANDLE)_get_osfhandle(fd);
+    if (os_file == INVALID_HANDLE_VALUE) {
+        return aws_translate_and_raise_io_error(errno);
+    }
+
+    LARGE_INTEGER os_size;
+    if (!GetFileSizeEx(os_file, &os_size)) {
+        return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE);
+    }
+
+    int64_t size = os_size.QuadPart;
+    if (size < 0) {
+        return aws_raise_error(AWS_ERROR_SYS_CALL_FAILURE);
+    }
+
+    *length = size;
+
+    return AWS_OP_SUCCESS;
+}

tests/CMakeLists.txt

@@ -465,6 +465,7 @@ add_test_case(test_scheduler_cancellation_for_pending_scheduled_task)
 add_test_case(aws_fopen_non_ascii_read_existing_file_test)
 add_test_case(aws_fopen_non_ascii_test)
 add_test_case(aws_fopen_ascii_test)
+add_test_case(test_home_directory_not_null)
 
 add_test_case(promise_test_wait_forever)
 add_test_case(promise_test_wait_for_a_bit)