From 43d1b922dbf6a6bc3fe22c49b9cfef4ffccdbd9d Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 14:39:21 -0700 Subject: [PATCH 01/18] Added program to dump system info we can determine for use with testing, added api for detecting virt vendor. --- CMakeLists.txt | 3 ++ bin/system_info/CMakeLists.txt | 25 ++++++++++ bin/system_info/print_system_info.c | 28 +++++++++++ include/aws/common/private/system_info_priv.h | 27 +++++++++++ include/aws/common/system_info.h | 18 ++++++++ source/linux/system_info.c | 17 +++++++ source/system_info.c | 46 +++++++++++++++++++ tests/system_info_tests.c | 4 +- 8 files changed, 166 insertions(+), 2 deletions(-) create mode 100644 bin/system_info/CMakeLists.txt create mode 100644 bin/system_info/print_system_info.c create mode 100644 include/aws/common/private/system_info_priv.h create mode 100644 source/linux/system_info.c create mode 100644 source/system_info.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 8375ea518..73993e47c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,8 @@ else () # Don't add the exact path to CoreFoundation as this would hardcode the SDK version list(APPEND PLATFORM_LIBS dl Threads::Threads "-framework CoreFoundation") list (APPEND AWS_COMMON_OS_SRC "source/darwin/*.c") # OS specific includes + list (APPEND AWS_COMMON_OS_SRC "source/linux/*.c") # just for the moment + elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") # Android does not link to libpthread nor librt, so this is fine list(APPEND PLATFORM_LIBS dl m Threads::Threads rt) list (APPEND AWS_COMMON_OS_SRC "source/linux/*.c") # OS specific includes @@ -298,6 +300,7 @@ configure_file(${CONFIG_HEADER_TEMPLATE} if (ALLOW_CROSS_COMPILED_TESTS OR NOT CMAKE_CROSSCOMPILING) if (BUILD_TESTING) add_subdirectory(tests) + add_subdirectory(bin/system_info) endif() endif() diff --git a/bin/system_info/CMakeLists.txt b/bin/system_info/CMakeLists.txt new file mode 100644 index 000000000..c72c811c4 --- /dev/null +++ b/bin/system_info/CMakeLists.txt @@ -0,0 +1,25 @@ +project(print-sys-info C) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib/cmake") + +file(GLOB SI_SRC + "*.c" + ) + +set(SI_PROJECT_NAME print-sys-info) +add_executable(${SI_PROJECT_NAME} ${SI_SRC}) +aws_set_common_properties(${SI_PROJECT_NAME}) + + +target_include_directories(${SI_PROJECT_NAME} PUBLIC + $ + $) + +target_link_libraries(${SI_PROJECT_NAME} PRIVATE aws-c-common) + +install(TARGETS ${SI_PROJECT_NAME} + EXPORT ${SI_PROJECT_NAME}-targets + COMPONENT Runtime + RUNTIME + DESTINATION bin + COMPONENT Runtime) \ No newline at end of file diff --git a/bin/system_info/print_system_info.c b/bin/system_info/print_system_info.c new file mode 100644 index 000000000..b3681e907 --- /dev/null +++ b/bin/system_info/print_system_info.c @@ -0,0 +1,28 @@ + + + +#include +#include + +int main(void) { + struct aws_allocator *allocator = aws_default_allocator(); + struct aws_system_environment *env = aws_system_environment_load(allocator); + + fprintf(stdout, "crt-detected env: {\n"); + + struct aws_byte_cursor virtualization_vendor = aws_system_environment_get_virtualization_vendor(env); + fprintf(stdout, " 'virtualization vendor': '" PRInSTR "'\n", (int)virtualization_vendor.len, virtualization_vendor.ptr); + fprintf(stdout, " 'number of processors': '%lu'\n", (unsigned long)aws_system_environment_get_processor_count(env)); + size_t numa_nodes = aws_system_environment_get_cpu_group_count(env); + + if (numa_nodes > 1) { + fprintf(stdout, " 'numa architecture': 'true'\n"); + fprintf(stdout, " 'number of numa nodes': '%lu'\n", (unsigned long)numa_nodes); + } else { + fprintf(stdout, " 'numa architecture': 'false'\n"); + } + + fprintf(stdout, "}\n"); + aws_system_environment_destroy(env); + return 0; +} diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h new file mode 100644 index 000000000..85dbdd902 --- /dev/null +++ b/include/aws/common/private/system_info_priv.h @@ -0,0 +1,27 @@ +#ifndef AWS_COMMON_SYSTEM_INFO_PRIV_H +#define AWS_COMMON_SYSTEM_INFO_PRIV_H +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include + +struct aws_system_environment { + struct aws_allocator *allocator; + struct aws_byte_buf virtualization_vendor; + enum aws_platform_os os; + size_t cpu_count; + size_t cpu_group_count; + void *additional_impl_data; +}; + +int aws_system_environment_load_platform_impl(struct aws_system_environment *env); +void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env); + + +void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_environment *env); + +#endif /* AWS_COMMON_SYSTEM_INFO_PRIV_H */ diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index fe7604120..1476e0f44 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -21,8 +21,26 @@ struct aws_cpu_info { bool suspected_hyper_thread; }; +struct aws_system_environment; + AWS_EXTERN_C_BEGIN +AWS_COMMON_API +struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator); + +AWS_COMMON_API +void aws_system_environment_destroy(struct aws_system_environment *env); + + +AWS_COMMON_API +struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct aws_system_environment *env); + +AWS_COMMON_API +size_t aws_system_environment_get_processor_count(struct aws_system_environment *env); + +AWS_COMMON_API +size_t aws_system_environment_get_cpu_group_count(struct aws_system_environment *env); + /* Returns the OS this was built under */ AWS_COMMON_API enum aws_platform_os aws_get_platform_build_os(void); diff --git a/source/linux/system_info.c b/source/linux/system_info.c new file mode 100644 index 000000000..a48bb15ad --- /dev/null +++ b/source/linux/system_info.c @@ -0,0 +1,17 @@ + +#include +#include + +int aws_system_environment_load_platform_impl(struct aws_system_environment *env) { + (void)env; + + return AWS_OP_SUCCESS; +} + +void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env) { + (void)env; +} + +void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_environment *env) { + aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor"); +} diff --git a/source/system_info.c b/source/system_info.c new file mode 100644 index 000000000..8ac988191 --- /dev/null +++ b/source/system_info.c @@ -0,0 +1,46 @@ +/** +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0. +*/ +#include + +struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator) { + struct aws_system_environment *env = aws_mem_calloc(allocator, 1, sizeof(struct aws_system_environment)); + env->allocator = allocator; + + if (aws_system_environment_load_platform_impl(env)) { + goto error; + } + + aws_system_environment_load_virtualization_vendor_impl(env); + env->os = aws_get_platform_build_os(); + env->cpu_count = aws_system_info_processor_count(); + env->cpu_group_count = aws_get_cpu_group_count(); + + return env; +error: + aws_mem_release(allocator, env); + + return NULL; +} + +void aws_system_environment_destroy(struct aws_system_environment *env) { + if (env) { + aws_system_environment_destroy_platform_impl(env); + aws_mem_release(env->allocator, env); + } + +} + +struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct aws_system_environment *env) { + return aws_byte_cursor_from_buf(&env->virtualization_vendor); +} + +size_t aws_system_environment_get_processor_count(struct aws_system_environment *env) { + return env->cpu_count; +} + +AWS_COMMON_API +size_t aws_system_environment_get_cpu_group_count(struct aws_system_environment *env) { + return env->cpu_group_count; +} diff --git a/tests/system_info_tests.c b/tests/system_info_tests.c index 8bd01ca4f..f0e0268b8 100644 --- a/tests/system_info_tests.c +++ b/tests/system_info_tests.c @@ -12,7 +12,7 @@ static int s_test_cpu_count_at_least_works_superficially_fn(struct aws_allocator (void)allocator; (void)ctx; - size_t processor_count = aws_system_info_processor_count(); + size_t processor_count = aws_system_environment_get_processor_count(); /* I think this is a fairly reasonable assumption given the circumstances * (you know this test is part of a program * that must be running on at least one core).... */ @@ -119,7 +119,7 @@ static int s_test_sanity_check_numa_discovery(struct aws_allocator *allocator, v (void)ctx; aws_common_library_init(allocator); - size_t processor_count = aws_system_info_processor_count(); + size_t processor_count = aws_system_environment_get_processor_count(); ASSERT_TRUE(processor_count > 0); uint16_t group_count = aws_get_cpu_group_count(); From b9f2de1ca996a0c1368165805dd89aa6a691ae32 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 14:50:38 -0700 Subject: [PATCH 02/18] Fix broken test from ide refactor. --- tests/system_info_tests.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system_info_tests.c b/tests/system_info_tests.c index f0e0268b8..8bd01ca4f 100644 --- a/tests/system_info_tests.c +++ b/tests/system_info_tests.c @@ -12,7 +12,7 @@ static int s_test_cpu_count_at_least_works_superficially_fn(struct aws_allocator (void)allocator; (void)ctx; - size_t processor_count = aws_system_environment_get_processor_count(); + size_t processor_count = aws_system_info_processor_count(); /* I think this is a fairly reasonable assumption given the circumstances * (you know this test is part of a program * that must be running on at least one core).... */ @@ -119,7 +119,7 @@ static int s_test_sanity_check_numa_discovery(struct aws_allocator *allocator, v (void)ctx; aws_common_library_init(allocator); - size_t processor_count = aws_system_environment_get_processor_count(); + size_t processor_count = aws_system_info_processor_count(); ASSERT_TRUE(processor_count > 0); uint16_t group_count = aws_get_cpu_group_count(); From c0ac6f572f36129f9f49e497c3a5758d888879be Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 15:02:29 -0700 Subject: [PATCH 03/18] add some logging. --- bin/system_info/print_system_info.c | 14 ++++++++++++++ source/system_info.c | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/bin/system_info/print_system_info.c b/bin/system_info/print_system_info.c index b3681e907..e01888627 100644 --- a/bin/system_info/print_system_info.c +++ b/bin/system_info/print_system_info.c @@ -3,9 +3,20 @@ #include #include +#include int main(void) { struct aws_allocator *allocator = aws_default_allocator(); + aws_common_library_init(allocator); + struct aws_logger_standard_options options = { + .file = stderr, + .level = AWS_LOG_LEVEL_TRACE, + }; + + struct aws_logger logger; + aws_logger_init_standard(&logger, allocator, &options); + aws_logger_set(&logger); + struct aws_system_environment *env = aws_system_environment_load(allocator); fprintf(stdout, "crt-detected env: {\n"); @@ -24,5 +35,8 @@ int main(void) { fprintf(stdout, "}\n"); aws_system_environment_destroy(env); + aws_logger_clean_up(&logger); + + aws_common_library_clean_up(); return 0; } diff --git a/source/system_info.c b/source/system_info.c index 8ac988191..02f5cdc5a 100644 --- a/source/system_info.c +++ b/source/system_info.c @@ -33,7 +33,8 @@ void aws_system_environment_destroy(struct aws_system_environment *env) { } struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct aws_system_environment *env) { - return aws_byte_cursor_from_buf(&env->virtualization_vendor); + struct aws_byte_cursor vendor_string = aws_byte_cursor_from_buf(&env->virtualization_vendor); + return aws_byte_cursor_trim_pred(&vendor_string, aws_char_is_space); } size_t aws_system_environment_get_processor_count(struct aws_system_environment *env) { From 239260edc2faf6868dff686ce1edfd25e1ff6217 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 15:12:49 -0700 Subject: [PATCH 04/18] Allow file reads to be less than their seek values. --- source/file.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/source/file.c b/source/file.c index 00723555f..264bbfe5f 100644 --- a/source/file.c +++ b/source/file.c @@ -73,14 +73,25 @@ int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocat size_t read = fread(out_buf->buffer, 1, out_buf->len, fp); if (read < out_buf->len) { int errno_value = ferror(fp) ? errno : 0; /* Always cache errno before potential side-effect */ - aws_translate_and_raise_io_error_or(errno_value, AWS_ERROR_FILE_READ_FAILURE); - AWS_LOGF_ERROR( - AWS_LS_COMMON_IO, - "static: Failed reading file:'%s' errno:%d aws-error:%s", - filename, - errno_value, - aws_error_name(aws_last_error())); - goto error; + if (errno_value != 0) { + aws_translate_and_raise_io_error_or(errno_value, AWS_ERROR_FILE_READ_FAILURE); + AWS_LOGF_ERROR( + AWS_LS_COMMON_IO, + "static: Failed reading file:'%s' errno:%d aws-error:%s", + filename, + errno_value, + aws_error_name(aws_last_error())); + goto error; + } else { + AWS_LOGF_WARN( + AWS_LS_COMMON_IO, + "static: reading file:'%s' reports longer length %d than actually reading from it: %d. This is not " + "necessarily an error as devices will usually report a page for length regardless of actual length.", + filename, + (int)read, + (int)out_buf->len); + out_buf->len = read; + } } fclose(fp); From 4dc4fc10e7a354027bac683a91df501eed486f4e Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 15:22:11 -0700 Subject: [PATCH 05/18] Add product name to the mix. --- bin/system_info/print_system_info.c | 2 ++ include/aws/common/private/system_info_priv.h | 2 ++ include/aws/common/system_info.h | 3 +++ source/linux/system_info.c | 4 ++++ source/system_info.c | 7 +++++++ 5 files changed, 18 insertions(+) diff --git a/bin/system_info/print_system_info.c b/bin/system_info/print_system_info.c index e01888627..722b7122c 100644 --- a/bin/system_info/print_system_info.c +++ b/bin/system_info/print_system_info.c @@ -23,6 +23,8 @@ int main(void) { struct aws_byte_cursor virtualization_vendor = aws_system_environment_get_virtualization_vendor(env); fprintf(stdout, " 'virtualization vendor': '" PRInSTR "'\n", (int)virtualization_vendor.len, virtualization_vendor.ptr); + struct aws_byte_cursor product_name = aws_system_environment_get_virtualization_product_name(env); + fprintf(stdout, " 'product name': '" PRInSTR "'\n", (int)product_name.len, product_name.ptr); fprintf(stdout, " 'number of processors': '%lu'\n", (unsigned long)aws_system_environment_get_processor_count(env)); size_t numa_nodes = aws_system_environment_get_cpu_group_count(env); diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h index 85dbdd902..dbfd2d782 100644 --- a/include/aws/common/private/system_info_priv.h +++ b/include/aws/common/private/system_info_priv.h @@ -12,6 +12,7 @@ struct aws_system_environment { struct aws_allocator *allocator; struct aws_byte_buf virtualization_vendor; + struct aws_byte_buf product_name; enum aws_platform_os os; size_t cpu_count; size_t cpu_group_count; @@ -23,5 +24,6 @@ void aws_system_environment_destroy_platform_impl(struct aws_system_environment void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_environment *env); +void aws_system_environment_load_virtualization_product_name_impl(struct aws_system_environment *env); #endif /* AWS_COMMON_SYSTEM_INFO_PRIV_H */ diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index 1476e0f44..65a93515c 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -35,6 +35,9 @@ void aws_system_environment_destroy(struct aws_system_environment *env); AWS_COMMON_API struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct aws_system_environment *env); +AWS_COMMON_API +struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(struct aws_system_environment *env); + AWS_COMMON_API size_t aws_system_environment_get_processor_count(struct aws_system_environment *env); diff --git a/source/linux/system_info.c b/source/linux/system_info.c index a48bb15ad..211e93f35 100644 --- a/source/linux/system_info.c +++ b/source/linux/system_info.c @@ -15,3 +15,7 @@ void aws_system_environment_destroy_platform_impl(struct aws_system_environment void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_environment *env) { aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor"); } + +void aws_system_environment_load_virtualization_product_name_impl(struct aws_system_environment *env) { + aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); +} diff --git a/source/system_info.c b/source/system_info.c index 02f5cdc5a..a2247a56f 100644 --- a/source/system_info.c +++ b/source/system_info.c @@ -13,6 +13,8 @@ struct aws_system_environment *aws_system_environment_load(struct aws_allocator } aws_system_environment_load_virtualization_vendor_impl(env); + aws_system_environment_load_virtualization_product_name_impl(env); + env->os = aws_get_platform_build_os(); env->cpu_count = aws_system_info_processor_count(); env->cpu_group_count = aws_get_cpu_group_count(); @@ -37,6 +39,11 @@ struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct a return aws_byte_cursor_trim_pred(&vendor_string, aws_char_is_space); } +struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(struct aws_system_environment *env) { + struct aws_byte_cursor product_name_str = aws_byte_cursor_from_buf(&env->product_name); + return aws_byte_cursor_trim_pred(&product_name_str, aws_char_is_space); +} + size_t aws_system_environment_get_processor_count(struct aws_system_environment *env) { return env->cpu_count; } From be2849ce9ae551ee50c245d50af887df5182cf63 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 15:24:10 -0700 Subject: [PATCH 06/18] Do not overwirte yourself this time. --- source/linux/system_info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/linux/system_info.c b/source/linux/system_info.c index 211e93f35..6879bff43 100644 --- a/source/linux/system_info.c +++ b/source/linux/system_info.c @@ -17,5 +17,5 @@ void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_en } void aws_system_environment_load_virtualization_product_name_impl(struct aws_system_environment *env) { - aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); + aws_byte_buf_init_from_file(&env->product_name, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); } From 6a7c30c38659f9ee18c635568a283352c51953a6 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 15:26:26 -0700 Subject: [PATCH 07/18] fixed json output formatting. --- bin/system_info/print_system_info.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/system_info/print_system_info.c b/bin/system_info/print_system_info.c index 722b7122c..5c86b8187 100644 --- a/bin/system_info/print_system_info.c +++ b/bin/system_info/print_system_info.c @@ -22,14 +22,14 @@ int main(void) { fprintf(stdout, "crt-detected env: {\n"); struct aws_byte_cursor virtualization_vendor = aws_system_environment_get_virtualization_vendor(env); - fprintf(stdout, " 'virtualization vendor': '" PRInSTR "'\n", (int)virtualization_vendor.len, virtualization_vendor.ptr); + fprintf(stdout, " 'virtualization vendor': '" PRInSTR "',\n", (int)virtualization_vendor.len, virtualization_vendor.ptr); struct aws_byte_cursor product_name = aws_system_environment_get_virtualization_product_name(env); - fprintf(stdout, " 'product name': '" PRInSTR "'\n", (int)product_name.len, product_name.ptr); - fprintf(stdout, " 'number of processors': '%lu'\n", (unsigned long)aws_system_environment_get_processor_count(env)); + fprintf(stdout, " 'product name': '" PRInSTR "',\n", (int)product_name.len, product_name.ptr); + fprintf(stdout, " 'number of processors': '%lu',\n", (unsigned long)aws_system_environment_get_processor_count(env)); size_t numa_nodes = aws_system_environment_get_cpu_group_count(env); if (numa_nodes > 1) { - fprintf(stdout, " 'numa architecture': 'true'\n"); + fprintf(stdout, " 'numa architecture': 'true',\n"); fprintf(stdout, " 'number of numa nodes': '%lu'\n", (unsigned long)numa_nodes); } else { fprintf(stdout, " 'numa architecture': 'false'\n"); From 5c1917e08d87cd11a1c0c86672bb9ed3e9b0a42d Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Thu, 5 Oct 2023 16:30:00 -0700 Subject: [PATCH 08/18] dont need all those functions for an internal api. --- include/aws/common/private/system_info_priv.h | 4 ---- source/linux/system_info.c | 13 ++++--------- source/system_info.c | 3 --- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h index dbfd2d782..46760cea4 100644 --- a/include/aws/common/private/system_info_priv.h +++ b/include/aws/common/private/system_info_priv.h @@ -22,8 +22,4 @@ struct aws_system_environment { int aws_system_environment_load_platform_impl(struct aws_system_environment *env); void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env); - -void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_environment *env); -void aws_system_environment_load_virtualization_product_name_impl(struct aws_system_environment *env); - #endif /* AWS_COMMON_SYSTEM_INFO_PRIV_H */ diff --git a/source/linux/system_info.c b/source/linux/system_info.c index 6879bff43..c39545c23 100644 --- a/source/linux/system_info.c +++ b/source/linux/system_info.c @@ -3,19 +3,14 @@ #include int aws_system_environment_load_platform_impl(struct aws_system_environment *env) { - (void)env; + aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor"); + aws_byte_buf_init_from_file(&env->product_name, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); return AWS_OP_SUCCESS; } void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env) { - (void)env; + aws_byte_buf_clean_up(&env->virtualization_vendor); + aws_byte_buf_clean_up(&env->product_name); } -void aws_system_environment_load_virtualization_vendor_impl(struct aws_system_environment *env) { - aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor"); -} - -void aws_system_environment_load_virtualization_product_name_impl(struct aws_system_environment *env) { - aws_byte_buf_init_from_file(&env->product_name, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); -} diff --git a/source/system_info.c b/source/system_info.c index a2247a56f..e4b08a77e 100644 --- a/source/system_info.c +++ b/source/system_info.c @@ -12,9 +12,6 @@ struct aws_system_environment *aws_system_environment_load(struct aws_allocator goto error; } - aws_system_environment_load_virtualization_vendor_impl(env); - aws_system_environment_load_virtualization_product_name_impl(env); - env->os = aws_get_platform_build_os(); env->cpu_count = aws_system_info_processor_count(); env->cpu_group_count = aws_get_cpu_group_count(); From bc220a4acd398acc8b02e70ea8916083c8870005 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Fri, 6 Oct 2023 12:52:49 -0700 Subject: [PATCH 09/18] Make it const. --- include/aws/common/system_info.h | 4 ++-- source/system_info.c | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index 65a93515c..3a111092e 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -33,10 +33,10 @@ void aws_system_environment_destroy(struct aws_system_environment *env); AWS_COMMON_API -struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct aws_system_environment *env); +struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(const struct aws_system_environment *env); AWS_COMMON_API -struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(struct aws_system_environment *env); +struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(const struct aws_system_environment *env); AWS_COMMON_API size_t aws_system_environment_get_processor_count(struct aws_system_environment *env); diff --git a/source/system_info.c b/source/system_info.c index e4b08a77e..5615463cb 100644 --- a/source/system_info.c +++ b/source/system_info.c @@ -28,15 +28,14 @@ void aws_system_environment_destroy(struct aws_system_environment *env) { aws_system_environment_destroy_platform_impl(env); aws_mem_release(env->allocator, env); } - } -struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(struct aws_system_environment *env) { +struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(const struct aws_system_environment *env) { struct aws_byte_cursor vendor_string = aws_byte_cursor_from_buf(&env->virtualization_vendor); return aws_byte_cursor_trim_pred(&vendor_string, aws_char_is_space); } -struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(struct aws_system_environment *env) { +struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(const struct aws_system_environment *env) { struct aws_byte_cursor product_name_str = aws_byte_cursor_from_buf(&env->product_name); return aws_byte_cursor_trim_pred(&product_name_str, aws_char_is_space); } From 70d38a607bd1d2fa6511577bc3a03238ebe0eab8 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 13:11:54 -0700 Subject: [PATCH 10/18] Added proper platform fallback, a sanity test, and some documentation. --- CMakeLists.txt | 9 +++++-- bin/system_info/CMakeLists.txt | 7 ----- include/aws/common/private/system_info_priv.h | 13 ++++++++- include/aws/common/system_info.h | 21 ++++++++++++++- source/linux/system_info.c | 10 ++++--- source/platform_fallback_stubs/system_info.c | 21 +++++++++++++++ source/system_info.c | 27 ++++++++++++++++--- tests/CMakeLists.txt | 1 + tests/system_info_tests.c | 19 +++++++++++++ 9 files changed, 110 insertions(+), 18 deletions(-) create mode 100644 source/platform_fallback_stubs/system_info.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 73993e47c..b8b827c90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,21 +108,26 @@ else () # Don't add the exact path to CoreFoundation as this would hardcode the SDK version list(APPEND PLATFORM_LIBS dl Threads::Threads "-framework CoreFoundation") list (APPEND AWS_COMMON_OS_SRC "source/darwin/*.c") # OS specific includes - list (APPEND AWS_COMMON_OS_SRC "source/linux/*.c") # just for the moment - + list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c") elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Linux") # Android does not link to libpthread nor librt, so this is fine list(APPEND PLATFORM_LIBS dl m Threads::Threads rt) list (APPEND AWS_COMMON_OS_SRC "source/linux/*.c") # OS specific includes elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") list(APPEND PLATFORM_LIBS dl m thr execinfo) + list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c") elseif(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") list(APPEND PLATFORM_LIBS dl m Threads::Threads execinfo) + list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c") elseif(CMAKE_SYSTEM_NAME STREQUAL "OpenBSD") list(APPEND PLATFORM_LIBS m Threads::Threads execinfo) + list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c") elseif(CMAKE_SYSTEM_NAME STREQUAL "Android") list(APPEND PLATFORM_LIBS log) file(GLOB ANDROID_SRC "source/android/*.c") list(APPEND AWS_COMMON_OS_SRC "${ANDROID_SRC}") + list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c") + else() + list (APPEND AWS_COMMON_OS_SRC "source/platform_fallback_stubs/system_info.c") endif() endif() diff --git a/bin/system_info/CMakeLists.txt b/bin/system_info/CMakeLists.txt index c72c811c4..cb4ca0081 100644 --- a/bin/system_info/CMakeLists.txt +++ b/bin/system_info/CMakeLists.txt @@ -16,10 +16,3 @@ target_include_directories(${SI_PROJECT_NAME} PUBLIC $) target_link_libraries(${SI_PROJECT_NAME} PRIVATE aws-c-common) - -install(TARGETS ${SI_PROJECT_NAME} - EXPORT ${SI_PROJECT_NAME}-targets - COMPONENT Runtime - RUNTIME - DESTINATION bin - COMPONENT Runtime) \ No newline at end of file diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h index 46760cea4..81191dc41 100644 --- a/include/aws/common/private/system_info_priv.h +++ b/include/aws/common/private/system_info_priv.h @@ -5,9 +5,9 @@ * SPDX-License-Identifier: Apache-2.0. */ -#include #include #include +#include struct aws_system_environment { struct aws_allocator *allocator; @@ -19,7 +19,18 @@ struct aws_system_environment { void *additional_impl_data; }; +/** + * For internal implementors. Fill in info in env that you're able to grab, such as dmi info, os version strings etc... + * in here. The default just returns AWS_OP_SUCCESS. This is currently only implemented for linux. + * + * Returns AWS_OP_ERR if the implementation wasn't able to fill in required information for the platform. + */ int aws_system_environment_load_platform_impl(struct aws_system_environment *env); + +/** + * For internal implementors. Cleans up anything allocated in aws_system_environment_load_platform_impl, + * but does not release the memory for env. + */ void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env); #endif /* AWS_COMMON_SYSTEM_INFO_PRIV_H */ diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index 3a111092e..a9edfc488 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -25,22 +25,41 @@ struct aws_system_environment; AWS_EXTERN_C_BEGIN +/** + * Allocates and initializes information about the system the current process is executing on. + * If successful returns an instance of aws_system_environment. If it fails, it will return NULL. + */ AWS_COMMON_API struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator); AWS_COMMON_API void aws_system_environment_destroy(struct aws_system_environment *env); - +/** + * Returns the virtualization vendor for the specified compute environment, e.g. "Xen, Amazon EC2, etc..." + * + * The return value may be empty and in that case no vendor was detected. + */ AWS_COMMON_API struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(const struct aws_system_environment *env); +/** + * Returns the product name for the specified compute environment. For example, the Amazon EC2 Instance type. + * + * The return value may be empty and in that case no vendor was detected. + */ AWS_COMMON_API struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(const struct aws_system_environment *env); +/** + * Returns the number of processors for the specified compute environment. + */ AWS_COMMON_API size_t aws_system_environment_get_processor_count(struct aws_system_environment *env); +/** + * Returns the number of separate cpu groupings (multi-socket configurations or NUMA). + */ AWS_COMMON_API size_t aws_system_environment_get_cpu_group_count(struct aws_system_environment *env); diff --git a/source/linux/system_info.c b/source/linux/system_info.c index c39545c23..bdfd2a99b 100644 --- a/source/linux/system_info.c +++ b/source/linux/system_info.c @@ -1,9 +1,14 @@ - -#include +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ #include +#include int aws_system_environment_load_platform_impl(struct aws_system_environment *env) { aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor"); + /* whether this one works depends on if this is a sysfs filesystem. If it fails, it will just be empty + * and these APIs are a best effort at the moment. We can add fallbacks as the loaders get more complicated. */ aws_byte_buf_init_from_file(&env->product_name, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); return AWS_OP_SUCCESS; @@ -13,4 +18,3 @@ void aws_system_environment_destroy_platform_impl(struct aws_system_environment aws_byte_buf_clean_up(&env->virtualization_vendor); aws_byte_buf_clean_up(&env->product_name); } - diff --git a/source/platform_fallback_stubs/system_info.c b/source/platform_fallback_stubs/system_info.c new file mode 100644 index 000000000..2b81469a8 --- /dev/null +++ b/source/platform_fallback_stubs/system_info.c @@ -0,0 +1,21 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include + +#include + +int aws_system_environment_load_platform_impl(struct aws_system_environment *env) { + (void)env; + AWS_LOGF_DEBUG( + AWS_LS_COMMON_GENERAL, + "id=%p: platform specific environment loading is not implemented for this platform.", + (void *)env); + + return AWS_OP_SUCCESS; +} + +void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env) { + (void)env; +} diff --git a/source/system_info.c b/source/system_info.c index 5615463cb..dcc4b57a8 100644 --- a/source/system_info.c +++ b/source/system_info.c @@ -1,17 +1,35 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0. -*/ + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ #include +#include + struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator) { struct aws_system_environment *env = aws_mem_calloc(allocator, 1, sizeof(struct aws_system_environment)); env->allocator = allocator; if (aws_system_environment_load_platform_impl(env)) { + AWS_LOGF_ERROR( + AWS_LS_COMMON_GENERAL, + "id=%p: failed to load system environment with error %s.", + (void *)env, + aws_error_debug_str(aws_last_error())); goto error; } + AWS_LOGF_TRACE( + AWS_LS_COMMON_GENERAL, + "id=%p: virtualization vendor detected as \"" PRInSTR "\"", + (void *)env, + AWS_BYTE_CURSOR_PRI(aws_system_environment_get_virtualization_vendor(env))); + AWS_LOGF_TRACE( + AWS_LS_COMMON_GENERAL, + "id=%p: virtualization product name detected as \"" PRInSTR " \"", + (void *)env, + AWS_BYTE_CURSOR_PRI(aws_system_environment_get_virtualization_vendor(env))); + env->os = aws_get_platform_build_os(); env->cpu_count = aws_system_info_processor_count(); env->cpu_group_count = aws_get_cpu_group_count(); @@ -35,7 +53,8 @@ struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(const st return aws_byte_cursor_trim_pred(&vendor_string, aws_char_is_space); } -struct aws_byte_cursor aws_system_environment_get_virtualization_product_name(const struct aws_system_environment *env) { +struct aws_byte_cursor aws_system_environment_get_virtualization_product_name( + const struct aws_system_environment *env) { struct aws_byte_cursor product_name_str = aws_byte_cursor_from_buf(&env->product_name); return aws_byte_cursor_trim_pred(&product_name_str, aws_char_is_space); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f2b580dfa..9a186f750 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -304,6 +304,7 @@ add_test_case(test_cpu_count_at_least_works_superficially) add_test_case(test_stack_trace_decoding) add_test_case(test_platform_build_os) add_test_case(test_sanity_check_numa_discovery) +add_test_case(test_sanity_check_environment_loader) add_test_case(test_realloc_fallback) add_test_case(test_realloc_passthrough) diff --git a/tests/system_info_tests.c b/tests/system_info_tests.c index 8bd01ca4f..571b6cbc5 100644 --- a/tests/system_info_tests.c +++ b/tests/system_info_tests.c @@ -166,3 +166,22 @@ static int s_test_sanity_check_numa_discovery(struct aws_allocator *allocator, v } AWS_TEST_CASE(test_sanity_check_numa_discovery, s_test_sanity_check_numa_discovery) + +static int s_test_sanity_check_environment_loader(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + aws_common_library_init(allocator); + struct aws_system_environment *env = aws_system_environment_load(allocator); + ASSERT_NOT_NULL(env); + struct aws_byte_cursor virt_vendor = aws_system_environment_get_virtualization_vendor(env); + ASSERT_TRUE(aws_byte_cursor_is_valid(&virt_vendor)); + struct aws_byte_cursor virt_product = aws_system_environment_get_virtualization_product_name(env); + ASSERT_TRUE(aws_byte_cursor_is_valid(&virt_product)); + + aws_system_environment_destroy(env); + + aws_common_library_clean_up(); + return 0; +} + +AWS_TEST_CASE(test_sanity_check_environment_loader, s_test_sanity_check_environment_loader) From cab0f56a78b976912a45f7b0376adee0ed2f99e5 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 13:17:07 -0700 Subject: [PATCH 11/18] fix build and include guard. --- include/aws/common/private/system_info_priv.h | 6 +++--- include/aws/common/system_info.h | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h index 81191dc41..d6373a8c0 100644 --- a/include/aws/common/private/system_info_priv.h +++ b/include/aws/common/private/system_info_priv.h @@ -1,5 +1,5 @@ -#ifndef AWS_COMMON_SYSTEM_INFO_PRIV_H -#define AWS_COMMON_SYSTEM_INFO_PRIV_H +#ifndef AWS_COMMON_PRIVATE_SYSTEM_INFO_PRIV_H +#define AWS_COMMON_PRIVATE_SYSTEM_INFO_PRIV_H /** * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. @@ -33,4 +33,4 @@ int aws_system_environment_load_platform_impl(struct aws_system_environment *env */ void aws_system_environment_destroy_platform_impl(struct aws_system_environment *env); -#endif /* AWS_COMMON_SYSTEM_INFO_PRIV_H */ +#endif // AWS_COMMON_PRIVATE_SYSTEM_INFO_PRIV_H diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index a9edfc488..cd3bb944a 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -7,6 +7,7 @@ */ #include +#include AWS_PUSH_SANE_WARNING_LEVEL From cba1cbc0537b1963882d9f4624858094968307e3 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 13:18:59 -0700 Subject: [PATCH 12/18] Fix include order. --- include/aws/common/system_info.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index cd3bb944a..5e840ad35 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -6,8 +6,8 @@ * SPDX-License-Identifier: Apache-2.0. */ -#include #include +#include AWS_PUSH_SANE_WARNING_LEVEL From 74fd1fa312acd12bb40d62c71974edd7a44e4ba4 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 13:23:08 -0700 Subject: [PATCH 13/18] whoops! sorry windows. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8b827c90..0ff1d9e7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,7 @@ if (WIN32) file(GLOB AWS_COMMON_OS_SRC "source/windows/*.c" + "source/platform_fallback_stubs/system_info.c" ) if (MSVC) From f2adc4943de5535475f9afcdc6825554795ce537 Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 14:54:20 -0700 Subject: [PATCH 14/18] Update include/aws/common/private/system_info_priv.h trivial PR merge. Co-authored-by: Michael Graeb --- include/aws/common/private/system_info_priv.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h index d6373a8c0..608bd056c 100644 --- a/include/aws/common/private/system_info_priv.h +++ b/include/aws/common/private/system_info_priv.h @@ -16,7 +16,7 @@ struct aws_system_environment { enum aws_platform_os os; size_t cpu_count; size_t cpu_group_count; - void *additional_impl_data; + void *impl; }; /** From 14f0e748be938fe5f47469ea77cc5e4b20c121aa Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 17:49:21 -0700 Subject: [PATCH 15/18] Addressed PR feedback, rewrote the file to buffer API and added a test (there wasn't one). --- bin/system_info/print_system_info.c | 14 +++-- include/aws/common/private/system_info_priv.h | 3 +- include/aws/common/system_info.h | 7 ++- source/file.c | 56 ++++++++++--------- source/system_info.c | 27 ++++++--- tests/CMakeLists.txt | 1 + tests/file_test.c | 28 ++++++++++ tests/system_info_tests.c | 2 +- 8 files changed, 95 insertions(+), 43 deletions(-) diff --git a/bin/system_info/print_system_info.c b/bin/system_info/print_system_info.c index 5c86b8187..c29877086 100644 --- a/bin/system_info/print_system_info.c +++ b/bin/system_info/print_system_info.c @@ -1,9 +1,8 @@ - -#include #include #include +#include int main(void) { struct aws_allocator *allocator = aws_default_allocator(); @@ -22,10 +21,15 @@ int main(void) { fprintf(stdout, "crt-detected env: {\n"); struct aws_byte_cursor virtualization_vendor = aws_system_environment_get_virtualization_vendor(env); - fprintf(stdout, " 'virtualization vendor': '" PRInSTR "',\n", (int)virtualization_vendor.len, virtualization_vendor.ptr); + fprintf( + stdout, + " 'virtualization vendor': '" PRInSTR "',\n", + (int)virtualization_vendor.len, + virtualization_vendor.ptr); struct aws_byte_cursor product_name = aws_system_environment_get_virtualization_product_name(env); fprintf(stdout, " 'product name': '" PRInSTR "',\n", (int)product_name.len, product_name.ptr); - fprintf(stdout, " 'number of processors': '%lu',\n", (unsigned long)aws_system_environment_get_processor_count(env)); + fprintf( + stdout, " 'number of processors': '%lu',\n", (unsigned long)aws_system_environment_get_processor_count(env)); size_t numa_nodes = aws_system_environment_get_cpu_group_count(env); if (numa_nodes > 1) { @@ -36,7 +40,7 @@ int main(void) { } fprintf(stdout, "}\n"); - aws_system_environment_destroy(env); + aws_system_environment_release(env); aws_logger_clean_up(&logger); aws_common_library_clean_up(); diff --git a/include/aws/common/private/system_info_priv.h b/include/aws/common/private/system_info_priv.h index 608bd056c..27b1d4ad1 100644 --- a/include/aws/common/private/system_info_priv.h +++ b/include/aws/common/private/system_info_priv.h @@ -4,13 +4,14 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ - #include +#include #include #include struct aws_system_environment { struct aws_allocator *allocator; + struct aws_ref_count ref_count; struct aws_byte_buf virtualization_vendor; struct aws_byte_buf product_name; enum aws_platform_os os; diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index 5e840ad35..be4554fd4 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -34,7 +34,10 @@ AWS_COMMON_API struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator); AWS_COMMON_API -void aws_system_environment_destroy(struct aws_system_environment *env); +struct aws_system_environment *aws_system_environment_acquire(struct aws_system_environment *env); + +AWS_COMMON_API +void aws_system_environment_release(struct aws_system_environment *env); /** * Returns the virtualization vendor for the specified compute environment, e.g. "Xen, Amazon EC2, etc..." @@ -62,7 +65,7 @@ size_t aws_system_environment_get_processor_count(struct aws_system_environment * Returns the number of separate cpu groupings (multi-socket configurations or NUMA). */ AWS_COMMON_API -size_t aws_system_environment_get_cpu_group_count(struct aws_system_environment *env); +size_t aws_system_environment_get_cpu_group_count(const struct aws_system_environment *env); /* Returns the OS this was built under */ AWS_COMMON_API diff --git a/source/file.c b/source/file.c index 264bbfe5f..e29458146 100644 --- a/source/file.c +++ b/source/file.c @@ -62,39 +62,43 @@ int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocat goto error; } + /* + * This number is usually correct, but in cases of device files that don't correspond to storage on disk, + * it may just be the size of a page. Go ahead and use it as a good hint of how much to allocate initially, + * but otherwise don't rely on it. + */ size_t allocation_size = (size_t)len64 + 1; aws_byte_buf_init(out_buf, alloc, allocation_size); - /* Ensure compatibility with null-terminated APIs, but don't consider - * the null terminator part of the length of the payload */ - out_buf->len = out_buf->capacity - 1; - out_buf->buffer[out_buf->len] = 0; - - size_t read = fread(out_buf->buffer, 1, out_buf->len, fp); - if (read < out_buf->len) { - int errno_value = ferror(fp) ? errno : 0; /* Always cache errno before potential side-effect */ - if (errno_value != 0) { - aws_translate_and_raise_io_error_or(errno_value, AWS_ERROR_FILE_READ_FAILURE); - AWS_LOGF_ERROR( - AWS_LS_COMMON_IO, - "static: Failed reading file:'%s' errno:%d aws-error:%s", - filename, - errno_value, - aws_error_name(aws_last_error())); - goto error; - } else { - AWS_LOGF_WARN( - AWS_LS_COMMON_IO, - "static: reading file:'%s' reports longer length %d than actually reading from it: %d. This is not " - "necessarily an error as devices will usually report a page for length regardless of actual length.", - filename, - (int)read, - (int)out_buf->len); - out_buf->len = read; + size_t read = -1; + size_t total_read = 0; + do { + if (total_read == out_buf->capacity) { + /* just add allocation size space to read some more. It's not perfect but it's plenty good. */ + aws_byte_buf_reserve_relative(out_buf, allocation_size); } + read = fread(out_buf->buffer + out_buf->len, 1, out_buf->capacity - out_buf->len, fp); + out_buf->len += read; + total_read += read; + } while (read > 0); + + int errno_value = ferror(fp) ? errno : 0; /* Always cache errno before potential side-effect */ + if (errno_value != 0) { + aws_translate_and_raise_io_error_or(errno_value, AWS_ERROR_FILE_READ_FAILURE); + AWS_LOGF_ERROR( + AWS_LS_COMMON_IO, + "static: Failed reading file:'%s' errno:%d aws-error:%s", + filename, + errno_value, + aws_error_name(aws_last_error())); + goto error; } fclose(fp); + /* write the NULL terminator out. */ + aws_byte_buf_write_u8(out_buf, 0x00); + /* we wrote the NULL terminator, but don't include it in the length. */ + out_buf->len -= 1; return AWS_OP_SUCCESS; error: diff --git a/source/system_info.c b/source/system_info.c index dcc4b57a8..4b721f63a 100644 --- a/source/system_info.c +++ b/source/system_info.c @@ -6,9 +6,19 @@ #include +void s_destroy_env(void *arg) { + struct aws_system_environment *env = arg; + + if (env) { + aws_system_environment_destroy_platform_impl(env); + aws_mem_release(env->allocator, env); + } +} + struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator) { struct aws_system_environment *env = aws_mem_calloc(allocator, 1, sizeof(struct aws_system_environment)); env->allocator = allocator; + aws_ref_count_init(&env->ref_count, env, s_destroy_env); if (aws_system_environment_load_platform_impl(env)) { AWS_LOGF_ERROR( @@ -36,16 +46,17 @@ struct aws_system_environment *aws_system_environment_load(struct aws_allocator return env; error: - aws_mem_release(allocator, env); - + s_destroy_env(env); return NULL; } -void aws_system_environment_destroy(struct aws_system_environment *env) { - if (env) { - aws_system_environment_destroy_platform_impl(env); - aws_mem_release(env->allocator, env); - } +struct aws_system_environment *aws_system_environment_acquire(struct aws_system_environment *env) { + aws_ref_count_acquire(&env->ref_count); + return env; +} + +void aws_system_environment_release(struct aws_system_environment *env) { + aws_ref_count_release(&env->ref_count); } struct aws_byte_cursor aws_system_environment_get_virtualization_vendor(const struct aws_system_environment *env) { @@ -64,6 +75,6 @@ size_t aws_system_environment_get_processor_count(struct aws_system_environment } AWS_COMMON_API -size_t aws_system_environment_get_cpu_group_count(struct aws_system_environment *env) { +size_t aws_system_environment_get_cpu_group_count(const struct aws_system_environment *env) { return env->cpu_group_count; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9a186f750..8d254bfc2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -481,6 +481,7 @@ add_test_case(directory_move_src_non_existent_test) add_test_case(test_home_directory_not_null) add_test_case(test_normalize_posix_directory_separator) add_test_case(test_normalize_windows_directory_separator) +add_test_case(test_byte_buf_file_read) add_test_case(promise_test_wait_forever) add_test_case(promise_test_wait_for_a_bit) diff --git a/tests/file_test.c b/tests/file_test.c index 6eedd264e..6205a6f89 100644 --- a/tests/file_test.c +++ b/tests/file_test.c @@ -439,3 +439,31 @@ static int s_test_normalize_windows_directory_separator(struct aws_allocator *al } AWS_TEST_CASE(test_normalize_windows_directory_separator, s_test_normalize_windows_directory_separator); + +static int s_test_byte_buf_file_read(struct aws_allocator *allocator, void *ctx) { + (void)ctx; + + const char *test_string = "This is a message that's going to test a read loop."; + struct aws_byte_buf test_buf = aws_byte_buf_from_c_str(test_string); + struct aws_string *test_file = aws_string_new_from_c_str(allocator, "test_file.txt"); + struct aws_string *test_file_mode = aws_string_new_from_c_str(allocator, "w"); + + FILE *output_file = aws_fopen("test_file", "w"); + ASSERT_NOT_NULL(output_file); + ASSERT_UINT_EQUALS(test_buf.len, fwrite(test_buf.buffer, 1, test_buf.len, output_file)); + fclose(output_file); + + struct aws_byte_buf output_buf; + AWS_ZERO_STRUCT(output_buf); + ASSERT_SUCCESS(aws_byte_buf_init_from_file(&output_buf, allocator, "test_file")); + aws_file_delete(test_file); + ASSERT_BIN_ARRAYS_EQUALS(test_buf.buffer, test_buf.len, output_buf.buffer, output_buf.len); + + aws_byte_buf_clean_up(&output_buf); + aws_string_destroy(test_file_mode); + aws_string_destroy(test_file); + + return AWS_OP_SUCCESS; +} + +AWS_TEST_CASE(test_byte_buf_file_read, s_test_byte_buf_file_read); diff --git a/tests/system_info_tests.c b/tests/system_info_tests.c index 571b6cbc5..7a4324797 100644 --- a/tests/system_info_tests.c +++ b/tests/system_info_tests.c @@ -178,7 +178,7 @@ static int s_test_sanity_check_environment_loader(struct aws_allocator *allocato struct aws_byte_cursor virt_product = aws_system_environment_get_virtualization_product_name(env); ASSERT_TRUE(aws_byte_cursor_is_valid(&virt_product)); - aws_system_environment_destroy(env); + aws_system_environment_release(env); aws_common_library_clean_up(); return 0; From 3df39f53aa70a478263e50d648dcf17e0ee7ffeb Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 17:54:03 -0700 Subject: [PATCH 16/18] Sorry about that msvc. good catch. --- source/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/file.c b/source/file.c index e29458146..050d9cc62 100644 --- a/source/file.c +++ b/source/file.c @@ -70,7 +70,7 @@ int aws_byte_buf_init_from_file(struct aws_byte_buf *out_buf, struct aws_allocat size_t allocation_size = (size_t)len64 + 1; aws_byte_buf_init(out_buf, alloc, allocation_size); - size_t read = -1; + size_t read = 0; size_t total_read = 0; do { if (total_read == out_buf->capacity) { From 4595a05d56cac42524e6499e5453b7f3b58191af Mon Sep 17 00:00:00 2001 From: "Jonathan M. Henson" Date: Wed, 11 Oct 2023 18:00:02 -0700 Subject: [PATCH 17/18] Add disclaimer to the env api. --- include/aws/common/system_info.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/aws/common/system_info.h b/include/aws/common/system_info.h index be4554fd4..91da41f9d 100644 --- a/include/aws/common/system_info.h +++ b/include/aws/common/system_info.h @@ -29,6 +29,9 @@ AWS_EXTERN_C_BEGIN /** * Allocates and initializes information about the system the current process is executing on. * If successful returns an instance of aws_system_environment. If it fails, it will return NULL. + * + * Note: This api is used internally and is still early in its evolution. + * It may change in incompatible ways in the future. */ AWS_COMMON_API struct aws_system_environment *aws_system_environment_load(struct aws_allocator *allocator); From 93672d5557d1d273ef9bc9c8bf8a3789a848f954 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Fri, 13 Oct 2023 08:46:59 -0700 Subject: [PATCH 18/18] Use new aws_byte_buf_init_from_file_with_size_hint() function, so don't waste 4KB on these short strings. --- source/linux/system_info.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/linux/system_info.c b/source/linux/system_info.c index bdfd2a99b..2d9c5a120 100644 --- a/source/linux/system_info.c +++ b/source/linux/system_info.c @@ -6,10 +6,14 @@ #include int aws_system_environment_load_platform_impl(struct aws_system_environment *env) { - aws_byte_buf_init_from_file(&env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor"); + /* provide size_hint when reading "special files", since some platforms mis-report these files' size as 4KB */ + aws_byte_buf_init_from_file_with_size_hint( + &env->virtualization_vendor, env->allocator, "/sys/devices/virtual/dmi/id/sys_vendor", 32 /*size_hint*/); + /* whether this one works depends on if this is a sysfs filesystem. If it fails, it will just be empty * and these APIs are a best effort at the moment. We can add fallbacks as the loaders get more complicated. */ - aws_byte_buf_init_from_file(&env->product_name, env->allocator, "/sys/devices/virtual/dmi/id/product_name"); + aws_byte_buf_init_from_file_with_size_hint( + &env->product_name, env->allocator, "/sys/devices/virtual/dmi/id/product_name", 32 /*size_hint*/); return AWS_OP_SUCCESS; }