From 765b3c61087715ee4c6c0d952f5bb0599a279b5f Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Tue, 17 Dec 2024 16:43:35 -0600 Subject: [PATCH] Add working code to use AWS_SESSION_TOKEN/temporary credentials for ros3. --- src/H5FDros3.c | 10 ++++++++++ src/H5FDros3.h | 1 + src/H5FDs3comms.c | 41 ++++++++++++++++++++++++++++------------- src/H5FDs3comms.h | 2 +- test/ros3.c | 19 ++++++++++++++++++- test/s3comms.c | 23 ++++++++++++++--------- 6 files changed, 72 insertions(+), 24 deletions(-) diff --git a/src/H5FDros3.c b/src/H5FDros3.c index 02698c59399..aaa84574dac 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -967,6 +967,16 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2) else if (f2->fa.secret_key[0] != '\0') HGOTO_DONE(-1); + /* FAPL: SESSION_TOKEN */ + if (f1->fa.session_token[0] != '\0' && f2->fa.session_token[0] != '\0') { + if (strcmp(f1->fa.session_token, f2->fa.session_token)) + HGOTO_DONE(-1); + } + else if (f1->fa.session_token[0] != '\0') + HGOTO_DONE(-1); + else if (f2->fa.session_token[0] != '\0') + HGOTO_DONE(-1); + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD__ros3_cmp() */ diff --git a/src/H5FDros3.h b/src/H5FDros3.h index ff51d6baae2..c25a51a8357 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -102,6 +102,7 @@ typedef struct H5FD_ros3_fapl_t { char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1]; char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1]; char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]; + char session_token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1]; } H5FD_ros3_fapl_t; #ifdef __cplusplus diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 98326c50a74..51abe447860 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -905,7 +905,7 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const *************************************/ if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) || - (token != NULL)) { + (token != NULL && *token != '\0')) { /* if one exists, all three must exist */ if (region == NULL || region[0] == '\0') @@ -1745,21 +1745,23 @@ H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m */ static herr_t H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, char *key_id, char *access_key, - char *aws_region) + char *session_token, char *aws_region) { char profile_line[32]; - char buffer[128]; + char buffer[4096]; const char *setting_names[] = { "region", "aws_access_key_id", "aws_secret_access_key", + "aws_session_token", }; char *const setting_pointers[] = { aws_region, key_id, access_key, + session_token, }; - unsigned setting_count = 3; + unsigned setting_count = 4; herr_t ret_value = SUCCEED; unsigned setting_i = 0; int found_setting = 0; @@ -1775,9 +1777,9 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha /* look for start of profile */ do { /* clear buffer */ - memset(buffer, 0, 128); + memset(buffer, 0, 4096); - line_buffer = fgets(line_buffer, 128, file); + line_buffer = fgets(line_buffer, 4096, file); if (line_buffer == NULL) /* reached end of file */ goto done; } while (strncmp(line_buffer, profile_line, strlen(profile_line))); @@ -1785,11 +1787,11 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha /* extract credentials from lines */ do { /* clear buffer and flag */ - memset(buffer, 0, 128); + memset(buffer, 0, 4096); found_setting = 0; /* collect a line from file */ - line_buffer = fgets(line_buffer, 128, file); + line_buffer = fgets(line_buffer, 4096, file); if (line_buffer == NULL) goto done; /* end of file */ @@ -1858,11 +1860,13 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha * */ static herr_t -H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, char *aws_region) +H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, + char *session_token, char *aws_region) { herr_t ret_value = SUCCEED; char *key_id_env = NULL; char *secret_access_key_env = NULL; + char *session_token_env = NULL; char *aws_region_env = NULL; FUNC_ENTER_PACKAGE @@ -1873,7 +1877,7 @@ H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, cha */ key_id_env = getenv("AWS_ACCESS_KEY_ID"); if (key_id_env != NULL && key_id_env[0] != '\0') { - if (strlen(key_id) == 0 || strncmp(key_id, key_id_env, strlen(key_id) != 0)) + if (strlen(key_id) == 0 || strncmp(key_id, key_id_env, strlen(key_id)) != 0) strncpy(key_id, key_id_env, strlen(key_id_env)); key_id[strlen(key_id_env)] = '\0'; } @@ -1888,6 +1892,16 @@ H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, cha } } + /* AWS_SESSION_TOKEN values are unbounded, but for now assume < 4096 */ + session_token_env = getenv("AWS_SESSION_TOKEN"); + if (session_token_env != NULL && session_token_env[0] != '\0') { + if (strlen(session_token) == 0 || + strncmp(session_token, session_token_env, strlen(session_token_env)) != 0) { + strncpy(session_token, session_token_env, strlen(session_token_env)); + session_token[strlen(session_token_env)] = '\0'; + } + } + /* AWS_REGION values are 9 - ~12 characters */ aws_region_env = getenv("AWS_REGION"); if (aws_region_env != NULL && aws_region_env[0] != '\0') { @@ -1931,7 +1945,7 @@ H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, cha */ herr_t H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *secret_access_key_out, - char *aws_region_out) + char *session_token_out, char *aws_region_out) { herr_t ret_value = SUCCEED; FILE *credfile = NULL; @@ -1955,7 +1969,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * credfile = fopen(filepath, "r"); if (credfile != NULL) { if (H5FD__s3comms_load_aws_creds_from_file(credfile, profile_name, key_id_out, secret_access_key_out, - aws_region_out) == FAIL) + session_token_out, aws_region_out) == FAIL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws credentials"); if (fclose(credfile) == EOF) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close credentials file"); @@ -1970,6 +1984,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * if (H5FD__s3comms_load_aws_creds_from_file( credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL, (*secret_access_key_out == 0) ? secret_access_key_out : NULL, + (*session_token_out == 0) ? session_token_out : NULL, (*aws_region_out == 0) ? aws_region_out : NULL) == FAIL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws config"); if (fclose(credfile) == EOF) @@ -1980,7 +1995,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * /* Check for credentials in environment variables. Environment variables will override * credentials from credentials/config files and just load them if there were none in * the files. */ - ret_value = H5FD__s3comms_load_aws_creds_from_env(key_id_out, secret_access_key_out, aws_region_out); + ret_value = H5FD__s3comms_load_aws_creds_from_env(key_id_out, secret_access_key_out, session_token_out, aws_region_out); /* fail if not all three settings were loaded */ if (*key_id_out == 0 || *secret_access_key_out == 0 || *aws_region_out == 0) diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index fdf13ac6b99..055089e6b42 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -496,7 +496,7 @@ H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, size_t msg_len, char *dest); H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name, char *key_id_out, char *secret_access_key_out, - char *aws_region_out); + char *aws_session_token_out, char *aws_region_out); H5_DLL herr_t H5FD_s3comms_parse_url(const char *str, parsed_url_t **purl); diff --git a/test/ros3.c b/test/ros3.c index b5307a7305f..ce2e6623ec4 100644 --- a/test/ros3.c +++ b/test/ros3.c @@ -56,6 +56,7 @@ static int s3_test_credentials_loaded = 0; static char s3_test_aws_region[16]; static char s3_test_aws_access_key_id[64]; static char s3_test_aws_secret_access_key[128]; +static char s3_test_aws_session_token[4096]; H5FD_ros3_fapl_t restricted_access_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, /* fapl version */ true, /* authenticate */ @@ -504,6 +505,8 @@ test_eof_eoa(void) TEST_ERROR; if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0) TEST_ERROR; + if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0) + TEST_ERROR; /* Open and verify EOA and EOF in a sample file */ if (NULL == (fd = H5FDopen(url_text_restricted, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) @@ -647,6 +650,8 @@ test_vfl_read(void) TEST_ERROR; if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0) TEST_ERROR; + if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0) + TEST_ERROR; if (NULL == (fd = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) TEST_ERROR; @@ -739,6 +744,8 @@ test_vfl_read_without_eoa_set_fails(void) TEST_ERROR; if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0) TEST_ERROR; + if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0) + TEST_ERROR; /* Open w/ VFL call */ if (NULL == (fd = H5FDopen(url_text_restricted, H5F_ACC_RDONLY, fapl_id, MAXADDR))) @@ -811,6 +818,8 @@ test_noops_and_autofails(void) TEST_ERROR; if (H5Pset_fapl_ros3(fapl_id, &anonymous_fa) < 0) TEST_ERROR; + if (H5Pset_fapl_ros3_token(fapl_id, &anonymous_fa.session_token) < 0) + TEST_ERROR; if (NULL == (fd = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) TEST_ERROR; @@ -896,6 +905,8 @@ test_cmp(void) TEST_ERROR; if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0) TEST_ERROR; + if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0) + TEST_ERROR; /* Open files */ if (NULL == (fd_raven = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) @@ -974,6 +985,8 @@ test_ros3_access_modes(void) TEST_ERROR; if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0) TEST_ERROR; + if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0) + TEST_ERROR; /* Read-Write Open access is not allowed with this file driver */ H5E_BEGIN_TRY @@ -1086,17 +1099,21 @@ main(void) /* Clear profile data strings */ s3_test_aws_access_key_id[0] = '\0'; s3_test_aws_secret_access_key[0] = '\0'; + s3_test_aws_session_token[0] = '\0'; s3_test_aws_region[0] = '\0'; /* Attempt to load test credentials - if unable, certain tests will be skipped */ if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id, - s3_test_aws_secret_access_key, s3_test_aws_region)) { + s3_test_aws_secret_access_key, + s3_test_aws_session_token, s3_test_aws_region)) { s3_test_credentials_loaded = 1; strncpy(restricted_access_fa.aws_region, (const char *)s3_test_aws_region, H5FD_ROS3_MAX_REGION_LEN); strncpy(restricted_access_fa.secret_id, (const char *)s3_test_aws_access_key_id, H5FD_ROS3_MAX_SECRET_ID_LEN); strncpy(restricted_access_fa.secret_key, (const char *)s3_test_aws_secret_access_key, H5FD_ROS3_MAX_SECRET_KEY_LEN); + strncpy(restricted_access_fa.session_token, (const char *)s3_test_aws_session_token, + H5FD_ROS3_MAX_SECRET_TOK_LEN); } /****************** diff --git a/test/s3comms.c b/test/s3comms.c index 61d73f37f35..599d754bb94 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -42,9 +42,9 @@ */ static int s3_test_credentials_loaded = 0; static char s3_test_aws_region[16] = ""; -static char s3_test_aws_access_key_id[64] = ""; +static char s3_test_aws_access_key_id[128] = ""; static char s3_test_aws_secret_access_key[128] = ""; -static char s3_test_aws_security_token[1024] = ""; +static char s3_test_aws_session_token[4096] = ""; static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; static bool s3_test_bucket_defined = false; @@ -1507,7 +1507,7 @@ test_s3r_open(void) { handle = H5FD_s3comms_s3r_open( url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, - (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); + (const unsigned char *)signing_key, (const char *)s3_test_aws_session_token); } H5E_END_TRY if (handle != NULL) @@ -1531,7 +1531,7 @@ test_s3r_open(void) { handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID", (const unsigned char *)signing_key, - (const char *)s3_test_aws_security_token); + (const char *)s3_test_aws_session_token); } H5E_END_TRY if (handle != NULL) @@ -1542,7 +1542,7 @@ test_s3r_open(void) { handle = H5FD_s3comms_s3r_open( url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, - (const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token); + (const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_session_token); } H5E_END_TRY if (handle != NULL) @@ -1565,7 +1565,7 @@ test_s3r_open(void) /* Using authentication on anonymously-accessible file? */ handle = H5FD_s3comms_s3r_open( url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, - (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); + (const unsigned char *)signing_key, (const char *)s3_test_aws_session_token); if (handle == NULL) TEST_ERROR; if (6464 != H5FD_s3comms_s3r_get_filesize(handle)) @@ -1577,7 +1577,7 @@ test_s3r_open(void) /* Authenticating */ handle = H5FD_s3comms_s3r_open( url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id, - (const unsigned char *)signing_key, (const char *)s3_test_aws_security_token); + (const unsigned char *)signing_key, (const char *)s3_test_aws_session_token); if (handle == NULL) TEST_ERROR; if (5458199 != H5FD_s3comms_s3r_get_filesize(handle)) @@ -1646,6 +1646,7 @@ test_s3r_read(void) if (6464 != H5FD_s3comms_s3r_get_filesize(handle)) TEST_ERROR; + if (handle != NULL) /***************************** * Tests that should succeed * *****************************/ @@ -1767,6 +1768,7 @@ main(void) /* "clear" profile data strings */ s3_test_aws_access_key_id[0] = '\0'; s3_test_aws_secret_access_key[0] = '\0'; + s3_test_aws_session_token[0] = '\0'; s3_test_aws_region[0] = '\0'; s3_test_bucket_url[0] = '\0'; @@ -1777,8 +1779,11 @@ main(void) /* attempt to load test credentials * if unable, certain tests will be skipped */ - if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id, - s3_test_aws_secret_access_key, s3_test_aws_region)) { + if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, + s3_test_aws_access_key_id, + s3_test_aws_secret_access_key, + s3_test_aws_session_token, + s3_test_aws_region)) { s3_test_credentials_loaded = 1; }