Skip to content

Commit

Permalink
Add working code to use AWS_SESSION_TOKEN/temporary credentials for
Browse files Browse the repository at this point in the history 
ros3.
  • Loading branch information
@lrknox
lrknox committed Dec 17, 2024
1 parent d581624 commit 765b3c6
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 24 deletions.
10 changes: 10 additions & 0 deletions src/H5FDros3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -967,6 +967,16 @@ H5FD__ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
else if (f2->fa.secret_key[0] != '\0')
HGOTO_DONE(-1);

/* FAPL: SESSION_TOKEN */
if (f1->fa.session_token[0] != '\0' && f2->fa.session_token[0] != '\0') {
if (strcmp(f1->fa.session_token, f2->fa.session_token))
HGOTO_DONE(-1);
}
else if (f1->fa.session_token[0] != '\0')
HGOTO_DONE(-1);
else if (f2->fa.session_token[0] != '\0')
HGOTO_DONE(-1);

done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5FD__ros3_cmp() */
Expand Down
1 change: 1 addition & 0 deletions src/H5FDros3.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ typedef struct H5FD_ros3_fapl_t {
char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1];
char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1];
char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1];
char session_token[H5FD_ROS3_MAX_SECRET_TOK_LEN + 1];
} H5FD_ros3_fapl_t;

#ifdef __cplusplus
Expand Down
41 changes: 28 additions & 13 deletions src/H5FDs3comms.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -905,7 +905,7 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const
*************************************/

if ((region != NULL && *region != '\0') || (id != NULL && *id != '\0') || (signing_key != NULL) ||
(token != NULL)) {
(token != NULL && *token != '\0')) {

/* if one exists, all three must exist */
if (region == NULL || region[0] == '\0')
Expand Down Expand Up @@ -1745,21 +1745,23 @@ H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m
*/
static herr_t
H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, char *key_id, char *access_key,
char *aws_region)
char *session_token, char *aws_region)
{
char profile_line[32];
char buffer[128];
char buffer[4096];
const char *setting_names[] = {
"region",
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
};
char *const setting_pointers[] = {
aws_region,
key_id,
access_key,
session_token,
};
unsigned setting_count = 3;
unsigned setting_count = 4;
herr_t ret_value = SUCCEED;
unsigned setting_i = 0;
int found_setting = 0;
Expand All @@ -1775,21 +1777,21 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha
/* look for start of profile */
do {
/* clear buffer */
memset(buffer, 0, 128);
memset(buffer, 0, 4096);

line_buffer = fgets(line_buffer, 128, file);
line_buffer = fgets(line_buffer, 4096, file);
if (line_buffer == NULL) /* reached end of file */
goto done;
} while (strncmp(line_buffer, profile_line, strlen(profile_line)));

/* extract credentials from lines */
do {
/* clear buffer and flag */
memset(buffer, 0, 128);
memset(buffer, 0, 4096);
found_setting = 0;

/* collect a line from file */
line_buffer = fgets(line_buffer, 128, file);
line_buffer = fgets(line_buffer, 4096, file);
if (line_buffer == NULL)
goto done; /* end of file */

Expand Down Expand Up @@ -1858,11 +1860,13 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha
*
*/
static herr_t
H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, char *aws_region)
H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key,
char *session_token, char *aws_region)
{
herr_t ret_value = SUCCEED;
char *key_id_env = NULL;
char *secret_access_key_env = NULL;
char *session_token_env = NULL;
char *aws_region_env = NULL;

FUNC_ENTER_PACKAGE
Expand All @@ -1873,7 +1877,7 @@ H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, cha
*/
key_id_env = getenv("AWS_ACCESS_KEY_ID");
if (key_id_env != NULL && key_id_env[0] != '\0') {
if (strlen(key_id) == 0 || strncmp(key_id, key_id_env, strlen(key_id) != 0))
if (strlen(key_id) == 0 || strncmp(key_id, key_id_env, strlen(key_id)) != 0)
strncpy(key_id, key_id_env, strlen(key_id_env));
key_id[strlen(key_id_env)] = '\0';
}
Expand All @@ -1888,6 +1892,16 @@ H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, cha
}
}

/* AWS_SESSION_TOKEN values are unbounded, but for now assume < 4096 */
session_token_env = getenv("AWS_SESSION_TOKEN");
if (session_token_env != NULL && session_token_env[0] != '\0') {
if (strlen(session_token) == 0 ||
strncmp(session_token, session_token_env, strlen(session_token_env)) != 0) {
strncpy(session_token, session_token_env, strlen(session_token_env));
session_token[strlen(session_token_env)] = '\0';
}
}

/* AWS_REGION values are 9 - ~12 characters */
aws_region_env = getenv("AWS_REGION");
if (aws_region_env != NULL && aws_region_env[0] != '\0') {
Expand Down Expand Up @@ -1931,7 +1945,7 @@ H5FD__s3comms_load_aws_creds_from_env(char *key_id, char *secret_access_key, cha
*/
herr_t
H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *secret_access_key_out,
char *aws_region_out)
char *session_token_out, char *aws_region_out)
{
herr_t ret_value = SUCCEED;
FILE *credfile = NULL;
Expand All @@ -1955,7 +1969,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *
credfile = fopen(filepath, "r");
if (credfile != NULL) {
if (H5FD__s3comms_load_aws_creds_from_file(credfile, profile_name, key_id_out, secret_access_key_out,
aws_region_out) == FAIL)
session_token_out, aws_region_out) == FAIL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws credentials");
if (fclose(credfile) == EOF)
HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close credentials file");
Expand All @@ -1970,6 +1984,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *
if (H5FD__s3comms_load_aws_creds_from_file(
credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL,
(*secret_access_key_out == 0) ? secret_access_key_out : NULL,
(*session_token_out == 0) ? session_token_out : NULL,
(*aws_region_out == 0) ? aws_region_out : NULL) == FAIL)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to load from aws config");
if (fclose(credfile) == EOF)
Expand All @@ -1980,7 +1995,7 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *
/* Check for credentials in environment variables. Environment variables will override
* credentials from credentials/config files and just load them if there were none in
* the files. */
ret_value = H5FD__s3comms_load_aws_creds_from_env(key_id_out, secret_access_key_out, aws_region_out);
ret_value = H5FD__s3comms_load_aws_creds_from_env(key_id_out, secret_access_key_out, session_token_out, aws_region_out);

/* fail if not all three settings were loaded */
if (*key_id_out == 0 || *secret_access_key_out == 0 || *aws_region_out == 0)
Expand Down
2 changes: 1 addition & 1 deletion src/H5FDs3comms.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,7 @@ H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len,
size_t msg_len, char *dest);

H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name, char *key_id_out, char *secret_access_key_out,
char *aws_region_out);
char *aws_session_token_out, char *aws_region_out);

H5_DLL herr_t H5FD_s3comms_parse_url(const char *str, parsed_url_t **purl);

Expand Down
19 changes: 18 additions & 1 deletion test/ros3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ static int s3_test_credentials_loaded = 0;
static char s3_test_aws_region[16];
static char s3_test_aws_access_key_id[64];
static char s3_test_aws_secret_access_key[128];
static char s3_test_aws_session_token[4096];

H5FD_ros3_fapl_t restricted_access_fa = {H5FD_CURR_ROS3_FAPL_T_VERSION, /* fapl version */
true, /* authenticate */
Expand Down Expand Up @@ -504,6 +505,8 @@ test_eof_eoa(void)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0)
TEST_ERROR;

/* Open and verify EOA and EOF in a sample file */
if (NULL == (fd = H5FDopen(url_text_restricted, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
Expand Down Expand Up @@ -647,6 +650,8 @@ test_vfl_read(void)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0)
TEST_ERROR;

if (NULL == (fd = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
TEST_ERROR;
Expand Down Expand Up @@ -739,6 +744,8 @@ test_vfl_read_without_eoa_set_fails(void)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0)
TEST_ERROR;

/* Open w/ VFL call */
if (NULL == (fd = H5FDopen(url_text_restricted, H5F_ACC_RDONLY, fapl_id, MAXADDR)))
Expand Down Expand Up @@ -811,6 +818,8 @@ test_noops_and_autofails(void)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &anonymous_fa) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3_token(fapl_id, &anonymous_fa.session_token) < 0)
TEST_ERROR;
if (NULL == (fd = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
TEST_ERROR;

Expand Down Expand Up @@ -896,6 +905,8 @@ test_cmp(void)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0)
TEST_ERROR;

/* Open files */
if (NULL == (fd_raven = H5FDopen(url_text_public, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF)))
Expand Down Expand Up @@ -974,6 +985,8 @@ test_ros3_access_modes(void)
TEST_ERROR;
if (H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) < 0)
TEST_ERROR;
if (H5Pset_fapl_ros3_token(fapl_id, &restricted_access_fa.session_token) < 0)
TEST_ERROR;

/* Read-Write Open access is not allowed with this file driver */
H5E_BEGIN_TRY
Expand Down Expand Up @@ -1086,17 +1099,21 @@ main(void)
/* Clear profile data strings */
s3_test_aws_access_key_id[0] = '\0';
s3_test_aws_secret_access_key[0] = '\0';
s3_test_aws_session_token[0] = '\0';
s3_test_aws_region[0] = '\0';

/* Attempt to load test credentials - if unable, certain tests will be skipped */
if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id,
s3_test_aws_secret_access_key, s3_test_aws_region)) {
s3_test_aws_secret_access_key,
s3_test_aws_session_token, s3_test_aws_region)) {
s3_test_credentials_loaded = 1;
strncpy(restricted_access_fa.aws_region, (const char *)s3_test_aws_region, H5FD_ROS3_MAX_REGION_LEN);
strncpy(restricted_access_fa.secret_id, (const char *)s3_test_aws_access_key_id,
H5FD_ROS3_MAX_SECRET_ID_LEN);
strncpy(restricted_access_fa.secret_key, (const char *)s3_test_aws_secret_access_key,
H5FD_ROS3_MAX_SECRET_KEY_LEN);
strncpy(restricted_access_fa.session_token, (const char *)s3_test_aws_session_token,
H5FD_ROS3_MAX_SECRET_TOK_LEN);
}

/******************
Expand Down
23 changes: 14 additions & 9 deletions test/s3comms.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@
*/
static int s3_test_credentials_loaded = 0;
static char s3_test_aws_region[16] = "";
static char s3_test_aws_access_key_id[64] = "";
static char s3_test_aws_access_key_id[128] = "";
static char s3_test_aws_secret_access_key[128] = "";
static char s3_test_aws_security_token[1024] = "";
static char s3_test_aws_session_token[4096] = "";
static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = "";
static bool s3_test_bucket_defined = false;

Expand Down Expand Up @@ -1507,7 +1507,7 @@ test_s3r_open(void)
{
handle = H5FD_s3comms_s3r_open(
url_missing, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
(const unsigned char *)signing_key, (const char *)s3_test_aws_session_token);
}
H5E_END_TRY
if (handle != NULL)
Expand All @@ -1531,7 +1531,7 @@ test_s3r_open(void)
{
handle = H5FD_s3comms_s3r_open(url_shakespeare, (const char *)s3_test_aws_region, "I_MADE_UP_MY_ID",
(const unsigned char *)signing_key,
(const char *)s3_test_aws_security_token);
(const char *)s3_test_aws_session_token);
}
H5E_END_TRY
if (handle != NULL)
Expand All @@ -1542,7 +1542,7 @@ test_s3r_open(void)
{
handle = H5FD_s3comms_s3r_open(
url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_security_token);
(const unsigned char *)EMPTY_SHA256, (const char *)s3_test_aws_session_token);
}
H5E_END_TRY
if (handle != NULL)
Expand All @@ -1565,7 +1565,7 @@ test_s3r_open(void)
/* Using authentication on anonymously-accessible file? */
handle = H5FD_s3comms_s3r_open(
url_raven, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
(const unsigned char *)signing_key, (const char *)s3_test_aws_session_token);
if (handle == NULL)
TEST_ERROR;
if (6464 != H5FD_s3comms_s3r_get_filesize(handle))
Expand All @@ -1577,7 +1577,7 @@ test_s3r_open(void)
/* Authenticating */
handle = H5FD_s3comms_s3r_open(
url_shakespeare, (const char *)s3_test_aws_region, (const char *)s3_test_aws_access_key_id,
(const unsigned char *)signing_key, (const char *)s3_test_aws_security_token);
(const unsigned char *)signing_key, (const char *)s3_test_aws_session_token);
if (handle == NULL)
TEST_ERROR;
if (5458199 != H5FD_s3comms_s3r_get_filesize(handle))
Expand Down Expand Up @@ -1646,6 +1646,7 @@ test_s3r_read(void)
if (6464 != H5FD_s3comms_s3r_get_filesize(handle))
TEST_ERROR;

if (handle != NULL)
/*****************************
* Tests that should succeed *
*****************************/
Expand Down Expand Up @@ -1767,6 +1768,7 @@ main(void)
/* "clear" profile data strings */
s3_test_aws_access_key_id[0] = '\0';
s3_test_aws_secret_access_key[0] = '\0';
s3_test_aws_session_token[0] = '\0';
s3_test_aws_region[0] = '\0';
s3_test_bucket_url[0] = '\0';

Expand All @@ -1777,8 +1779,11 @@ main(void)
/* attempt to load test credentials
* if unable, certain tests will be skipped
*/
if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME, s3_test_aws_access_key_id,
s3_test_aws_secret_access_key, s3_test_aws_region)) {
if (SUCCEED == H5FD_s3comms_load_aws_profile(S3_TEST_PROFILE_NAME,
s3_test_aws_access_key_id,
s3_test_aws_secret_access_key,
s3_test_aws_session_token,
s3_test_aws_region)) {
s3_test_credentials_loaded = 1;
}

Expand Down

0 comments on commit 765b3c6

Please sign in to comment.