Skip to content

Commit

Permalink
Merge pull request #7997 from naveenpaul1/anonymous_namespace_req
Browse files Browse the repository at this point in the history 
NSFS | Support anonymous NSFS requests
  • Loading branch information
@naveenpaul1
naveenpaul1 authored May 29, 2024
2 parents 958ad06 + 8501413 commit a97af1d
Show file tree
Hide file tree
Showing 23 changed files with 909 additions and 50 deletions.
3 changes: 3 additions & 0 deletions config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,9 @@ config.NSFS_LOW_FREE_SPACE_MB_UNLEASH = 10 * 1024;
// operations safely.
config.NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH = 0.10;

// anonymous account name
config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';

////////////////////////////
// NSFS NON CONTAINERIZED //
////////////////////////////
Expand Down
57 changes: 57 additions & 0 deletions docs/non_containerized_NSFS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,3 +614,60 @@ Access is restricted to these whitelist IPs. If there are no IPs mentioned all I
```
sudo noobaa-cli whitelist --ips '["127.0.0.1", "192.000.10.000", "3002:0bd6:0000:0000:0000:ee00:0033:000"]' 2>/dev/null
```

## Anonymous request

Anonymous requests are those sent without access and secret key and Noobaa NSFS allow it with supporting bucket policy. Users have to create an anonymous account before accessing the resources. Noobaa uses anonymous account's uid and gid or user(distinguished_name) for accessing bucket storage paths. Noobaa can have one anonymous account. Commands for creating anonumous account are
```
noobaa-cli account add --anonymous --uid {uid} --gid {gid}
```
or
```
noobaa-cli account add --anonymous --user {user}
```

Update :
```
noobaa-cli account update --anonymous --uid {new_uid} --gid {new_gid}
```
or

```
noobaa-cli account update --anonymous --user {user}
```

Delete:
```
noobaa-cli account delete --anonymous
```
Status
```
noobaa-cli account status --anonymous
```

Bellow policy will allow anonymous user access and you can control the anonymous access using different policy configuration.

```
AWS_ACCESS_KEY_ID={access_key} AWS_SECRET_ACCESS_KEY={secret_key} aws s3api put-bucket-policy --endpoint {endpoint} --bucket {bucket_name} --policy file:///tmp/policy.json;
```

policy.json:

```
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "AWS": [ "*" ] },
"Action": [ "s3:*" ],
"Resource": [ "arn:aws:s3:::{bucket_name}/*",
"arn:aws:s3:::{bucket_name}" ]
}
]
}
```

User can perform all the actions on above bucket resource without providing access and secret key(not recommended).

`aws s3api list-objects --bucket {bucket_name} --endpoint {endpoint} --no-sign-request`
45 changes: 29 additions & 16 deletions src/cmd/manage_nsfs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils');
const { TYPES, ACTIONS, LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS,
GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants');
const { throw_cli_error, write_stdout_response, get_config_file_path, get_symlink_config_file_path,
get_config_data, get_boolean_or_string_value} = require('../manage_nsfs/manage_nsfs_cli_utils');
get_config_data, get_boolean_or_string_value, has_access_keys} = require('../manage_nsfs/manage_nsfs_cli_utils');
const { validate_input_types, validate_bucket_args, validate_account_args,
verify_delete_account, validate_whitelist_arg, verify_whitelist_ips,
_validate_access_keys } = require('../manage_nsfs/manage_nsfs_validations');
Expand Down Expand Up @@ -288,6 +288,10 @@ async function manage_bucket_operations(action, data, user_input) {

async function account_management(action, user_input) {
const show_secrets = get_boolean_or_string_value(user_input.show_secrets);
if (get_boolean_or_string_value(user_input.anonymous)) {
user_input.name = config.ANONYMOUS_ACCOUNT_NAME;
user_input.email = config.ANONYMOUS_ACCOUNT_NAME;
}
const data = await fetch_account_data(action, user_input);
await manage_account_operations(action, data, show_secrets, user_input);
}
Expand Down Expand Up @@ -315,7 +319,7 @@ function set_access_keys(access_key, secret_key, generate) {

// in name and new_name we allow type number, hence convert it to string
async function fetch_account_data(action, user_input) {
const { access_keys, new_access_key } = get_access_keys(action, user_input);
const { access_keys = [], new_access_key = undefined } = user_input.anonymous ? {} : get_access_keys(action, user_input);
let data = {
// added undefined values to keep the order the properties when printing the data object
_id: undefined,
Expand All @@ -342,11 +346,13 @@ async function fetch_account_data(action, user_input) {

// override values
// access_key as SensitiveString
data.access_keys[0].access_key = _.isUndefined(data.access_keys[0].access_key) ? undefined :
if (!has_access_keys(data.access_keys)) {
data.access_keys[0].access_key = _.isUndefined(data.access_keys[0].access_key) ? undefined :
new SensitiveString(String(data.access_keys[0].access_key));
// secret_key as SensitiveString
data.access_keys[0].secret_key = _.isUndefined(data.access_keys[0].secret_key) ? undefined :
// secret_key as SensitiveString
data.access_keys[0].secret_key = _.isUndefined(data.access_keys[0].secret_key) ? undefined :
new SensitiveString(String(data.access_keys[0].secret_key));
}
// fs_backend deletion specified with empty string '' (but it is not part of the schema)
data.nsfs_account_config.fs_backend = data.nsfs_account_config.fs_backend || undefined;
// new_buckets_path deletion specified with empty string ''
Expand Down Expand Up @@ -406,7 +412,7 @@ async function add_account(data) {
await validate_account_args(data, ACTIONS.ADD);

const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
const access_key = data.access_keys[0].access_key;
const access_key = has_access_keys(data.access_keys) ? undefined : data.access_keys[0].access_key;
const account_config_path = get_config_file_path(accounts_dir_path, data.name);
const account_config_relative_path = get_config_file_path(acounts_dir_relative_path, data.name);
const account_config_access_key_path = get_symlink_config_file_path(access_keys_dir_path, access_key);
Expand All @@ -423,13 +429,17 @@ async function add_account(data) {
const encrypted_account = await nc_mkm.encrypt_access_keys(data);
data.master_key_id = encrypted_account.master_key_id;
const encrypted_data = JSON.stringify(encrypted_account);
data = _.omitBy(data, _.isUndefined);
// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
// for validating against the schema we need an object, hence we parse it back to object
nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data);
await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR);
await nb_native().fs.symlink(fs_context, account_config_relative_path, account_config_access_key_path);
const account = encrypted_data ? JSON.parse(encrypted_data) : data;
nsfs_schema_utils.validate_account_schema(account);
await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, JSON.stringify(account));
if (!has_access_keys(data.access_keys)) {
await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR);
await nb_native().fs.symlink(fs_context, account_config_relative_path, account_config_access_key_path);
}
write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg });
}

Expand All @@ -440,7 +450,7 @@ async function update_account(data) {
const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
const cur_name = data.name;
const new_name = data.new_name;
const cur_access_key = data.access_keys[0].access_key;
const cur_access_key = has_access_keys(data.access_keys) ? undefined : data.access_keys[0].access_key;
const update_name = new_name && cur_name && data.new_name !== cur_name;
const update_access_key = data.new_access_key && cur_access_key && data.new_access_key !== cur_access_key;

Expand All @@ -449,11 +459,13 @@ async function update_account(data) {
const encrypted_account = await nc_mkm.encrypt_access_keys(data);
data.master_key_id = encrypted_account.master_key_id;
const encrypted_data = JSON.stringify(encrypted_account);
data = _.omitBy(data, _.isUndefined);
// We take an object that was stringify
// (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters)
// for validating against the schema we need an object, hence we parse it back to object
nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data));
await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data);
const account = encrypted_data ? JSON.parse(encrypted_data) : data;
nsfs_schema_utils.validate_account_schema(account);
await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, JSON.stringify(account));
write_stdout_response(ManageCLIResponse.AccountUpdated, data);
return;
}
Expand Down Expand Up @@ -502,10 +514,11 @@ async function delete_account(data) {

const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
const account_config_path = get_config_file_path(accounts_dir_path, data.name);
const access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key);

await native_fs_utils.delete_config_file(fs_context, accounts_dir_path, account_config_path);
await nb_native().fs.unlink(fs_context, access_key_config_path);
if (!has_access_keys(data.access_keys)) {
const access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key);
await nb_native().fs.unlink(fs_context, access_key_config_path);
}
write_stdout_response(ManageCLIResponse.AccountDeleted, '', {account: data.name});
}

Expand Down
6 changes: 6 additions & 0 deletions src/manage_nsfs/manage_nsfs_cli_errors.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ ManageCLIError.InvalidFlagsCombination = Object.freeze({
http_code: 400,
});

ManageCLIError.InvalidAccountName = Object.freeze({
code: 'InvalidAccountName',
message: 'Account name is invalid',
http_code: 400,
});

//////////////////////////////
//// IP WHITE LIST ERRORS ////
//////////////////////////////
Expand Down
10 changes: 10 additions & 0 deletions src/manage_nsfs/manage_nsfs_cli_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,15 @@ async function get_options_from_file(file_path) {
}
}

/**
* has_access_keys will return anonymous or not depending on the access key length
* (instead of flags) and return its content
* @param {object[]} access_keys
*/
function has_access_keys(access_keys) {
return access_keys.length === 0;
}

// EXPORTS
exports.throw_cli_error = throw_cli_error;
exports.write_stdout_response = write_stdout_response;
Expand All @@ -118,3 +127,4 @@ exports.get_boolean_or_string_value = get_boolean_or_string_value;
exports.get_config_data = get_config_data;
exports.get_bucket_owner_account = get_bucket_owner_account;
exports.get_options_from_file = get_options_from_file;
exports.has_access_keys = has_access_keys;
13 changes: 12 additions & 1 deletion src/manage_nsfs/manage_nsfs_constants.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ const CONFIG_SUBDIRS = {
const GLOBAL_CONFIG_ROOT = 'config_root';
const GLOBAL_CONFIG_OPTIONS = new Set([GLOBAL_CONFIG_ROOT, 'config_root_backend']);
const FROM_FILE = 'from_file';
const ANONYMOUS = 'anonymous';

const VALID_OPTIONS_ACCOUNT = {
'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]),
Expand All @@ -40,6 +41,13 @@ const VALID_OPTIONS_ACCOUNT = {
'status': new Set(['name', 'access_key', 'show_secrets', ...GLOBAL_CONFIG_OPTIONS]),
};

const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
'add': new Set(['uid', 'gid', 'user', 'anonymous', GLOBAL_CONFIG_ROOT]),
'update': new Set(['uid', 'gid', 'user', 'anonymous', GLOBAL_CONFIG_ROOT]),
'delete': new Set(['anonymous', GLOBAL_CONFIG_ROOT]),
'status': new Set(['anonymous', GLOBAL_CONFIG_ROOT]),
};

const VALID_OPTIONS_BUCKET = {
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]),
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', ...GLOBAL_CONFIG_OPTIONS]),
Expand All @@ -64,6 +72,7 @@ const VALID_OPTIONS = {
glacier_options: VALID_OPTIONS_GLACIER,
whitelist_options: VALID_OPTIONS_WHITELIST,
from_file_options: VALID_OPTIONS_FROM_FILE,
anonymous_account_options: VALID_OPTIONS_ANONYMOUS_ACCOUNT,
};

const OPTION_TYPE = {
Expand All @@ -88,7 +97,8 @@ const OPTION_TYPE = {
wide: 'boolean',
show_secrets: 'boolean',
ips: 'string',
force: 'boolean'
force: 'boolean',
anonymous: 'boolean'
};

const BOOLEAN_STRING_VALUES = ['true', 'false'];
Expand All @@ -112,3 +122,4 @@ exports.LIST_UNSETABLE_OPTIONS = LIST_UNSETABLE_OPTIONS;

exports.LIST_ACCOUNT_FILTERS = LIST_ACCOUNT_FILTERS;
exports.LIST_BUCKET_FILTERS = LIST_BUCKET_FILTERS;
exports.ANONYMOUS = ANONYMOUS;
45 changes: 36 additions & 9 deletions src/manage_nsfs/manage_nsfs_validations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ const native_fs_utils = require('../util/native_fs_utils');
const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
const { throw_cli_error, get_config_file_path, get_bucket_owner_account,
get_config_data, get_options_from_file } = require('../manage_nsfs/manage_nsfs_cli_utils');
get_config_data, get_options_from_file, has_access_keys } = require('../manage_nsfs/manage_nsfs_cli_utils');
const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES,
GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS } = require('../manage_nsfs/manage_nsfs_constants');
GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS, ANONYMOUS } = require('../manage_nsfs/manage_nsfs_constants');

/////////////////////////////
//// GENERAL VALIDATIONS ////
Expand All @@ -39,6 +39,7 @@ async function validate_input_types(type, action, argv) {
validate_no_extra_options(type, action, input_options, false);
validate_options_type_by_value(input_options_with_data);
validate_flags_combination(type, action, input_options);
validate_flags_value_combination(type, action, input_options_with_data);
if (action === ACTIONS.UPDATE) validate_min_flags_for_update(type, input_options_with_data);

// currently we use from_file only in add action
Expand All @@ -53,6 +54,7 @@ async function validate_input_types(type, action, argv) {
validate_no_extra_options(type, action, input_options_from_file, true);
validate_options_type_by_value(input_options_with_data_from_file);
validate_flags_combination(type, action, input_options_from_file);
validate_flags_value_combination(type, action, input_options_with_data);
return input_options_with_data_from_file;
}
}
Expand Down Expand Up @@ -90,7 +92,11 @@ function validate_no_extra_options(type, action, input_options, is_options_from_
} else if (type === TYPES.BUCKET) {
valid_options = VALID_OPTIONS.bucket_options[action];
} else if (type === TYPES.ACCOUNT) {
valid_options = VALID_OPTIONS.account_options[action];
if (input_options.includes(ANONYMOUS)) {
valid_options = VALID_OPTIONS.anonymous_account_options[action];
} else {
valid_options = VALID_OPTIONS.account_options[action];
}
} else if (type === TYPES.GLACIER) {
valid_options = VALID_OPTIONS.glacier_options[action];
} else {
Expand All @@ -102,7 +108,6 @@ function validate_no_extra_options(type, action, input_options, is_options_from_
valid_options.delete('config_root');
valid_options.delete('config_root_backend');
}

const invalid_input_options = input_options.filter(element => !valid_options.has(element));
if (invalid_input_options.length > 0) {
const type_and_action = type === TYPES.IP_WHITELIST ? type : `${type} ${action}`;
Expand Down Expand Up @@ -134,7 +139,7 @@ function validate_options_type_by_value(input_options_with_data) {
continue;
}
// special case for boolean values
if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force', 'force_md5_etag'].includes(option) && validate_boolean_string_value(value)) {
if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force', 'force_md5_etag', 'anonymous'].includes(option) && validate_boolean_string_value(value)) {
continue;
}
// special case for bucket_policy (from_file)
Expand Down Expand Up @@ -174,7 +179,7 @@ function validate_min_flags_for_update(type, input_options_with_data) {

// GAP - mandatory flags check should be earlier in the calls in general
if (_.isUndefined(input_options_with_data.name)) {
if (type === TYPES.ACCOUNT) throw_cli_error(ManageCLIError.MissingAccountNameFlag);
if (type === TYPES.ACCOUNT && !input_options_with_data.anonymous) throw_cli_error(ManageCLIError.MissingAccountNameFlag);
if (type === TYPES.BUCKET) throw_cli_error(ManageCLIError.MissingBucketNameFlag);
}

Expand Down Expand Up @@ -213,6 +218,24 @@ function validate_flags_combination(type, action, input_options) {
}
}

/**
* validate_flags_value_combination checks flags and value combination.
* 1. account add or update - name should not be anonymous
* @param {string} type
* @param {string} action
* @param {object} input_options_with_data
*/
function validate_flags_value_combination(type, action, input_options_with_data) {
if (type === TYPES.ACCOUNT) {
if (action === ACTIONS.ADD || action === ACTIONS.UPDATE) {
if (input_options_with_data.name === ANONYMOUS || input_options_with_data.new_name === ANONYMOUS) {
const detail = 'Account name \'anonymous\' is not valid';
throw_cli_error(ManageCLIError.InvalidAccountName, detail);
}
}
}
}

/////////////////////////////
//// BUCKET VALIDATIONS /////
/////////////////////////////
Expand Down Expand Up @@ -300,16 +323,20 @@ async function validate_bucket_args(config_root_backend, accounts_dir_path, data
*/
async function validate_account_args(data, action) {
if (action === ACTIONS.STATUS || action === ACTIONS.DELETE) {
if (_.isUndefined(data.access_keys[0].access_key) && _.isUndefined(data.name)) {
if (!has_access_keys(data.access_keys) && _.isUndefined(data.access_keys[0].access_key) && _.isUndefined(data.name)) {
throw_cli_error(ManageCLIError.MissingIdentifier);
}
} else {
if ((action !== ACTIONS.UPDATE && data.new_name)) throw_cli_error(ManageCLIError.InvalidNewNameAccountIdentifier);
if ((action !== ACTIONS.UPDATE && data.new_access_key)) throw_cli_error(ManageCLIError.InvalidNewAccessKeyIdentifier);
if (_.isUndefined(data.name)) throw_cli_error(ManageCLIError.MissingAccountNameFlag);

if (_.isUndefined(data.access_keys[0].secret_key)) throw_cli_error(ManageCLIError.MissingAccountSecretKeyFlag);
if (_.isUndefined(data.access_keys[0].access_key)) throw_cli_error(ManageCLIError.MissingAccountAccessKeyFlag);
if (!has_access_keys(data.access_keys) && _.isUndefined(data.access_keys[0].secret_key)) {
throw_cli_error(ManageCLIError.MissingAccountSecretKeyFlag);
}
if (!has_access_keys(data.access_keys) && _.isUndefined(data.access_keys[0].access_key)) {
throw_cli_error(ManageCLIError.MissingAccountAccessKeyFlag);
}
if (data.nsfs_account_config.gid && data.nsfs_account_config.uid === undefined) {
throw_cli_error(ManageCLIError.MissingAccountNSFSConfigUID, data.nsfs_account_config);
}
Expand Down
Loading

0 comments on commit a97af1d

Please sign in to comment.