From 8501413f165db24fbfad354dea35ebe35a8a2b94 Mon Sep 17 00:00:00 2001 From: naveenpaul1 Date: Wed, 24 Apr 2024 20:39:10 +0530 Subject: [PATCH] Support anonymous NSFS requests Signed-off-by: naveenpaul1 --- config.js | 3 + docs/non_containerized_NSFS.md | 57 +++ src/cmd/manage_nsfs.js | 45 ++- src/manage_nsfs/manage_nsfs_cli_errors.js | 6 + src/manage_nsfs/manage_nsfs_cli_utils.js | 10 + src/manage_nsfs/manage_nsfs_constants.js | 13 +- src/manage_nsfs/manage_nsfs_validations.js | 45 ++- src/sdk/bucketspace_fs.js | 12 +- src/sdk/bucketspace_nb.js | 8 + src/sdk/nb.d.ts | 3 + src/sdk/object_sdk.js | 21 +- .../schemas/nsfs_account_schema.js | 2 +- .../nsfs_s3_tests_black_list.txt | 2 +- .../s3-tests-lists/s3_tests_black_list.txt | 1 + .../test_ceph_nsfs_s3_config_setup.js | 29 +- .../ceph_s3_tests/test_ceph_s3_constants.js | 2 + src/test/system_tests/test_utils.js | 12 + .../test_nc_nsfs_account_cli.test.js | 20 +- .../test_nc_nsfs_anonymous_cli.test.js | 352 ++++++++++++++++++ src/test/unit_tests/nc_coretest.js | 3 +- src/test/unit_tests/test_bucketspace.js | 302 ++++++++++++++- src/test/unit_tests/test_s3_bucket_policy.js | 6 - src/util/native_fs_utils.js | 5 +- 23 files changed, 909 insertions(+), 50 deletions(-) create mode 100644 src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js diff --git a/config.js b/config.js index a2ce5dd5e1..75e623a271 100644 --- a/config.js +++ b/config.js @@ -826,6 +826,9 @@ config.NSFS_LOW_FREE_SPACE_MB_UNLEASH = 10 * 1024; // operations safely. config.NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH = 0.10; +// anonymous account name +config.ANONYMOUS_ACCOUNT_NAME = 'anonymous'; + //////////////////////////// // NSFS NON CONTAINERIZED // //////////////////////////// diff --git a/docs/non_containerized_NSFS.md b/docs/non_containerized_NSFS.md index 49213ddc27..508eadd74f 100644 --- a/docs/non_containerized_NSFS.md +++ b/docs/non_containerized_NSFS.md @@ -614,3 +614,60 @@ Access is restricted to these whitelist IPs. If there are no IPs mentioned all I ``` sudo noobaa-cli whitelist --ips '["127.0.0.1", "192.000.10.000", "3002:0bd6:0000:0000:0000:ee00:0033:000"]' 2>/dev/null ``` + +## Anonymous request + +Anonymous requests are those sent without access and secret key and Noobaa NSFS allow it with supporting bucket policy. Users have to create an anonymous account before accessing the resources. Noobaa uses anonymous account's uid and gid or user(distinguished_name) for accessing bucket storage paths. Noobaa can have one anonymous account. Commands for creating anonumous account are +``` +noobaa-cli account add --anonymous --uid {uid} --gid {gid} +``` +or +``` +noobaa-cli account add --anonymous --user {user} +``` + +Update : +``` +noobaa-cli account update --anonymous --uid {new_uid} --gid {new_gid} +``` +or + +``` +noobaa-cli account update --anonymous --user {user} +``` + +Delete: +``` +noobaa-cli account delete --anonymous +``` +Status +``` +noobaa-cli account status --anonymous +``` + +Bellow policy will allow anonymous user access and you can control the anonymous access using different policy configuration. + +``` +AWS_ACCESS_KEY_ID={access_key} AWS_SECRET_ACCESS_KEY={secret_key} aws s3api put-bucket-policy --endpoint {endpoint} --bucket {bucket_name} --policy file:///tmp/policy.json; +``` + +policy.json: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "AWS": [ "*" ] }, + "Action": [ "s3:*" ], + "Resource": [ "arn:aws:s3:::{bucket_name}/*", + "arn:aws:s3:::{bucket_name}" ] + } + ] +} +``` + +User can perform all the actions on above bucket resource without providing access and secret key(not recommended). + +`aws s3api list-objects --bucket {bucket_name} --endpoint {endpoint} --no-sign-request` \ No newline at end of file diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index 61470ee201..4088fdd825 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -20,7 +20,7 @@ const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils'); const { TYPES, ACTIONS, LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS, GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants'); const { throw_cli_error, write_stdout_response, get_config_file_path, get_symlink_config_file_path, - get_config_data, get_boolean_or_string_value} = require('../manage_nsfs/manage_nsfs_cli_utils'); + get_config_data, get_boolean_or_string_value, has_access_keys} = require('../manage_nsfs/manage_nsfs_cli_utils'); const { validate_input_types, validate_bucket_args, validate_account_args, verify_delete_account, validate_whitelist_arg, verify_whitelist_ips, _validate_access_keys } = require('../manage_nsfs/manage_nsfs_validations'); @@ -288,6 +288,10 @@ async function manage_bucket_operations(action, data, user_input) { async function account_management(action, user_input) { const show_secrets = get_boolean_or_string_value(user_input.show_secrets); + if (get_boolean_or_string_value(user_input.anonymous)) { + user_input.name = config.ANONYMOUS_ACCOUNT_NAME; + user_input.email = config.ANONYMOUS_ACCOUNT_NAME; + } const data = await fetch_account_data(action, user_input); await manage_account_operations(action, data, show_secrets, user_input); } @@ -315,7 +319,7 @@ function set_access_keys(access_key, secret_key, generate) { // in name and new_name we allow type number, hence convert it to string async function fetch_account_data(action, user_input) { - const { access_keys, new_access_key } = get_access_keys(action, user_input); + const { access_keys = [], new_access_key = undefined } = user_input.anonymous ? {} : get_access_keys(action, user_input); let data = { // added undefined values to keep the order the properties when printing the data object _id: undefined, @@ -342,11 +346,13 @@ async function fetch_account_data(action, user_input) { // override values // access_key as SensitiveString - data.access_keys[0].access_key = _.isUndefined(data.access_keys[0].access_key) ? undefined : + if (!has_access_keys(data.access_keys)) { + data.access_keys[0].access_key = _.isUndefined(data.access_keys[0].access_key) ? undefined : new SensitiveString(String(data.access_keys[0].access_key)); - // secret_key as SensitiveString - data.access_keys[0].secret_key = _.isUndefined(data.access_keys[0].secret_key) ? undefined : + // secret_key as SensitiveString + data.access_keys[0].secret_key = _.isUndefined(data.access_keys[0].secret_key) ? undefined : new SensitiveString(String(data.access_keys[0].secret_key)); + } // fs_backend deletion specified with empty string '' (but it is not part of the schema) data.nsfs_account_config.fs_backend = data.nsfs_account_config.fs_backend || undefined; // new_buckets_path deletion specified with empty string '' @@ -406,7 +412,7 @@ async function add_account(data) { await validate_account_args(data, ACTIONS.ADD); const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); - const access_key = data.access_keys[0].access_key; + const access_key = has_access_keys(data.access_keys) ? undefined : data.access_keys[0].access_key; const account_config_path = get_config_file_path(accounts_dir_path, data.name); const account_config_relative_path = get_config_file_path(acounts_dir_relative_path, data.name); const account_config_access_key_path = get_symlink_config_file_path(access_keys_dir_path, access_key); @@ -423,13 +429,17 @@ async function add_account(data) { const encrypted_account = await nc_mkm.encrypt_access_keys(data); data.master_key_id = encrypted_account.master_key_id; const encrypted_data = JSON.stringify(encrypted_account); + data = _.omitBy(data, _.isUndefined); // We take an object that was stringify // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters) // for validating against the schema we need an object, hence we parse it back to object - nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data)); - await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data); - await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR); - await nb_native().fs.symlink(fs_context, account_config_relative_path, account_config_access_key_path); + const account = encrypted_data ? JSON.parse(encrypted_data) : data; + nsfs_schema_utils.validate_account_schema(account); + await native_fs_utils.create_config_file(fs_context, accounts_dir_path, account_config_path, JSON.stringify(account)); + if (!has_access_keys(data.access_keys)) { + await native_fs_utils._create_path(access_keys_dir_path, fs_context, config.BASE_MODE_CONFIG_DIR); + await nb_native().fs.symlink(fs_context, account_config_relative_path, account_config_access_key_path); + } write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg }); } @@ -440,7 +450,7 @@ async function update_account(data) { const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); const cur_name = data.name; const new_name = data.new_name; - const cur_access_key = data.access_keys[0].access_key; + const cur_access_key = has_access_keys(data.access_keys) ? undefined : data.access_keys[0].access_key; const update_name = new_name && cur_name && data.new_name !== cur_name; const update_access_key = data.new_access_key && cur_access_key && data.new_access_key !== cur_access_key; @@ -449,11 +459,13 @@ async function update_account(data) { const encrypted_account = await nc_mkm.encrypt_access_keys(data); data.master_key_id = encrypted_account.master_key_id; const encrypted_data = JSON.stringify(encrypted_account); + data = _.omitBy(data, _.isUndefined); // We take an object that was stringify // (it unwraps ths sensitive strings, creation_date to string and removes undefined parameters) // for validating against the schema we need an object, hence we parse it back to object - nsfs_schema_utils.validate_account_schema(JSON.parse(encrypted_data)); - await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, encrypted_data); + const account = encrypted_data ? JSON.parse(encrypted_data) : data; + nsfs_schema_utils.validate_account_schema(account); + await native_fs_utils.update_config_file(fs_context, accounts_dir_path, account_config_path, JSON.stringify(account)); write_stdout_response(ManageCLIResponse.AccountUpdated, data); return; } @@ -502,10 +514,11 @@ async function delete_account(data) { const fs_context = native_fs_utils.get_process_fs_context(config_root_backend); const account_config_path = get_config_file_path(accounts_dir_path, data.name); - const access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key); - await native_fs_utils.delete_config_file(fs_context, accounts_dir_path, account_config_path); - await nb_native().fs.unlink(fs_context, access_key_config_path); + if (!has_access_keys(data.access_keys)) { + const access_key_config_path = get_symlink_config_file_path(access_keys_dir_path, data.access_keys[0].access_key); + await nb_native().fs.unlink(fs_context, access_key_config_path); + } write_stdout_response(ManageCLIResponse.AccountDeleted, '', {account: data.name}); } diff --git a/src/manage_nsfs/manage_nsfs_cli_errors.js b/src/manage_nsfs/manage_nsfs_cli_errors.js index ebf9c11664..79bbb05c05 100644 --- a/src/manage_nsfs/manage_nsfs_cli_errors.js +++ b/src/manage_nsfs/manage_nsfs_cli_errors.js @@ -119,6 +119,12 @@ ManageCLIError.InvalidFlagsCombination = Object.freeze({ http_code: 400, }); +ManageCLIError.InvalidAccountName = Object.freeze({ + code: 'InvalidAccountName', + message: 'Account name is invalid', + http_code: 400, +}); + ////////////////////////////// //// IP WHITE LIST ERRORS //// ////////////////////////////// diff --git a/src/manage_nsfs/manage_nsfs_cli_utils.js b/src/manage_nsfs/manage_nsfs_cli_utils.js index 186dcbbc78..e93ddbfa02 100644 --- a/src/manage_nsfs/manage_nsfs_cli_utils.js +++ b/src/manage_nsfs/manage_nsfs_cli_utils.js @@ -109,6 +109,15 @@ async function get_options_from_file(file_path) { } } +/** + * has_access_keys will return anonymous or not depending on the access key length + * (instead of flags) and return its content + * @param {object[]} access_keys + */ +function has_access_keys(access_keys) { + return access_keys.length === 0; +} + // EXPORTS exports.throw_cli_error = throw_cli_error; exports.write_stdout_response = write_stdout_response; @@ -118,3 +127,4 @@ exports.get_boolean_or_string_value = get_boolean_or_string_value; exports.get_config_data = get_config_data; exports.get_bucket_owner_account = get_bucket_owner_account; exports.get_options_from_file = get_options_from_file; +exports.has_access_keys = has_access_keys; diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js index 13fd235a1d..3373c085a5 100644 --- a/src/manage_nsfs/manage_nsfs_constants.js +++ b/src/manage_nsfs/manage_nsfs_constants.js @@ -31,6 +31,7 @@ const CONFIG_SUBDIRS = { const GLOBAL_CONFIG_ROOT = 'config_root'; const GLOBAL_CONFIG_OPTIONS = new Set([GLOBAL_CONFIG_ROOT, 'config_root_backend']); const FROM_FILE = 'from_file'; +const ANONYMOUS = 'anonymous'; const VALID_OPTIONS_ACCOUNT = { 'add': new Set(['name', 'uid', 'gid', 'new_buckets_path', 'user', 'access_key', 'secret_key', 'fs_backend', 'allow_bucket_creation', 'force_md5_etag', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]), @@ -40,6 +41,13 @@ const VALID_OPTIONS_ACCOUNT = { 'status': new Set(['name', 'access_key', 'show_secrets', ...GLOBAL_CONFIG_OPTIONS]), }; +const VALID_OPTIONS_ANONYMOUS_ACCOUNT = { + 'add': new Set(['uid', 'gid', 'user', 'anonymous', GLOBAL_CONFIG_ROOT]), + 'update': new Set(['uid', 'gid', 'user', 'anonymous', GLOBAL_CONFIG_ROOT]), + 'delete': new Set(['anonymous', GLOBAL_CONFIG_ROOT]), + 'status': new Set(['anonymous', GLOBAL_CONFIG_ROOT]), +}; + const VALID_OPTIONS_BUCKET = { 'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', FROM_FILE, ...GLOBAL_CONFIG_OPTIONS]), 'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', ...GLOBAL_CONFIG_OPTIONS]), @@ -64,6 +72,7 @@ const VALID_OPTIONS = { glacier_options: VALID_OPTIONS_GLACIER, whitelist_options: VALID_OPTIONS_WHITELIST, from_file_options: VALID_OPTIONS_FROM_FILE, + anonymous_account_options: VALID_OPTIONS_ANONYMOUS_ACCOUNT, }; const OPTION_TYPE = { @@ -88,7 +97,8 @@ const OPTION_TYPE = { wide: 'boolean', show_secrets: 'boolean', ips: 'string', - force: 'boolean' + force: 'boolean', + anonymous: 'boolean' }; const BOOLEAN_STRING_VALUES = ['true', 'false']; @@ -112,3 +122,4 @@ exports.LIST_UNSETABLE_OPTIONS = LIST_UNSETABLE_OPTIONS; exports.LIST_ACCOUNT_FILTERS = LIST_ACCOUNT_FILTERS; exports.LIST_BUCKET_FILTERS = LIST_BUCKET_FILTERS; +exports.ANONYMOUS = ANONYMOUS; diff --git a/src/manage_nsfs/manage_nsfs_validations.js b/src/manage_nsfs/manage_nsfs_validations.js index fb07e7f534..b8799f39de 100644 --- a/src/manage_nsfs/manage_nsfs_validations.js +++ b/src/manage_nsfs/manage_nsfs_validations.js @@ -13,9 +13,9 @@ const native_fs_utils = require('../util/native_fs_utils'); const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); const { throw_cli_error, get_config_file_path, get_bucket_owner_account, - get_config_data, get_options_from_file } = require('../manage_nsfs/manage_nsfs_cli_utils'); + get_config_data, get_options_from_file, has_access_keys } = require('../manage_nsfs/manage_nsfs_cli_utils'); const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES, - GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS } = require('../manage_nsfs/manage_nsfs_constants'); + GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS, ANONYMOUS } = require('../manage_nsfs/manage_nsfs_constants'); ///////////////////////////// //// GENERAL VALIDATIONS //// @@ -39,6 +39,7 @@ async function validate_input_types(type, action, argv) { validate_no_extra_options(type, action, input_options, false); validate_options_type_by_value(input_options_with_data); validate_flags_combination(type, action, input_options); + validate_flags_value_combination(type, action, input_options_with_data); if (action === ACTIONS.UPDATE) validate_min_flags_for_update(type, input_options_with_data); // currently we use from_file only in add action @@ -53,6 +54,7 @@ async function validate_input_types(type, action, argv) { validate_no_extra_options(type, action, input_options_from_file, true); validate_options_type_by_value(input_options_with_data_from_file); validate_flags_combination(type, action, input_options_from_file); + validate_flags_value_combination(type, action, input_options_with_data); return input_options_with_data_from_file; } } @@ -90,7 +92,11 @@ function validate_no_extra_options(type, action, input_options, is_options_from_ } else if (type === TYPES.BUCKET) { valid_options = VALID_OPTIONS.bucket_options[action]; } else if (type === TYPES.ACCOUNT) { - valid_options = VALID_OPTIONS.account_options[action]; + if (input_options.includes(ANONYMOUS)) { + valid_options = VALID_OPTIONS.anonymous_account_options[action]; + } else { + valid_options = VALID_OPTIONS.account_options[action]; + } } else if (type === TYPES.GLACIER) { valid_options = VALID_OPTIONS.glacier_options[action]; } else { @@ -102,7 +108,6 @@ function validate_no_extra_options(type, action, input_options, is_options_from_ valid_options.delete('config_root'); valid_options.delete('config_root_backend'); } - const invalid_input_options = input_options.filter(element => !valid_options.has(element)); if (invalid_input_options.length > 0) { const type_and_action = type === TYPES.IP_WHITELIST ? type : `${type} ${action}`; @@ -134,7 +139,7 @@ function validate_options_type_by_value(input_options_with_data) { continue; } // special case for boolean values - if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force', 'force_md5_etag'].includes(option) && validate_boolean_string_value(value)) { + if (['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force', 'force_md5_etag', 'anonymous'].includes(option) && validate_boolean_string_value(value)) { continue; } // special case for bucket_policy (from_file) @@ -174,7 +179,7 @@ function validate_min_flags_for_update(type, input_options_with_data) { // GAP - mandatory flags check should be earlier in the calls in general if (_.isUndefined(input_options_with_data.name)) { - if (type === TYPES.ACCOUNT) throw_cli_error(ManageCLIError.MissingAccountNameFlag); + if (type === TYPES.ACCOUNT && !input_options_with_data.anonymous) throw_cli_error(ManageCLIError.MissingAccountNameFlag); if (type === TYPES.BUCKET) throw_cli_error(ManageCLIError.MissingBucketNameFlag); } @@ -213,6 +218,24 @@ function validate_flags_combination(type, action, input_options) { } } +/** + * validate_flags_value_combination checks flags and value combination. + * 1. account add or update - name should not be anonymous + * @param {string} type + * @param {string} action + * @param {object} input_options_with_data + */ +function validate_flags_value_combination(type, action, input_options_with_data) { + if (type === TYPES.ACCOUNT) { + if (action === ACTIONS.ADD || action === ACTIONS.UPDATE) { + if (input_options_with_data.name === ANONYMOUS || input_options_with_data.new_name === ANONYMOUS) { + const detail = 'Account name \'anonymous\' is not valid'; + throw_cli_error(ManageCLIError.InvalidAccountName, detail); + } + } + } +} + ///////////////////////////// //// BUCKET VALIDATIONS ///// ///////////////////////////// @@ -300,7 +323,7 @@ async function validate_bucket_args(config_root_backend, accounts_dir_path, data */ async function validate_account_args(data, action) { if (action === ACTIONS.STATUS || action === ACTIONS.DELETE) { - if (_.isUndefined(data.access_keys[0].access_key) && _.isUndefined(data.name)) { + if (!has_access_keys(data.access_keys) && _.isUndefined(data.access_keys[0].access_key) && _.isUndefined(data.name)) { throw_cli_error(ManageCLIError.MissingIdentifier); } } else { @@ -308,8 +331,12 @@ async function validate_account_args(data, action) { if ((action !== ACTIONS.UPDATE && data.new_access_key)) throw_cli_error(ManageCLIError.InvalidNewAccessKeyIdentifier); if (_.isUndefined(data.name)) throw_cli_error(ManageCLIError.MissingAccountNameFlag); - if (_.isUndefined(data.access_keys[0].secret_key)) throw_cli_error(ManageCLIError.MissingAccountSecretKeyFlag); - if (_.isUndefined(data.access_keys[0].access_key)) throw_cli_error(ManageCLIError.MissingAccountAccessKeyFlag); + if (!has_access_keys(data.access_keys) && _.isUndefined(data.access_keys[0].secret_key)) { + throw_cli_error(ManageCLIError.MissingAccountSecretKeyFlag); + } + if (!has_access_keys(data.access_keys) && _.isUndefined(data.access_keys[0].access_key)) { + throw_cli_error(ManageCLIError.MissingAccountAccessKeyFlag); + } if (data.nsfs_account_config.gid && data.nsfs_account_config.uid === undefined) { throw_cli_error(ManageCLIError.MissingAccountNSFSConfigUID, data.nsfs_account_config); } diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index b5dadaf0b8..ba7e9ebf1c 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -22,6 +22,7 @@ const { CONFIG_SUBDIRS } = require('../manage_nsfs/manage_nsfs_constants'); const KeysSemaphore = require('../util/keys_semaphore'); const native_fs_utils = require('../util/native_fs_utils'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; +const { anonymous_access_key } = require('./object_sdk'); const dbg = require('../util/debug_module')(__filename); const bucket_semaphore = new KeysSemaphore(1); @@ -97,7 +98,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { async read_account_by_access_key({ access_key }) { try { if (!access_key) throw new Error('no access key'); - const iam_path = this._get_access_keys_config_path(access_key); + const iam_path = access_key === anonymous_access_key ? this._get_account_config_path(config.ANONYMOUS_ACCOUNT_NAME) : + this._get_access_keys_config_path(access_key); const { data } = await nb_native().fs.readFile(this.fs_context, iam_path); const account = JSON.parse(data.toString()); nsfs_schema_utils.validate_account_schema(account); @@ -702,6 +704,14 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { throw err; } } + + is_nsfs_containerized_user_anonymous(token) { + return !token && !process.env.NC_NSFS_NO_DB_ENV; + } + + is_nsfs_non_containerized_user_anonymous(token) { + return !token && process.env.NC_NSFS_NO_DB_ENV; + } } module.exports = BucketSpaceFS; diff --git a/src/sdk/bucketspace_nb.js b/src/sdk/bucketspace_nb.js index f1235ac6ab..2567b1fcd0 100644 --- a/src/sdk/bucketspace_nb.js +++ b/src/sdk/bucketspace_nb.js @@ -267,6 +267,14 @@ class BucketSpaceNB { throw err; } } + + is_nsfs_containerized_user_anonymous(token) { + return !token && !process.env.NC_NSFS_NO_DB_ENV; + } + + is_nsfs_non_containerized_user_anonymous(token) { + return !token && process.env.NC_NSFS_NO_DB_ENV === 'true'; + } } module.exports = BucketSpaceNB; diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index 8298fb350e..b82385acb0 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -848,6 +848,9 @@ interface BucketSpace { get_object_lock_configuration(params: object, object_sdk: ObjectSDK): Promise; put_object_lock_configuration(params: object, object_sdk: ObjectSDK): Promise; + + is_nsfs_containerized_user_anonymous(token: string): boolean; + is_nsfs_non_containerized_user_anonymous(token: string): boolean; } /********************************************************** diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index 523c40e81b..c933abb43b 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -26,6 +26,7 @@ const NamespaceNetStorage = require('./namespace_net_storage'); const BucketSpaceNB = require('./bucketspace_nb'); const { RpcError } = require('../rpc'); +const anonymous_access_key = Symbol('anonymous_access_key'); const bucket_namespace_cache = new LRUCache({ name: 'ObjectSDK-Bucket-Namespace-Cache', // This is intentional. Cache entry expiration is handled by _validate_bucket_namespace(). @@ -207,10 +208,10 @@ class ObjectSDK { async load_requesting_account(req) { try { const token = this.get_auth_token(); - if (!token) return; + if (this._get_bucketspace().is_nsfs_containerized_user_anonymous(token)) return; this.requesting_account = await account_cache.get_with_cache({ bucketspace: this._get_bucketspace(), - access_key: token.access_key, + access_key: token ? token.access_key : anonymous_access_key, }); if (this.requesting_account?.nsfs_account_config?.distinguished_name) { const distinguished_name = this.requesting_account.nsfs_account_config.distinguished_name.unwrap(); @@ -245,10 +246,12 @@ class ObjectSDK { } // check for a specific bucket if (bucket && req.op_name !== 'put_bucket') { - // ANONYMOUS: cannot work without bucket, cannot work on namespace bucket (?) + // ANONYMOUS: cannot work without bucket. + // Return if the acount is anonymous + if (this._get_bucketspace().is_nsfs_non_containerized_user_anonymous(token)) return; const ns = await this.read_bucket_sdk_namespace_info(bucket); if (!token) { - // TODO: Anonymous access to namespace buckets not supported + // TODO: Anonymous access to namespace buckets not supported for containerized Noobaa if (ns) { throw new RpcError('UNAUTHORIZED', `Anonymous access to namespace buckets not supported`); } else { @@ -256,7 +259,6 @@ class ObjectSDK { return; } } - if (!this.has_non_nsfs_bucket_access(this.requesting_account, ns)) { throw new RpcError('UNAUTHORIZED', `No permission to access bucket`); } @@ -1109,6 +1111,15 @@ class ObjectSDK { } } +// EXPORT +module.exports = { + ObjectSDK, + anonymous_access_key: anonymous_access_key, + account_cache: account_cache, + dn_cache: dn_cache, +}; + module.exports = ObjectSDK; +module.exports.anonymous_access_key = anonymous_access_key; module.exports.account_cache = account_cache; module.exports.dn_cache = dn_cache; diff --git a/src/server/system_services/schemas/nsfs_account_schema.js b/src/server/system_services/schemas/nsfs_account_schema.js index fb8f2a3b80..7937b24bed 100644 --- a/src/server/system_services/schemas/nsfs_account_schema.js +++ b/src/server/system_services/schemas/nsfs_account_schema.js @@ -12,7 +12,7 @@ module.exports = { 'nsfs_account_config', 'creation_date', 'allow_bucket_creation', - 'master_key_id' + 'master_key_id', ], properties: { _id: { diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt index ee02277623..f781f2a6dd 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt @@ -123,6 +123,7 @@ s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_arg s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions +s3tests_boto3/functional/test_s3.py::test_post_object_wrong_bucket s3tests_boto3/functional/test_s3.py::test_put_object_ifmatch_nonexisted_failed s3tests_boto3/functional/test_s3.py::test_object_raw_get_bucket_gone s3tests_boto3/functional/test_s3.py::test_object_raw_get @@ -194,7 +195,6 @@ s3tests_boto3/functional/test_s3.py::test_access_bucket_publicread_object_public s3tests_boto3/functional/test_s3.py::test_access_bucket_publicread_object_publicreadwrite s3tests_boto3/functional/test_s3.py::test_access_bucket_publicreadwrite_object_private s3tests_boto3/functional/test_s3.py::test_access_bucket_publicreadwrite_object_publicread -s3tests_boto3/functional/test_s3.py::test_list_buckets_anonymous s3tests_boto3/functional/test_s3.py::test_access_bucket_publicreadwrite_object_publicreadwrite s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt index 849f5a712d..600246849a 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt @@ -123,6 +123,7 @@ s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_arg s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions +s3tests_boto3/functional/test_s3.py::test_post_object_wrong_bucket s3tests_boto3/functional/test_s3.py::test_put_object_ifmatch_nonexisted_failed s3tests_boto3/functional/test_s3.py::test_object_raw_get_bucket_gone s3tests_boto3/functional/test_s3.py::test_object_raw_get diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js b/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js index 51a5408c32..b14dac1293 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js @@ -12,9 +12,12 @@ const dbg = require('../../../util/debug_module')(__filename); dbg.set_process_name('test_ceph_s3'); const os_utils = require('../../../util/os_utils'); -const { CEPH_TEST, account_path, account_tenant_path } = require('./test_ceph_s3_constants.js'); +const config = require('../../../../config'); +const mongo_utils = require('../../../util/mongo_utils'); +const { CEPH_TEST, account_path, account_tenant_path, anonymous_account_path } = require('./test_ceph_s3_constants.js'); const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager').get_instance(); + async function main() { try { await run_test(); @@ -65,6 +68,8 @@ async function ceph_test_setup() { await os_utils.exec(`sed -i -e 's:s3_access_key:${access_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); await os_utils.exec(`sed -i -e 's:s3_secret_key:${secret_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); } + // create anonymous account + await create_anonymous_account(); } @@ -77,6 +82,28 @@ async function get_access_keys(path) { return {access_key, secret_key}; } +// Create an anonymous account for anonymous request. Use this account UID and GID for bucket access. +async function create_anonymous_account() { + const nsfs_account_config = { + uid: process.getuid(), + gid: process.getgid(), + }; + const { master_key_id } = await nc_mkm.encrypt_access_keys({}); + const data = { + _id: mongo_utils.mongoObjectId(), + name: config.ANONYMOUS_ACCOUNT_NAME, + email: config.ANONYMOUS_ACCOUNT_NAME, + nsfs_account_config: nsfs_account_config, + access_keys: [], + allow_bucket_creation: false, + creation_date: new Date().toISOString(), + master_key_id: master_key_id, + }; + const account_data = JSON.stringify(data); + await fs.promises.writeFile(anonymous_account_path, account_data); + console.log('Anonymous account created'); +} + if (require.main === module) { main(); } diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js b/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js index 3f614b74ee..270b338de7 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js @@ -23,6 +23,7 @@ const CEPH_TEST = { // For NSFS NC path (using default values) const account_path = '/etc/noobaa.conf.d/accounts/cephalt.json'; const account_tenant_path = '/etc/noobaa.conf.d/accounts/cephtenant.json'; +const anonymous_account_path = '/etc/noobaa.conf.d/accounts/anonymous.json'; const DEFAULT_NUMBER_OF_WORKERS = 5; //5 was the number of workers in the previous CI/CD process @@ -32,6 +33,7 @@ const AWS4_TEST_SUFFIX = '_aws4'; exports.CEPH_TEST = CEPH_TEST; exports.account_path = account_path; +exports.anonymous_account_path = anonymous_account_path; exports.account_tenant_path = account_tenant_path; exports.DEFAULT_NUMBER_OF_WORKERS = DEFAULT_NUMBER_OF_WORKERS; exports.TOX_ARGS = TOX_ARGS; diff --git a/src/test/system_tests/test_utils.js b/src/test/system_tests/test_utils.js index 706a88eb94..9610221d9d 100644 --- a/src/test/system_tests/test_utils.js +++ b/src/test/system_tests/test_utils.js @@ -331,6 +331,17 @@ function set_nc_config_dir_in_config(config_root) { config.NSFS_NC_CONF_DIR = config_root; } +function generate_anon_s3_client(endpoint) { + return new S3({ + forcePathStyle: true, + region: config.DEFAULT_REGION, + signer: { sign: async request => request }, + requestHandler: new NodeHttpHandler({ + httpAgent: new http.Agent({ keepAlive: false }) + }), + endpoint + }); +} function generate_s3_client(access_key, secret_key, endpoint) { return new S3({ @@ -361,4 +372,5 @@ exports.create_fs_user_by_platform = create_fs_user_by_platform; exports.delete_fs_user_by_platform = delete_fs_user_by_platform; exports.set_path_permissions_and_owner = set_path_permissions_and_owner; exports.set_nc_config_dir_in_config = set_nc_config_dir_in_config; +exports.generate_anon_s3_client = generate_anon_s3_client; exports.TMP_PATH = TMP_PATH; diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js index 79001cb685..ae23874f2d 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js @@ -14,7 +14,7 @@ const nb_native = require('../../../util/nb_native'); const { set_path_permissions_and_owner, create_fs_user_by_platform, delete_fs_user_by_platform, TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); const { get_process_fs_context } = require('../../../util/native_fs_utils'); -const { TYPES, ACTIONS, CONFIG_SUBDIRS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const { TYPES, ACTIONS, CONFIG_SUBDIRS, ANONYMOUS } = require('../../../manage_nsfs/manage_nsfs_constants'); const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; @@ -401,6 +401,15 @@ describe('manage nsfs cli account flow', () => { const res = await exec_manage_cli(type, action, account_options); expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidFlagsCombination.code); }); + + it('should fail - cli account add invalid account name(anonymous)', async function() { + const action = ACTIONS.ADD; + const { type, uid, gid } = defaults; + const name = ANONYMOUS; + const account_options = { config_root, name, uid, gid }; + const res = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.InvalidAccountName.code); + }); }); describe('cli update account', () => { @@ -723,6 +732,15 @@ describe('manage nsfs cli account flow', () => { expect(account_details.nsfs_account_config.gid).toBeUndefined(); expect(account_details.nsfs_account_config.distinguished_name).toBe(distinguished_name); }); + + it('should fail - cli update account with invalid new_name(anonymous)', async () => { + const action = ACTIONS.UPDATE; + const { name } = defaults; + const new_name = ANONYMOUS; + const account_options = { config_root, name, new_name}; + const res = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(res.stdout).error.message).toBe(ManageCLIError.InvalidAccountName.message); + }); }); describe('cli update account (has distinguished name)', () => { diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js new file mode 100644 index 0000000000..61b1665b8d --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js @@ -0,0 +1,352 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + +// disabling init_rand_seed as it takes longer than the actual test execution +process.env.DISABLE_INIT_RANDOM_SEED = "true"; + +const _ = require('lodash'); +const path = require('path'); +const P = require('../../../util/promise'); +const os_util = require('../../../util/os_utils'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const { TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); +const { TYPES, ACTIONS, CONFIG_SUBDIRS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; + +const tmp_fs_path = path.join(TMP_PATH, 'test_nc_nsfs_anon_account_cli'); +const DEFAULT_FS_CONFIG = get_process_fs_context(); +const config = require('../../../../config'); + +// eslint-disable-next-line max-lines-per-function +describe('manage nsfs cli anonymous account flow', () => { + describe('cli create anonymous account', () => { + const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); + const defaults = { + _id: 'account1', + type: TYPES.ACCOUNT, + name: config.ANONYMOUS_ACCOUNT_NAME, + user: 'root', + anonymous: true, + uid: 999, + gid: 999, + }; + beforeAll(async () => { + await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => + fs_utils.create_fresh_path(`${config_root}/${dir}`))); + await fs_utils.create_fresh_path(root_path); + set_nc_config_dir_in_config(config_root); + config.NSFS_NC_CONF_DIR = config_root; + }); + + beforeAll(async () => { + await fs_utils.folder_delete(`${config_root}`); + await fs_utils.folder_delete(`${root_path}`); + }); + + it('cli create anonymous account', async () => { + const action = ACTIONS.ADD; + const { type, uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid }; + await exec_manage_cli(type, action, account_options); + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + assert_account(account, account_options); + }); + + it('Should fail - cli create anonymous account again', async () => { + const action = ACTIONS.ADD; + const { type, uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid }; + const resp = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.AccountNameAlreadyExists.message); + }); + + it('Should fail - cli create anonymous account with invalid action', async () => { + const { type, uid, gid } = defaults; + const account_options = { config_root, uid, gid }; + const resp = await exec_manage_cli(type, 'reload', account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidAction.message); + }); + + it('Should fail - cli create anonymous account with name(not a valid argument)', async () => { + const action = ACTIONS.ADD; + const { type, name, uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid, name }; + const resp = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidArgument.message); + }); + + it('Should fail - cli create anonymous account with invalid type', async () => { + const action = ACTIONS.ADD; + const { uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid }; + const resp = await exec_manage_cli('account_anonymous', action, account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidType.message); + }); + + it('Should fail - cli create anonymous account with invalid uid', async () => { + const action = ACTIONS.ADD; + const { type, gid, anonymous } = defaults; + const uid_str = '0'; + const account_options = { anonymous, config_root, uid_str, gid }; + const resp = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidArgument.message); + }); + + it('Should fail - cli create anonymous account with invalid gid', async () => { + const action = ACTIONS.ADD; + const { type, uid, anonymous } = defaults; + const gid_str = '0'; + const account_options = { anonymous, config_root, uid, gid_str }; + const resp = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidArgument.message); + }); + + it('cli create anonymous account with distinguished_name', async () => { + let action = ACTIONS.DELETE; + const { type, user, anonymous } = defaults; + let account_options = { anonymous, config_root }; + let resp = await exec_manage_cli(type, action, account_options); + const res_json = JSON.parse(resp.trim()); + expect(res_json.response.code).toBe(ManageCLIResponse.AccountDeleted.code); + action = ACTIONS.ADD; + account_options = { anonymous, config_root, user }; + resp = await exec_manage_cli(type, action, account_options); + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + assert_account(account, account_options); + }); + + it('Should fail - cli create anonymous account with invalid user', async () => { + const action = ACTIONS.ADD; + const { type, anonymous } = defaults; + const user_int = 0; + const account_options = { anonymous, config_root, user_int }; + const resp = await exec_manage_cli(type, action, account_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidArgument.message); + }); + }); + + describe('cli update anonymous account', () => { + const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); + const defaults = { + _id: 'account2', + type: TYPES.ACCOUNT, + name: config.ANONYMOUS_ACCOUNT_NAME, + user: 'root', + anonymous: true, + uid: 999, + gid: 999, + }; + beforeAll(async () => { + await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => + fs_utils.create_fresh_path(`${config_root}/${dir}`))); + await fs_utils.create_fresh_path(root_path); + set_nc_config_dir_in_config(config_root); + config.NSFS_NC_CONF_DIR = config_root; + }); + + beforeAll(async () => { + await fs_utils.folder_delete(`${config_root}`); + await fs_utils.folder_delete(`${root_path}`); + }); + + it('cli update anonymous account', async () => { + let action = ACTIONS.ADD; + let { type, uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid }; + await exec_manage_cli(type, action, account_options); + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + assert_account(account, account_options); + + action = ACTIONS.UPDATE; + gid = 1001; + const account_update_options = { anonymous, config_root, uid, gid }; + await exec_manage_cli(type, action, account_update_options); + const update_account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + assert_account(update_account, account_update_options); + }); + + it('should fail - cli update anonymous account with string gid', async () => { + const action = ACTIONS.UPDATE; + const { type, uid, anonymous } = defaults; + const gid = 'str'; + const account_update_options = { anonymous, config_root, uid, gid }; + const resp = await exec_manage_cli(type, action, account_update_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidArgumentType.message); + }); + + it('should fail - cli update anonymous account with invalid user', async () => { + const action = ACTIONS.UPDATE; + const { type, anonymous } = defaults; + const user = 0; + const account_update_options = { anonymous, config_root, user }; + const resp = await exec_manage_cli(type, action, account_update_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.InvalidArgumentType.message); + }); + }); + + describe('cli delete anonymous account', () => { + const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); + const defaults = { + _id: 'account2', + type: TYPES.ACCOUNT, + name: config.ANONYMOUS_ACCOUNT_NAME, + user: 'root', + anonymous: true, + uid: 999, + gid: 999, + }; + beforeAll(async () => { + await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => + fs_utils.create_fresh_path(`${config_root}/${dir}`))); + await fs_utils.create_fresh_path(root_path); + set_nc_config_dir_in_config(config_root); + config.NSFS_NC_CONF_DIR = config_root; + }); + + beforeAll(async () => { + await fs_utils.folder_delete(`${config_root}`); + await fs_utils.folder_delete(`${root_path}`); + }); + + it('cli delete anonymous account', async () => { + let action = ACTIONS.ADD; + const { type, uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid }; + await exec_manage_cli(type, action, account_options); + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + assert_account(account, account_options); + + action = ACTIONS.DELETE; + const account_delete_options = { anonymous, config_root }; + const resp = await exec_manage_cli(type, action, account_delete_options); + const res_json = JSON.parse(resp.trim()); + expect(res_json.response.code).toBe(ManageCLIResponse.AccountDeleted.code); + }); + + it('should fail - Anonymous account try to delete again ', async () => { + const action = ACTIONS.DELETE; + const { type, anonymous } = defaults; + const account_delete_options = { anonymous, config_root }; + const resp = await exec_manage_cli(type, action, account_delete_options); + expect(JSON.parse(resp.stdout).error.message).toBe(ManageCLIError.NoSuchAccountName.message); + }); + }); + + describe('cli status anonymous account', () => { + const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); + const defaults = { + _id: 'account4', + type: TYPES.ACCOUNT, + name: config.ANONYMOUS_ACCOUNT_NAME, + user: 'root', + anonymous: true, + uid: 999, + gid: 999, + }; + beforeAll(async () => { + await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => + fs_utils.create_fresh_path(`${config_root}/${dir}`))); + await fs_utils.create_fresh_path(root_path); + set_nc_config_dir_in_config(config_root); + config.NSFS_NC_CONF_DIR = config_root; + }); + + beforeAll(async () => { + await fs_utils.folder_delete(`${config_root}`); + await fs_utils.folder_delete(`${root_path}`); + }); + + it('cli status anonymous account', async () => { + let action = ACTIONS.ADD; + const { type, uid, gid, anonymous } = defaults; + const account_options = { anonymous, config_root, uid, gid }; + await exec_manage_cli(type, action, account_options); + const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + assert_account(account, account_options); + + action = ACTIONS.STATUS; + const account_delete_options = { anonymous, config_root }; + const resp = await exec_manage_cli(type, action, account_delete_options); + const res_json = JSON.parse(resp.trim()); + assert_account(res_json.response.reply, account_options); + }); + }); + +}); + +/** + * read_config_file will read the config files + * @param {string} config_root + * @param {string} schema_dir + * @param {string} config_file_name the name of the config file + * @param {boolean} [is_symlink] a flag to set the suffix as a symlink instead of json + */ +async function read_config_file(config_root, schema_dir, config_file_name, is_symlink) { + const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? '.symlink' : '.json')); + const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); + const config_data = JSON.parse(data.toString()); + return config_data; +} + +/** + * assert_account will verify the fields of the accounts + * @param {object} account + * @param {object} account_options + */ +function assert_account(account, account_options) { + expect(account.name).toEqual(config.ANONYMOUS_ACCOUNT_NAME); + + if (account_options.distinguished_name) { + expect(account.nsfs_account_config.distinguished_name).toEqual(account_options.distinguished_name); + } else { + expect(account.nsfs_account_config.uid).toEqual(account_options.uid); + expect(account.nsfs_account_config.gid).toEqual(account_options.gid); + } +} + +/** + * exec_manage_cli will get the flags for the cli and runs the cli with it's flags + * @param {string} type + * @param {string} action + * @param {object} options + */ +async function exec_manage_cli(type, action, options) { + const command = create_command(type, action, options); + let res; + try { + res = await os_util.exec(command, { return_stdout: true }); + } catch (e) { + res = e; + } + return res; +} + +/** + * create_command would create the string needed to run the CLI command + * @param {string} type + * @param {string} action + * @param {object} options + */ +function create_command(type, action, options) { + let account_flags = ``; + for (const key in options) { + if (Object.hasOwn(options, key)) { + if (typeof options[key] === 'boolean') { + account_flags += `--${key} `; + } else { + account_flags += `--${key} ${options[key]} `; + } + } + } + account_flags = account_flags.trim(); + + const command = `node src/cmd/manage_nsfs ${type} ${action} ${account_flags}`; + return command; +} diff --git a/src/test/unit_tests/nc_coretest.js b/src/test/unit_tests/nc_coretest.js index 3d7eeb97dd..8e910c716a 100644 --- a/src/test/unit_tests/nc_coretest.js +++ b/src/test/unit_tests/nc_coretest.js @@ -15,6 +15,7 @@ const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); const NC_CORETEST = 'nc_coretest'; const config_dir_name = 'nc_coretest_config_root_path'; const master_key_location = `${TMP_PATH}/${config_dir_name}/master_keys.json`; +const NC_CORETEST_CONFIG_DIR_PATH = `${TMP_PATH}/${config_dir_name}`; process.env.DEBUG_MODE = 'true'; process.env.ACCOUNTS_CACHE_EXPIRY = '1'; process.env.NC_CORETEST = 'true'; @@ -49,7 +50,6 @@ const http_address = `http://localhost:${http_port}`; const https_address = `https://localhost:${https_port}`; const FIRST_BUCKET = 'first.bucket'; -const NC_CORETEST_CONFIG_DIR_PATH = p.join(TMP_PATH, config_dir_name); const NC_CORETEST_REDIRECT_FILE_PATH = p.join(config.NSFS_NC_DEFAULT_CONF_DIR, '/config_dir_redirect'); const NC_CORETEST_STORAGE_PATH = p.join(TMP_PATH, '/nc_coretest_storage_root_path/'); const FIRST_BUCKET_PATH = p.join(NC_CORETEST_STORAGE_PATH, FIRST_BUCKET, '/'); @@ -471,3 +471,4 @@ exports.rpc_client = rpc_cli_funcs_to_manage_nsfs_cli_cmds; exports.get_http_address = get_http_address; exports.get_https_address = get_https_address; exports.get_admin_account_details = get_admin_account_details; +exports.NC_CORETEST_CONFIG_DIR_PATH = NC_CORETEST_CONFIG_DIR_PATH; diff --git a/src/test/unit_tests/test_bucketspace.js b/src/test/unit_tests/test_bucketspace.js index 51e113927b..28f4fa0b04 100644 --- a/src/test/unit_tests/test_bucketspace.js +++ b/src/test/unit_tests/test_bucketspace.js @@ -1,4 +1,5 @@ /* Copyright (C) 2020 NooBaa */ +/*eslint max-lines: ["error", 2200]*/ /*eslint max-lines-per-function: ["error", 900]*/ /*eslint max-statements: ["error", 80, { "ignoreTopLevelFunctions": true }]*/ 'use strict'; @@ -20,15 +21,19 @@ const { TYPES } = require('../../manage_nsfs/manage_nsfs_constants'); const ManageCLIError = require('../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const { TMP_PATH, get_coretest_path, invalid_nsfs_root_permissions, generate_s3_policy, create_fs_user_by_platform, delete_fs_user_by_platform, - generate_s3_client, exec_manage_cli } = require('../system_tests/test_utils'); + generate_s3_client, exec_manage_cli, generate_anon_s3_client } = require('../system_tests/test_utils'); +const nc_mkm = require('../../manage_nsfs/nc_master_key_manager').get_instance(); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); +const native_fs_utils = require('../../util/native_fs_utils'); +const mongo_utils = require('../../util/mongo_utils'); +const accounts_dir_name = '/accounts'; const coretest_path = get_coretest_path(); const coretest = require(coretest_path); -const { rpc_client, EMAIL, PASSWORD, SYSTEM, get_admin_account_details } = coretest; +const { rpc_client, EMAIL, PASSWORD, SYSTEM, get_admin_account_details, NC_CORETEST_CONFIG_DIR_PATH } = coretest; coretest.setup({}); let CORETEST_ENDPOINT; const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); @@ -1727,7 +1732,6 @@ mocha.describe('s3 whitelist flow', async function() { const { access_key, secret_key } = res.access_keys[0]; CORETEST_ENDPOINT = coretest.get_http_address(); s3_client = generate_s3_client(access_key.unwrap(), secret_key.unwrap(), CORETEST_ENDPOINT); - }); mocha.after(async function() { @@ -1760,6 +1764,264 @@ mocha.describe('s3 whitelist flow', async function() { }); }); +/*eslint max-lines-per-function: ["error", 1300]*/ +mocha.describe('Namespace s3_bucket_policy', function() { + this.timeout(600000); // eslint-disable-line no-invalid-this + const anon_access_policy = { + Version: '2012-10-17', + Statement: [{ + Effect: 'Allow', + Principal: { AWS: ["*"] }, + Action: ['s3:GetObject', 's3:ListBucket'], + Resource: ['arn:aws:s3:::*'] + }] + }; + const KEY = 'file1.txt'; + const tmp_fs_root = path.join(TMP_PATH, 'test_bucket_namespace_fs'); + const ns_s3_buckets_dir = 'ns_s3_buckets_dir'; + const s3_new_ns_buckets_path = path.join(tmp_fs_root, ns_s3_buckets_dir); + const bucket_name = 'ns-src-bucket-s3'; + let s3_client; + let s3_anon_client; + const nsr = 'nsr'; + let accounts_dir_path; + let account_config_path; + if (process.env.NC_CORETEST) { + accounts_dir_path = path.join(NC_CORETEST_CONFIG_DIR_PATH, accounts_dir_name); + account_config_path = path.join(accounts_dir_path, config.ANONYMOUS_ACCOUNT_NAME + '.json'); + } + + mocha.before(async function() { + await fs_utils.create_fresh_path(tmp_fs_root); + await fs_utils.create_fresh_path(s3_new_ns_buckets_path); + + const account_s3 = await rpc_client.account.create_account({ + ...new_account_params, + email: 'account_ns_s3@noobaa.com', + name: 'account_ns_s3', + s3_access: true, + default_resource: nsr, + nsfs_account_config: { + uid: process.getuid(), + gid: process.getgid(), + new_buckets_path: ns_s3_buckets_dir, + nsfs_only: true + } + }); + const { access_key, secret_key } = account_s3.access_keys[0]; + s3_client = generate_s3_client(access_key.unwrap(), secret_key.unwrap(), CORETEST_ENDPOINT); + s3_anon_client = generate_anon_s3_client(CORETEST_ENDPOINT); + }); + mocha.after(async () => { + await fs_utils.file_delete(path.join(s3_new_ns_buckets_path, KEY)); + await fs_utils.folder_delete(s3_new_ns_buckets_path); + await fs_utils.folder_delete(tmp_fs_root); + }); + mocha.it('Missing anonymous account - user should not be able to list bucket objects', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + await s3_client.createBucket({ Bucket: bucket_name }); + await fs_utils.file_must_exist(path.join(s3_new_ns_buckets_path, bucket_name)); + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_access_policy) + }); + await s3_client.getBucketPolicy({ + Bucket: bucket_name, + }); + await s3_client.listObjects({ Bucket: bucket_name}); + await assert_throws_async(s3_anon_client.listObjects({ Bucket: bucket_name }), 'The AWS access key Id you provided does not exist in our records.'); + }); + + mocha.it('Namesapce - anonymous user should be able to list bucket objects', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + await fs_utils.file_must_exist(path.join(s3_new_ns_buckets_path, bucket_name)); + // Create anonymous account + const nsfs_account_config = { + uid: process.getuid(), + gid: process.getgid(), + }; + await add_anonymous_account(nsfs_account_config, accounts_dir_path, account_config_path); + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_access_policy) + }); + await s3_client.getBucketPolicy({ + Bucket: bucket_name, + }); + await s3_client.listObjects({ Bucket: bucket_name}); + await s3_anon_client.listObjects({ Bucket: bucket_name}); + }); + + mocha.it('anonymous user should not be able to list bucket objects when there is no policy', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + await s3_client.deleteBucketPolicy({ + Bucket: bucket_name, + }); + await assert_throws_async(s3_anon_client.listObjects({ Bucket: bucket_name })); + }); + + mocha.it('anonymous user should not be able to list bucket objects when policy doesn\'t allow', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + const anon_deny_policy = { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Deny', + Principal: { AWS: "*" }, + Action: ['s3:GetObject', 's3:ListBucket'], + Resource: [`arn:aws:s3:::*`] + }, + ] + }; + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_deny_policy) + }); + await assert_throws_async(s3_anon_client.listObjects({ Bucket: bucket_name })); + }); + + mocha.it('anonymous user should not be able to getObject when not explicitly allowed', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + // Ensure that the bucket has no policy + await s3_client.deleteBucketPolicy({ + Bucket: bucket_name, + }); + await assert_throws_async(s3_anon_client.getObject({ + Bucket: bucket_name, + Key: KEY, + })); + }); + + mocha.it('anonymous user should not be able to read_object_md when explicitly allowed to access only specific key', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + const anon_read_policy_2 = { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: "*"}, + Action: ['s3:GetObject'], + Resource: [`arn:aws:s3:::${bucket_name}/${KEY}`] + }, + ] + }; + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_read_policy_2) + }); + await assert_throws_async(s3_anon_client.getObject({ + Bucket: bucket_name, + Key: KEY + "/", + })); + }); + + + mocha.it('anonymous user should not be able to putObject when not explicitly allowed', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + // Ensure that the bucket has no policy + await s3_client.deleteBucketPolicy({ + Bucket: bucket_name, + }); + const body1 = 'AAAAABBBBBCCCCC'; + await assert_throws_async(s3_anon_client.putObject({ + Bucket: bucket_name, + Key: KEY, + Body: body1, + })); + }); + + mocha.it('anonymous user should be able to putBucketPolicy when explicitly allowed', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + const anon_read_policy = { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: "*" }, + Action: ['s3:GetObject', 's3:PutObject'], + Resource: [`arn:aws:s3:::*`] + }, + ] + }; + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_read_policy) + }); + const body1 = 'AAAAABBBBBCCCCC'; + s3_anon_client.putObject({ + Bucket: bucket_name, + Key: KEY, + Body: body1, + }); + }); + + mocha.it('anonymous user should be able to putBucketPolicy when explicitly allowed to access only specific key', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + const anon_read_policy_2 = { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: "*" + }, + Action: ['s3:GetObject'], + Resource: [`arn:aws:s3:::${bucket_name}/${KEY}`] + }, + ] + }; + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_read_policy_2) + }); + + await assert_throws_async(s3_anon_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_read_policy_2), + })); + }); + + mocha.it('Namesapce - distinguished_name - Anonymous user should be able to list bucket objects', async function() { + // Skipping because only NC NSFS will have anonymous account. + if (!process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this + await fs_utils.file_must_exist(path.join(s3_new_ns_buckets_path, bucket_name)); + await delete_anonymous_account(accounts_dir_path, account_config_path); + // Create anonymous account with distinguished_name + const nsfs_account_config = { + distinguished_name: 'root', + }; + await add_anonymous_account(nsfs_account_config, accounts_dir_path, account_config_path); + await s3_client.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(anon_access_policy) + }); + await s3_client.getBucketPolicy({ + Bucket: bucket_name, + }); + await s3_client.listObjects({ Bucket: bucket_name}); + await s3_anon_client.listObjects({ Bucket: bucket_name}); + }); + +}); + +async function assert_throws_async(promise, expected_message = 'Access Denied') { + try { + await promise; + assert.fail('Test was suppose to fail on ' + expected_message); + } catch (err) { + if (err.message !== expected_message) { + throw err; + } + } +} async function generate_nsfs_account(options = {}) { const { uid, gid, new_buckets_path, nsfs_only, admin, default_resource, account_name } = options; @@ -1862,3 +2124,37 @@ async function update_account_nsfs_config(email, default_resource, new_nsfs_acco } } +// Create an anonymous account for anonymous request. Use this account UID and GID for bucket access. +async function add_anonymous_account(nsfs_account_config, accounts_dir_path, account_config_path) { + const { master_key_id } = await nc_mkm.encrypt_access_keys({}); + const data = { + _id: mongo_utils.mongoObjectId(), + name: config.ANONYMOUS_ACCOUNT_NAME, + email: config.ANONYMOUS_ACCOUNT_NAME, + nsfs_account_config: nsfs_account_config, + access_keys: [], + allow_bucket_creation: false, + creation_date: new Date().toISOString(), + master_key_id: master_key_id, + }; + const account_data = JSON.stringify(data); + const name_exists = await native_fs_utils.is_path_exists(DEFAULT_FS_CONFIG, account_config_path); + if (name_exists) { + console.warn('Error: Anonymous account already exist.'); + return; + } + await native_fs_utils.create_config_file(DEFAULT_FS_CONFIG, accounts_dir_path, account_config_path, account_data); + console.log('Anonymous account created'); +} + +// Delete an anonymous account for anonymous request. +async function delete_anonymous_account(accounts_dir_path, account_config_path) { + const name_exists = await native_fs_utils.is_path_exists(DEFAULT_FS_CONFIG, account_config_path); + if (!name_exists) { + console.warn('Error: Anonymous account do not exist.'); + return; + } + await native_fs_utils.delete_config_file(DEFAULT_FS_CONFIG, accounts_dir_path, account_config_path); + console.log('Anonymous account Deleted'); +} + diff --git a/src/test/unit_tests/test_s3_bucket_policy.js b/src/test/unit_tests/test_s3_bucket_policy.js index ffc592103f..aa02c3bec5 100644 --- a/src/test/unit_tests/test_s3_bucket_policy.js +++ b/src/test/unit_tests/test_s3_bucket_policy.js @@ -520,8 +520,6 @@ mocha.describe('s3_bucket_policy', function() { }); mocha.it('anonymous user should be able to list bucket objects', async function() { - // anonymous not implemented on NC yet - skipping - if (process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this await s3_owner.putBucketPolicy({ Bucket: BKT, Policy: JSON.stringify(anon_access_policy) @@ -531,8 +529,6 @@ mocha.describe('s3_bucket_policy', function() { }); mocha.it('anonymous user should not be able to list bucket objects when there is no policy', async function() { - // anonymous not implemented on NC yet - skipping - if (process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this await s3_owner.deleteBucketPolicy({ Bucket: BKT, }); @@ -541,8 +537,6 @@ mocha.describe('s3_bucket_policy', function() { }); mocha.it('anonymous user should not be able to list bucket objects when policy doesn\'t allow', async function() { - // anonymous not implemented on NC yet - skipping - if (process.env.NC_CORETEST) this.skip(); // eslint-disable-line no-invalid-this const anon_deny_policy = { Version: '2012-10-17', Statement: [ diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js index 575fd8f400..ebbaa865e6 100644 --- a/src/util/native_fs_utils.js +++ b/src/util/native_fs_utils.js @@ -406,10 +406,7 @@ async function update_config_file(fs_context, schema_dir, config_path, config_da async function get_user_by_distinguished_name({ distinguished_name }) { try { if (!distinguished_name) throw new Error('no distinguished name'); - const context = { - uid: process.getuid(), - gid: process.getgid(), - }; + const context = get_process_fs_context(); const user = await nb_native().fs.getpwname(context, distinguished_name); return user; } catch (err) {