From 1b0624ea7ddde5b7a2bf56075d0f9be70375ae91 Mon Sep 17 00:00:00 2001 From: Georgy Moiseev Date: Wed, 27 Mar 2024 14:08:57 +0300 Subject: [PATCH] [WIP] crud: introduce basic tarantool 3 roles This patch introduce basic crud-router and crud-storage roles for Tarantool 3. The roles are similar to Cartridge ones. Roles support Tarantool 3.0.2, Tarantool 3.1.0 and newer due to [1, 2]. This commit does not yet introduce metrics configuration through roles config. [1] https://github.com/tarantool/tarantool/issues/9643 [2] https://github.com/tarantool/tarantool/issues/9649 Part of #415 --- .github/workflows/test_on_push.yaml | 2 + README.md | 293 ++++++++++++++++++++--- crud/common/roles.lua | 20 ++ crud/common/utils.lua | 37 +-- roles/crud-router.lua | 33 +++ roles/crud-storage.lua | 34 +++ test/helper.lua | 16 +- test/performance/perf_test.lua | 2 + test/tarantool3_helpers/cluster.lua | 41 +--- test/tarantool3_helpers/cluster_test.lua | 5 +- test/tarantool3_helpers/utils.lua | 25 ++ test/unit/not_initialized_test.lua | 1 - 12 files changed, 427 insertions(+), 82 deletions(-) create mode 100644 crud/common/roles.lua create mode 100644 roles/crud-router.lua create mode 100644 roles/crud-storage.lua diff --git a/.github/workflows/test_on_push.yaml b/.github/workflows/test_on_push.yaml index 21ec9e1b..0d8f7ca5 100644 --- a/.github/workflows/test_on_push.yaml +++ b/.github/workflows/test_on_push.yaml @@ -43,6 +43,8 @@ jobs: external-keydef-version: "0.0.4" - tarantool-version: "3.0.0" vshard-version: "0.1.25" + - tarantool-version: "master" + vshard-version: "0.1.26" fail-fast: false # Can't install older versions on 22.04, # see https://github.com/tarantool/setup-tarantool/issues/36 diff --git a/README.md b/README.md index fc35f3e9..5a3daffe 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,10 @@ It also provides the `crud-storage` and `crud-router` roles for - [Read view select conditions](#read-view-select-conditions) - [Read view pairs](#read-view-pairs) - [Schema](#schema) -- [Cartridge roles](#cartridge-roles) +- [Tarantool 3 roles](#tarantool-3-roles) - [Usage](#usage) +- [Cartridge roles](#cartridge-roles) + - [Usage](#usage-1) - [License](#license) @@ -54,38 +56,84 @@ It also provides the `crud-storage` and `crud-router` roles for First, [install Tarantool](https://www.tarantool.io/en/download). -Now you have the following options for learning the crud API and using it in a -project: - -* Play with crud on a test dataset on a single instance: - - ```shell - $ git clone https://github.com/tarantool/crud.git - $ cd crud - $ tt rocks make - $ ./doc/playground.lua - tarantool> crud.select('customers', {{'<=', 'age', 35}}, {first = 10}) - tarantool> crud.select('developers', nil, {first = 6}) - ``` -* Install crud into the current directory: - - ```shell - $ tt rocks install crud - ``` -* Add the [crud initialization code](#API) to router and storage instances - initialization code for [VShard](https://github.com/tarantool/vshard). -* Add crud into dependencies of a Cartridge application and add crud roles into - dependencies of your roles (see [Cartridge roles](#cartridge-roles) section). -* Add crud into dependencies of your application (rockspec, RPM spec -- depends - on your choice) and call crud initialization code from storage and router - code (see [API](#api) section). +### Install + +#### Manual install + +To try `crud` in your application, you may install it manually fron web +with `tt rocks` rock management tool. + +```bash +tt rocks install crud +``` + +#### Application dependency + +To use crud in your application, set it as a rockspec dependency. + +```lua +package = 'myapp' + +version = 'scm-1' + +source = { + url = '/dev/null', +} + +dependencies = { + 'tarantool >= 3.1.0', + 'crud >= -1', +} + +build = { + type = 'none'; +} +``` + +#### Repository clone + +You can also clone the repository to explore crud and try it inside a sandbox. + +```bash +git clone https://github.com/tarantool/crud.git +cd crud +tt rocks make +``` + +### Usage + +For Tarantool 3.x, enable crud roles on your application instances in a configuration +(see [Tarantool 3 roles](#tarantool-3-roles) section). +Roles support Tarantool 3.0.2, Tarantool 3.1.0 and newer. +Older versions are not supported due to +[tarantool/tarantool#9643](https://github.com/tarantool/tarantool/issues/9643) and +[tarantool/tarantool#9649](https://github.com/tarantool/tarantool/issues/9649) +issues. + +For Tarantool 1.10 and 2.x, add crud roles into dependencies of your roles +(see [Cartridge roles](#cartridge-roles) section). + +For Tarantool 1.10, 2.x and 3.x you can also manually call +the [crud initialization code](#API) on [VShard](https://github.com/tarantool/vshard) +router and storage instances. + +### Sandbox + +The repository provide a simple sandbox application with a test dataset on a single instance. + +```bash +./doc/playground.lua +tarantool> crud.select('customers', {{'<=', 'age', 35}}, {first = 10}) +tarantool> crud.select('developers', nil, {first = 6}) +``` ## API The CRUD operations should be called from router. All VShard storages should call `crud.init_storage()` after -`vshard.storage.cfg()` (or enable the `crud-storage` role for Cartridge) +`vshard.storage.cfg()` (or enable the `roles.crud-storage` role for Tarantool 3 +or the `crud-storage` role for Cartridge) first to initialize storage-side functions that are used to manipulate data across the cluster. The storage-side functions have the same access as a user calling `crud.init_storage()`. Therefore, if `crud` do not have @@ -98,7 +146,8 @@ asynchronous bootstrap is used for Tarantool 3.x and synchronous bootstrap is used for Tarantool 1.10 and 2.x. All VShard routers should call `crud.init_router()` after `vshard.router.cfg()` -(or enable the `crud-router` role for Cartridge) to make `crud` functions +(or enable the `roles.crud-storage` role for Tarantool 3 +or the `crud-router` role for Cartridge) to make `crud` functions callable via `net.box`. If a user is allowed to execute `crud` functions on the router-side then the user does not need additional access on storages. @@ -1833,6 +1882,194 @@ crud.schema() indexes: ... ``` +## Tarantool 3 roles + +`roles.crud-storage` is a Tarantool 3 role that initializes functions that +are used on the storage side to perform CRUD operations. Role must be enabled +on sharding storages. + +`cartridge.roles.crud-router` is a role that exposes public `crud` functions +to the global scope so that you can call them via `net.box` or with connectors. +Role must be enabled on sharding routers. + +Roles support Tarantool 3.0.2, Tarantool 3.1.0 and newer. +Older versions are not supported due to +[tarantool/tarantool#9643](https://github.com/tarantool/tarantool/issues/9643) and +[tarantool/tarantool#9649](https://github.com/tarantool/tarantool/issues/9649) +issues. + +### Usage + +1. Add `crud` to dependencies in the project rockspec. + + **Note**: it's better to use tagged version than `scm-1`. + Check the latest available [release](https://github.com/tarantool/crud/releases) tag and use it. + + ```lua + -- -scm-1.rockspec + dependencies = { + ... + 'crud == -1', + ... + } + ``` + +2. Add crud roles to your application configuration. + Application must be a sharded one. + It is required that `roles.crud-storage` is enabled on each + sharding storage. + + ```yaml + groups: + routers: + sharding: + roles: + - router + roles: + - roles.crud-router + replicasets: + router: + + storages: + sharding: + roles: + - storage + roles: + - roles.crud-storage + replicasets: + s-1: + s-2: + ``` + +
+ Full configuration example + + ```yaml + credentials: + users: + replicator: + password: replicating + roles: + - replication + storage: + password: storing-buckets + roles: + - sharding + guest: + roles: + - super + + sharding: + bucket_count: 30000 + + replication: + failover: manual + + iproto: + advertise: + peer: + login: replicator + sharding: + login: storage + + groups: + routers: + sharding: + roles: + - router + roles: + - roles.crud-router + replicasets: + router: + leader: router + instances: + router: + iproto: + listen: + - uri: localhost:3301 + storages: + sharding: + roles: + - storage + roles: + - roles.crud-storage + app: + module: mystorage + replicasets: + s-1: + leader: s1-master + instances: + s1-master: + iproto: + listen: + - uri: localhost:3302 + s1-replica: + iproto: + listen: + - uri: localhost:3303 + s-2: + leader: s2-master + instances: + s2-replica: + iproto: + listen: + - uri: localhost:3304 + s2-master: + iproto: + listen: + - uri: localhost:3305 + ``` +
+ +3. Set up your schema on storages (for example, through `app.module` section in Tarantool 3 configuration.) + + ```lua + -- mystorage.lua + + -- Schema setup is idempotent. + + + box.watch('box.status', function() + if box.info.ro then + return + end + + local customers_space = box.schema.space.create('customers', { + format = { + {name = 'id', type = 'unsigned'}, + {name = 'bucket_id', type = 'unsigned'}, + {name = 'name', type = 'string'}, + {name = 'age', type = 'number'}, + }, + if_not_exists = true, + }) + + customers_space:create_index('id', { + parts = { {field ='id', is_nullable = false} }, + if_not_exists = true, + }) + + customers_space:create_index('bucket_id', { + parts = { {field ='bucket_id', is_nullable = false} }, + if_not_exists = true, + }) + + customers_space:create_index('age', { + parts = { {field ='age'} }, + unique = false, + if_not_exists = true, + }) + end) + ``` + +4. Start the application cluster. You can check whether asynchronous bootstrap + had finished through `crud.storage_info()` calls on router. + +Now your cluster contains storages that are configured to be used for +CRUD-operations. +You can simply call CRUD functions on the router to insert, select, and update +data across the cluster. + ## Cartridge roles `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the diff --git a/crud/common/roles.lua b/crud/common/roles.lua new file mode 100644 index 00000000..66f8d89b --- /dev/null +++ b/crud/common/roles.lua @@ -0,0 +1,20 @@ +local config = require('config') + +local function is_sharding_role_enabled(expected_sharding_role) + -- Works only for versions newer than 3.0.1-10 (3.0.2 and following) + -- and newer than 3.1.0-entrypoint-77 (3.1.0 and following). + -- https://github.com/tarantool/tarantool/commit/ebb170cb8cf2b9c4634bcf0178665909f578c335 + local actual_sharding_roles = config:get('sharding.roles') + + for _, actual_sharding_role in ipairs(actual_sharding_roles or {}) do + if actual_sharding_role == expected_sharding_role then + return true + end + end + + return false +end + +return { + is_sharding_role_enabled = is_sharding_role_enabled, +} diff --git a/crud/common/utils.lua b/crud/common/utils.lua index 602cf4bd..0f0987a0 100644 --- a/crud/common/utils.lua +++ b/crud/common/utils.lua @@ -489,19 +489,6 @@ local function get_commits_since_from_version_part(commits_since_candidate) return 0 end - local ok, val = pcall(tonumber, commits_since_candidate) - if ok then - return val - else - return 0 - end -end - -local function get_commits_since_from_version_without_suffix(commits_since_candidate) - if commits_since_candidate == nil then - return 0 - end - local ok, val = pcall(tonumber, commits_since_candidate) if ok then return val @@ -739,20 +726,36 @@ local function determine_enabled_features() enabled_tarantool_features.tarantool_3 = is_version_ge(major, minor, patch, suffix, commits_since, 3, 0, 0, nil, nil) + + -- https://github.com/tarantool/tarantool/commit/ebb170cb8cf2b9c4634bcf0178665909f578c335 + -- https://github.com/tarantool/tarantool/commit/e0e1358cb60d6749c34daf508e05586e0959bf89 + enabled_tarantool_features.config_get_inside_roles = is_version_ge(major, minor, patch, suffix, commits_since, + 3, 1, 0, 'entrypoint', 77) + or is_version_in_range(major, minor, patch, suffix, + commits_since, + 3, 0, 1, nil, 10, + 3, 0, math.huge, nil, nil) + + -- https://github.com/tarantool/tarantool/commit/b982b46442e62e05ab6340343233aa766ad5e52c + -- https://github.com/tarantool/tarantool/commit/ee2faf7c328abc54631233342cb9b88e4ce8cae4 + enabled_tarantool_features.role_privileges_not_revoked = is_version_ge(major, minor, patch, suffix, commits_since, + 3, 1, 0, 'entrypoint', 179) + or is_version_in_range(major, minor, patch, suffix, + commits_since, + 3, 0, 1, nil, 57, + 3, 0, math.huge, nil, nil) end determine_enabled_features() -local function feature_in_list(feature_to_check, list_of_features) - -end - for feature_name, feature_enabled in pairs(enabled_tarantool_features) do local util_name if feature_name == 'tarantool_3' then util_name = ('is_%s'):format(feature_name) elseif feature_name == 'builtin_merger' then util_name = ('tarantool_has_%s'):format(feature_name) + elseif feature_name == 'role_privileges_not_revoked' then + util_name = ('tarantool_%s'):format(feature_name) else util_name = ('tarantool_supports_%s'):format(feature_name) end diff --git a/roles/crud-router.lua b/roles/crud-router.lua new file mode 100644 index 00000000..b891e9d5 --- /dev/null +++ b/roles/crud-router.lua @@ -0,0 +1,33 @@ +local errors = require('errors') + +local crud = require('crud') +local common_role_utils = require('crud.common.roles') +local common_utils = require('crud.common.utils') + +local TarantoolRoleConfigurationError = errors.new_class('TarantoolRoleConfigurationError', {capture_stack = false}) + +TarantoolRoleConfigurationError:assert( + common_utils.tarantool_supports_config_get_inside_roles(), + ('Tarantool 3 role is not supported for Tarantool %s, use 3.0.2 or newer'):format(box.info.version) +) + +local function validate() + TarantoolRoleConfigurationError:assert( + common_role_utils.is_sharding_role_enabled('router'), + 'instance must be a sharding router to enable roles.crud-router' + ) +end + +local function apply() + crud.init_router() +end + +local function stop() + crud.stop_router() +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/roles/crud-storage.lua b/roles/crud-storage.lua new file mode 100644 index 00000000..1b255312 --- /dev/null +++ b/roles/crud-storage.lua @@ -0,0 +1,34 @@ +local errors = require('errors') + +local crud = require('crud') +local common_role_utils = require('crud.common.roles') +local common_utils = require('crud.common.utils') + +local TarantoolRoleConfigurationError = errors.new_class('TarantoolRoleConfigurationError', {capture_stack = false}) + +TarantoolRoleConfigurationError:assert( + common_utils.tarantool_supports_config_get_inside_roles() + and common_utils.tarantool_role_privileges_not_revoked(), + ('Tarantool 3 role is not supported for Tarantool %s, use 3.0.2 or newer'):format(box.info.version) +) + +local function validate() + TarantoolRoleConfigurationError:assert( + common_role_utils.is_sharding_role_enabled('storage'), + 'instance must be a sharding storage to enable roles.crud-storage' + ) +end + +local function apply() + crud.init_storage{async = true} +end + +local function stop() + crud.stop_storage() +end + +return { + validate = validate, + apply = apply, + stop = stop, +} diff --git a/test/helper.lua b/test/helper.lua index 9a3c7409..5540feae 100644 --- a/test/helper.lua +++ b/test/helper.lua @@ -285,6 +285,7 @@ function helpers.get_test_config_groups() sharding = { roles = {'router'}, }, + roles = {'roles.crud-router'}, replicasets = { ['router'] = { leader = 'router', @@ -298,6 +299,7 @@ function helpers.get_test_config_groups() sharding = { roles = {'storage'}, }, + roles = {'roles.crud-storage'}, replicasets = { ['s-1'] = { leader = 's1-master', @@ -915,7 +917,7 @@ function helpers.start_cluster(g, cartridge_cfg, vshard_cfg, tarantool3_cluster_ elseif g.params.backend == helpers.backend.VSHARD then helpers.start_vshard_cluster(g, vshard_cfg) elseif g.params.backend == helpers.backend.CONFIG then - helpers.skip_if_tarantool_config_unsupported() + helpers.skip_if_tarantool3_crud_roles_unsupported() helpers.start_tarantool3_cluster(g, tarantool3_cluster_cfg) end @@ -1159,7 +1161,7 @@ function helpers.backend_matrix(base_matrix) ) end - if helpers.is_tarantool_config_supported() then + if helpers.is_tarantool3_crud_roles_supported() then table.insert(backend_params, { backend = helpers.backend.CONFIG, @@ -1453,4 +1455,14 @@ function helpers.skip_if_tarantool_config_unsupported() ("Tarantool %s does not support starting from config"):format(box.info.version)) end +function helpers.is_tarantool3_crud_roles_supported() + return crud_utils.tarantool_supports_config_get_inside_roles() + and crud_utils.tarantool_role_privileges_not_revoked() +end + +function helpers.skip_if_tarantool3_crud_roles_unsupported() + t.skip_if(not helpers.is_tarantool3_crud_roles_supported(), + ("Tarantool %s does not support crud roles"):format(box.info.version)) +end + return helpers diff --git a/test/performance/perf_test.lua b/test/performance/perf_test.lua index 63e41bc6..f17c882c 100644 --- a/test/performance/perf_test.lua +++ b/test/performance/perf_test.lua @@ -114,6 +114,7 @@ local tarantool3_cluster_cfg_template = { sharding = { roles = {'router'}, }, + roles = {'roles.crud-router'}, replicasets = { ['router'] = { leader = 'router', @@ -127,6 +128,7 @@ local tarantool3_cluster_cfg_template = { sharding = { roles = {'storage'}, }, + roles = {'roles.crud-storage'}, replicasets = { ['s-1'] = { leader = 's1-master', diff --git a/test/tarantool3_helpers/cluster.lua b/test/tarantool3_helpers/cluster.lua index c4b3c723..3b56ce83 100644 --- a/test/tarantool3_helpers/cluster.lua +++ b/test/tarantool3_helpers/cluster.lua @@ -51,8 +51,8 @@ function Cluster:initialize() for _, group in pairs(self.config.groups) do for _, replicaset in pairs(group.replicasets) do - local is_router = utils.is_replicaset_a_sharding_router(group, replicaset) - local is_storage = utils.is_replicaset_a_sharding_storage(group, replicaset) + local is_router = utils.is_replicaset_a_crud_router(group, replicaset) + local is_storage = utils.is_replicaset_a_crud_storage(group, replicaset) for alias, _ in pairs(replicaset.instances) do local dir = treegen.prepare_directory(self.treegen, {}, {}) @@ -154,30 +154,6 @@ function Cluster:bootstrap_vshard_routers() end end -local function bootstrap_crud_router() - local crud = require('crud') - crud.init_router() -end - -local function bootstrap_crud_storage() - local crud = require('crud') - crud.init_storage{async = true} -end - -function Cluster:bootstrap_crud() - for _, group in pairs(self.config.groups) do - for _, rs in pairs(group.replicasets) do - if utils.is_replicaset_a_sharding_router(group, rs) then - self:exec_on_replicaset(rs, bootstrap_crud_router) - end - - if utils.is_replicaset_a_sharding_storage(group, rs) then - self:exec_on_replicaset(rs, bootstrap_crud_storage) - end - end - end -end - function Cluster:wait_for_leaders_rw() for _, group in pairs(self.config.groups) do for _, rs in pairs(group.replicasets) do @@ -218,10 +194,6 @@ function Cluster:bootstrap() self:set_etalon_bucket_balance() self:bootstrap_vshard_routers() - if self.crud_init then - self:bootstrap_crud() - end - return self end @@ -260,6 +232,11 @@ end function Cluster:wait_crud_is_ready_on_cluster() local router = self:get_router() + if router == nil then + -- Cluster is not a crud cluster. + return self + end + local storages_in_topology = self:count_storages() local WAIT_TIMEOUT = 60 @@ -298,12 +275,12 @@ function Cluster:wait_modules_are_ready_on_cluster() for _, group in pairs(self.config.groups) do for _, rs in pairs(group.replicasets) do if self.router_wait_until_ready ~= nil - and utils.is_replicaset_a_sharding_router(group, rs) then + and utils.is_replicaset_a_crud_router(group, rs) then self:eval_on_replicaset(rs, self.router_wait_until_ready) end if self.storage_wait_until_ready ~= nil - and utils.is_replicaset_a_sharding_storage(group, rs) then + and utils.is_replicaset_a_crud_storage(group, rs) then self:eval_on_replicaset(rs, self.storage_wait_until_ready) end end diff --git a/test/tarantool3_helpers/cluster_test.lua b/test/tarantool3_helpers/cluster_test.lua index 095fabc7..9cb92d3b 100644 --- a/test/tarantool3_helpers/cluster_test.lua +++ b/test/tarantool3_helpers/cluster_test.lua @@ -6,7 +6,7 @@ local cluster_helpers = require('test.tarantool3_helpers.cluster') local g = t.group() g.before_all(function(cg) - helpers.skip_if_tarantool_config_unsupported() + helpers.skip_if_tarantool3_crud_roles_unsupported() local config = { credentials = { @@ -42,6 +42,7 @@ g.before_all(function(cg) sharding = { roles = {'router'}, }, + roles = {'roles.crud-router'}, replicasets = { ['router'] = { leader = 'router', @@ -62,6 +63,7 @@ g.before_all(function(cg) sharding = { roles = {'storage'}, }, + roles = {'roles.crud-storage'}, replicasets = { ['s-1'] = { leader = 's1-master', @@ -158,7 +160,6 @@ g.before_all(function(cg) error('timeout while waiting for storage bootstrap') ]], - crud_init = true, }) cg.cluster:start() diff --git a/test/tarantool3_helpers/utils.lua b/test/tarantool3_helpers/utils.lua index 1bce0644..7d0ad815 100644 --- a/test/tarantool3_helpers/utils.lua +++ b/test/tarantool3_helpers/utils.lua @@ -40,9 +40,34 @@ local function is_group_a_sharding_storage(group) return is_group_has_sharding_role(group, 'storage') end +local function is_group_a_crud_router(group) + return has_role(group, 'roles.crud-router') +end + +local function is_group_a_crud_storage(group) + return has_role(group, 'roles.crud-storage') +end + +local function is_replicaset_has_role(group, replicaset, role) + return has_role(group, role) or has_role(replicaset, role) +end + +local function is_replicaset_a_crud_router(group, replicaset) + return is_replicaset_has_role(group, replicaset, 'roles.crud-router') +end + +local function is_replicaset_a_crud_storage(group, replicaset) + return is_replicaset_has_role(group, replicaset, 'roles.crud-storage') +end + return { is_group_a_sharding_router = is_group_a_sharding_router, is_group_a_sharding_storage = is_group_a_sharding_storage, is_replicaset_a_sharding_router = is_replicaset_a_sharding_router, is_replicaset_a_sharding_storage = is_replicaset_a_sharding_storage, + + is_group_a_crud_router = is_group_a_crud_router, + is_group_a_crud_storage = is_group_a_crud_storage, + is_replicaset_a_crud_router = is_replicaset_a_crud_router, + is_replicaset_a_crud_storage = is_replicaset_a_crud_storage, } \ No newline at end of file diff --git a/test/unit/not_initialized_test.lua b/test/unit/not_initialized_test.lua index f246ae33..ae761252 100644 --- a/test/unit/not_initialized_test.lua +++ b/test/unit/not_initialized_test.lua @@ -76,7 +76,6 @@ local tarantool3_cluster_cfg_template = { }, bucket_count = 20, storage_entrypoint = helpers.entrypoint_vshard_storage('srv_not_initialized'), - crud_init = false, } pgroup.before_all(function(g)