scan: extention types filters

.github/workflows/test_on_push.yaml

@@ -17,7 +17,7 @@ jobs:
         # it uses its own metrics package.
         # We test old metrics with Cartridge 2.7.9 because since 2.8.0 it
         # requires metrics 1.0.0.
-        tarantool-version: ["1.10.6", "1.10", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.10", "2.11"]
+        tarantool-version: ["1.10.6", "1.10", "2.8", "2.10", "2.11"]
         metrics-version: [""]
         cartridge-version: ["2.8.0"]
         external-tuple-merger-version: [""]

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Changed
+* Explicitly forbid datetime interval conditions (#373).
+
+### Fixed
+* Working with datetime conditions in case of non-indexed fields or
+  non-iterating indexes (#373).
+* Precision loss for decimal conditions in case of non-indexed fields or
+  non-iterating indexes (#373).
+* Passing errors from storages for merger operations (`crud.select`,
+  `crud.pairs`, `readview:select`, `readview:pairs`) (#423).
+
 ## [1.4.3] - 05-02-24
 
 ### Fixed

crud/common/utils.lua

@@ -1,10 +1,13 @@
+local bit = require('bit')
 local errors = require('errors')
+local fiber = require('fiber')
 local ffi = require('ffi')
-local vshard = require('vshard')
 local fun = require('fun')
-local bit = require('bit')
+local vshard = require('vshard')
 local log = require('log')
 
+local datetime_supported, datetime = pcall(require, 'datetime')
+
 local is_cartridge, cartridge = pcall(require, 'cartridge')
 local is_cartridge_hotreload, cartridge_hotreload = pcall(require, 'cartridge.hotreload')
 
@@ -23,7 +26,6 @@ local NotInitializedError = errors.new_class('NotInitialized')
 local StorageInfoError = errors.new_class('StorageInfoError')
 local VshardRouterError = errors.new_class('VshardRouterError', {capture_stack = false})
 local UtilsInternalError = errors.new_class('UtilsInternalError', {capture_stack = false})
-local fiber = require('fiber')
 
 local utils = {}
 
@@ -1335,6 +1337,21 @@ function utils.is_cartridge_hotreload_supported()
     return true, cartridge_hotreload
 end
 
+local interval_supported = datetime_supported and (datetime.interval ~= nil)
+
+if interval_supported then
+    -- https://github.com/tarantool/tarantool/blob/0510ffa07afd84a70c9c6f1a4c28aacd73a393d6/src/lua/datetime.lua#L175-179
+    local interval_t = ffi.typeof('struct interval')
+
+    utils.is_interval = function(o)
+        return ffi.istype(interval_t, o)
+    end
+else
+    utils.is_interval = function()
+        return false
+    end
+end
+
 for k, v in pairs(require('crud.common.vshard_utils')) do
     utils[k] = v
 end

crud/compare/filters.lua

@@ -1,3 +1,6 @@
+local datetime_supported, datetime = pcall(require, 'datetime')
+local decimal_supported, decimal = pcall(require, 'decimal')
+
 local errors = require('errors')
 
 local utils = require('crud.common.utils')
@@ -157,12 +160,23 @@ local function format_value(value)
         return tostring(value)
     elseif type(value) == 'boolean' then
         return tostring(value)
+    elseif decimal_supported and decimal.is_decimal(value) then
+        -- decimal supports comparison with string.
+        return ("%q"):format(tostring(value))
     elseif utils.is_uuid(value) then
         return ("%q"):format(value)
+    elseif datetime_supported and datetime.is_datetime(value) then
+        return ("%q"):format(value:format())
+    elseif utils.is_interval(value) then
+        -- As for Tarantool 3.0 and older, datetime intervals
+        -- are not comparable. It's better to explicitly forbid them
+        -- for now.
+        -- https://github.com/tarantool/tarantool/issues/7659
+        GenFiltersError:assert(false, ('datetime interval conditions are not supported'))
     elseif type(value) == 'cdata' then
         return tostring(value)
     end
-    assert(false, ('Unexpected value %s (type %s)'):format(value, type(value)))
+    GenFiltersError:assert(false, ('Unexpected value %s (type %s)'):format(value, type(value)))
 end
 
 local PARSE_ARGS_TEMPLATE = 'local tuple = ...'
@@ -283,6 +297,8 @@ local function format_eq(cond)
             end
         elseif value_type == 'uuid' then
             func_name = 'eq_uuid'
+        elseif value_type == 'datetime' then
+            func_name = 'eq_datetime'
         end
 
         table.insert(cond_strings, format_comp_with_value(field, func_name, value))
@@ -309,6 +325,8 @@ local function format_lt(cond)
             func_name = add_collation_postfix('lt', value_opts)
         elseif value_type == 'uuid' then
             func_name = 'lt_uuid'
+        elseif value_type == 'datetime' then
+            func_name = 'lt_datetime'
         end
 
         func_name = add_strict_postfix(func_name, value_opts)
@@ -533,6 +551,30 @@ local function lt_uuid_strict(lhs, rhs)
     return tostring(lhs) < tostring(rhs)
 end
 
+local function opt_datetime_parse(v)
+    if type(v) == 'string' then
+        return datetime.parse(v)
+    end
+
+    return v
+end
+
+local function lt_datetime_nullable(lhs, rhs)
+    if lhs == nil and rhs ~= nil then
+        return true
+    elseif rhs == nil then
+        return false
+    end
+    return opt_datetime_parse(lhs) < opt_datetime_parse(rhs)
+end
+
+local function lt_datetime_strict(lhs, rhs)
+    if rhs == nil then
+        return false
+    end
+    return opt_datetime_parse(lhs) < opt_datetime_parse(rhs)
+end
+
 local function lt_unicode_ci_nullable(lhs, rhs)
     if lhs == nil and rhs ~= nil then
         return true
@@ -567,6 +609,20 @@ local function eq_uuid_strict(lhs, rhs)
     return tostring(lhs) == tostring(rhs)
 end
 
+local function eq_datetime(lhs, rhs)
+    if lhs == nil then
+        return rhs == nil
+    end
+    return opt_datetime_parse(lhs) == opt_datetime_parse(rhs)
+end
+
+local function eq_datetime_strict(lhs, rhs)
+    if rhs == nil then
+        return false
+    end
+    return opt_datetime_parse(lhs) == opt_datetime_parse(rhs)
+end
+
 local function eq_unicode_nullable(lhs, rhs)
     if lhs == nil and rhs == nil then
         return true
@@ -604,6 +660,8 @@ local library = {
     eq = eq,
     eq_uuid = eq_uuid,
     eq_uuid_strict = eq_uuid_strict,
+    eq_datetime = eq_datetime,
+    eq_datetime_strict = eq_datetime_strict,
     -- nullable
     eq_unicode = eq_unicode_nullable,
     eq_unicode_ci = eq_unicode_ci_nullable,
@@ -618,12 +676,14 @@ local library = {
     lt_unicode_ci = lt_unicode_ci_nullable,
     lt_boolean = lt_boolean_nullable,
     lt_uuid = lt_uuid_nullable,
+    lt_datetime = lt_datetime_nullable,
     -- strict
     lt_strict = lt_strict,
     lt_unicode_strict = lt_unicode_strict,
     lt_unicode_ci_strict = lt_unicode_ci_strict,
     lt_boolean_strict = lt_boolean_strict,
     lt_uuid_strict = lt_uuid_strict,
+    lt_datetime_strict = lt_datetime_strict,
 
     utf8 = utf8,
 

crud/readview.lua

@@ -133,22 +133,22 @@ local function select_readview_on_storage(space_name, index_id, conditions, opts
     end
 
     if space_readview == nil then
-        return cursor, ReadviewError:new("Space %q doesn't exist", space_name)
+        return ReadviewError:assert(false, "Space %q doesn't exist", space_name)
     end
 
     local space = box.space[space_name]
     if space == nil then
-        return cursor, ReadviewError:new("Space %q doesn't exist", space_name)
+        return ReadviewError:assert(false, "Space %q doesn't exist", space_name)
     end
     space_readview.format = space:format()
 
     local index_readview = space_readview.index[index_id]
     if index_readview == nil then
-        return cursor, ReadviewError:new("Index with ID %s doesn't exist", index_id)
+        return ReadviewError:assert(false, "Index with ID %s doesn't exist", index_id)
     end
     local index = space.index[index_id]
     if index == nil then
-        return cursor, ReadviewError:new("Index with ID %s doesn't exist", index_id)
+        return ReadviewError:assert(false, "Index with ID %s doesn't exist", index_id)
     end
 
     local _, err = sharding.check_sharding_hash(space_name,
@@ -179,7 +179,7 @@ local function select_readview_on_storage(space_name, index_id, conditions, opts
         readview_index = index_readview,
     })
     if err ~= nil then
-        return cursor, ReadviewError:new("Failed to execute select: %s", err)
+        return ReadviewError:assert(false, "Failed to execute select: %s", err)
     end
 
     if resp.tuples_fetched < opts.limit or opts.limit == 0 then

crud/select.lua

@@ -61,12 +61,12 @@ local function select_on_storage(space_name, index_id, conditions, opts)
 
     local space = box.space[space_name]
     if space == nil then
-        return cursor, SelectError:new("Space %q doesn't exist", space_name)
+        SelectError:assert(false, "Space %q doesn't exist", space_name)
     end
 
     local index = space.index[index_id]
     if index == nil then
-        return cursor, SelectError:new("Index with ID %s doesn't exist", index_id)
+        SelectError:assert(false, "Index with ID %s doesn't exist", index_id)
     end
 
     local _, err = sharding.check_sharding_hash(space_name,
@@ -83,7 +83,7 @@ local function select_on_storage(space_name, index_id, conditions, opts)
         scan_condition_num = opts.scan_condition_num,
     })
     if err ~= nil then
-        return cursor, SelectError:new("Failed to generate tuples filter: %s", err)
+        return SelectError:assert(false, "Failed to generate tuples filter: %s", err)
     end
 
     -- execute select
@@ -95,7 +95,7 @@ local function select_on_storage(space_name, index_id, conditions, opts)
         yield_every = opts.yield_every,
     })
     if err ~= nil then
-        return cursor, SelectError:new("Failed to execute select: %s", err)
+        return SelectError:assert(false, "Failed to execute select: %s", err)
     end
 
     if resp.tuples_fetched < opts.limit or opts.limit == 0 then