[fix](inverted index) disable range query in StringTypeInvertedIndexR…

…eader (#38218) ## Proposed changes disable range query in StringTypeInvertedIndexReader
apache · Jul 25, 2024 · 00fca3e · 00fca3e
1 parent 1293912
commit 00fca3e
Show file tree

Hide file tree

Showing 3 changed files with 61 additions and 2 deletions.
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -920,7 +920,8 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool
     if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) {
         return false;
     }
-    if (_inverted_index_iterators[pred->column_id()] == nullptr) {
+    auto pred_column_id = pred->column_id();
+    if (_inverted_index_iterators[pred_column_id] == nullptr) {
         //this column without inverted index
         return false;
     }
@@ -935,13 +936,21 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool
         return false;
     }
 
+    // UNTOKENIZED strings exceed ignore_above, they are written as null, causing range query errors
+    if (PredicateTypeTraits::is_range(pred->type()) &&
+        _inverted_index_iterators[pred_column_id] != nullptr &&
+        _inverted_index_iterators[pred_column_id]->get_inverted_index_reader_type() ==
+                InvertedIndexReaderType::STRING_TYPE) {
+        return false;
+    }
+
     // Function filter no apply inverted index
     if (dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(pred) != nullptr ||
         dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(pred) != nullptr) {
         return false;
     }
 
-    bool handle_by_fulltext = _column_has_fulltext_index(pred->column_id());
+    bool handle_by_fulltext = _column_has_fulltext_index(pred_column_id);
     if (handle_by_fulltext) {
         // when predicate in compound condition which except leafNode of andNode,
         // only can apply match query for fulltext index,

diff --git a/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out b/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out
@@ -2,3 +2,6 @@
 -- !sql --
 3
 
+-- !sql --
+772
+
diff --git a/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy b/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy
@@ -39,4 +39,51 @@ suite("test_ignore_above_in_index", "p0") {
     sql "insert into ${tableName} values (20, '1234567890');"
     sql "insert into ${tableName} values (20, '1234567890');"
     qt_sql "select count() from ${tableName} where c = '1234567890';"
+
+    def tableName2 = "test_ignore_above_in_index2"
+    sql "DROP TABLE IF EXISTS ${tableName2}"
+    sql """
+        CREATE TABLE ${tableName2} (
+          `@timestamp` int(11) NULL COMMENT "",
+          `clientip` string NULL COMMENT "",
+          `request` string NULL COMMENT "",
+          `status` int NULL COMMENT "",
+          `size` int NULL COMMENT "",
+          INDEX clientip_idx (`clientip`) USING INVERTED PROPERTIES("ignore_above"="5") COMMENT '',
+          INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '',
+          INDEX status_idx (`status`) USING INVERTED COMMENT '',
+          INDEX size_idx (`size`) USING INVERTED COMMENT ''
+          ) ENGINE=OLAP
+          DUPLICATE KEY(`@timestamp`)
+          COMMENT "OLAP"
+          DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
+          PROPERTIES (
+          "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+
+    // load the json data
+    streamLoad {
+        table "${tableName2}"
+
+        set 'read_json_by_line', 'true'
+        set 'format', 'json'
+        file 'documents-1000.json' // import json file
+        time 10000 // limit inflight 10s
+
+        // if declared a check callback, the default check condition will ignore.
+        // So you must check all condition
+        check { result, exception, startTime, endTime ->
+            if (exception != null) {
+                throw exception
+            }
+            log.info("Stream load result: ${result}".toString())
+            def json = parseJson(result)
+            assertEquals("success", json.Status.toLowerCase())
+            assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows)
+            assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+        }
+    }
+
+    qt_sql "select count() from ${tableName2} where clientip > '17.0';"
 }