Skip to content

Commit

Permalink
[Fix](segment iterator) fix shrink non-char column coredump (apache#3…
Browse files Browse the repository at this point in the history 
…6275)

If we execute a delete predicate in an inverted index table and then
query something from it, it will cause a core dump in
shrink_char_type_column_suffix_zero. This occurs because the delete
predicate column ID in _char_type_idx is incorrectly shifted to the
result column inserted by the inverted index result.

coredump stack like:
```
COULD NOT CREATE A LOGGINGFILE 20240604-145331!F20240604 14:53:31.991016 24178 column.h:134] Cannot get_shrinked_column() column Const(UInt8)
*** Check failure stack trace: ***
F20240604 14:53:31.991436 24158 column.h:134] Cannot get_shrinked_column() column Const(UInt8)
*** Check failure stack trace: ***
F20240604 14:53:31.991436 24158 column.h:134] Cannot get_shrinked_column() column Const(UInt8)F20240604 14:53:31.991645 24160 column.h:134] Cannot get_shrinked_column() column Const(UInt8)

#0  0x00007fa0bf784387 in ?? () from /mnt/disk2/tengjianping/local/ldb_toolchain/lib/libc.so.6
#1  0x0000557dc8e617a0 in google::LogMessage::Flush() ()
#2  0x0000557dc8e65599 in google::LogMessageFatal::~LogMessageFatal() ()
apache#3  0x0000557dbe4ec939 in doris::vectorized::IColumn::get_shrinked_column (this=0x7fa094ac16a0) at /home/zcp/repo_center/doris_release/doris/be/src/vec/columns/column.h:134
apache#4  0x0000557dc236d8a2 in doris::vectorized::Block::shrink_char_type_column_suffix_zero (this=0x7f9a372a4b70, char_type_idx=...)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/core/block.cpp:1126
apache#5  0x0000557dbee9ea95 in doris::segment_v2::SegmentIterator::_next_batch_internal (this=<optimized out>, block=0x7f9a372a4b70)
    at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:2408
apache#6  0x0000557dbee9cc5a in doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*)::$_0::operator()() const (this=<optimized out>)
    at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:2090
apache#7  doris::segment_v2::SegmentIterator::next_batch (this=0x1c5d, block=0x6) at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/segment_v2/segment_iterator.cpp:2089
apache#8  0x0000557dbeebbb44 in doris::BetaRowsetReader::next_block (this=0x7f9a3a041600, block=0x7f9a372a4b70)
    at /home/zcp/repo_center/doris_release/doris/be/src/olap/rowset/beta_rowset_reader.cpp:342
apache#9  0x0000557dc8590ddd in doris::vectorized::VCollectIterator::Level0Iterator::_refresh (this=0x7f9a372a1be0)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vcollect_iterator.h:256
apache#10 doris::vectorized::VCollectIterator::Level0Iterator::refresh_current_row (this=0x7f9a372a1be0)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vcollect_iterator.cpp:514
apache#11 0x0000557dc8591055 in doris::vectorized::VCollectIterator::Level0Iterator::ensure_first_row_ref (this=0x1c5d)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vcollect_iterator.cpp:493
apache#12 0x0000557dc85934f2 in doris::vectorized::VCollectIterator::Level1Iterator::ensure_first_row_ref (this=0x7fa090e36c00)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vcollect_iterator.cpp:692
apache#13 0x0000557dc858ddc9 in doris::vectorized::VCollectIterator::build_heap (this=0x7f9d80976dc0, rs_readers=...)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/vcollect_iterator.cpp:186
apache#14 0x0000557dc857e808 in doris::vectorized::BlockReader::_init_collect_iter (this=this@entry=0x7f9d80976800, read_params=...)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/block_reader.cpp:156
apache#15 0x0000557dc857f64d in doris::vectorized::BlockReader::init (this=<optimized out>, read_params=...)
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/olap/block_reader.cpp:228
apache#16 0x0000557dc370dc2c in doris::vectorized::NewOlapScanner::open (this=0x7f9a2aa68210, state=<optimized out>)
--Type <RET> for more, q to quit, c to continue without paging--c
    at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:227
apache#17 0x0000557dc3718d89 in doris::vectorized::ScannerScheduler::_scanner_scan (ctx=std::shared_ptr<doris::vectorized::ScannerContext> (use count 4, weak count 1) = {...}, scan_task=std::shared_ptr<doris::vectorized::ScanTask> (use count 2, weak count 0) = {...}) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:242
apache#18 0x0000557dc3719b77 in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const (this=0x0) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:158
apache#19 std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1&) (__f=...) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
apache#20 std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1&) (__fn=...) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111
apache#21 std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1>::_M_invoke(std::_Any_data const&) (__functor=...) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291
apache#22 0x0000557dbf3c7788 in doris::ThreadPool::dispatch_thread (this=0x7f9d0bbdc400) at /home/zcp/repo_center/doris_release/doris/be/src/util/threadpool.cpp:543
apache#23 0x0000557dbf3bcb41 in std::function<void ()>::operator()() const (this=0x1471) at /var/local/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
apache#24 doris::Thread::supervise_thread (arg=0x7f9d0bbdf4c0) at /home/zcp/repo_center/doris_release/doris/be/src/util/thread.cpp:498
apache#25 0x00007fa0bee1dea5 in sem_open () from /mnt/disk2/tengjianping/local/ldb_toolchain/lib/libpthread.so.0
```
  • Loading branch information
@airborne12 @dataroaring
airborne12 authored and dataroaring committed Jun 21, 2024
1 parent 0d6b80b commit 98fea6a
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 13 deletions.
28 changes: 16 additions & 12 deletions be/src/olap/rowset/segment_v2/segment_iterator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,10 +321,6 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
}

RETURN_IF_ERROR(init_iterators());
if (_char_type_idx.empty() && _char_type_idx_no_0.empty()) {
_is_char_type.resize(_schema->columns().size(), false);
_vec_init_char_column_id();
}

if (opts.output_columns != nullptr) {
_output_columns = *(opts.output_columns);
Expand Down Expand Up @@ -1843,19 +1839,23 @@ bool SegmentIterator::_has_char_type(const Field& column_desc) {
}
};

void SegmentIterator::_vec_init_char_column_id() {
void SegmentIterator::_vec_init_char_column_id(vectorized::Block* block) {
for (size_t i = 0; i < _schema->num_column_ids(); i++) {
auto cid = _schema->column_id(i);
const Field* column_desc = _schema->column(cid);

if (_has_char_type(*column_desc)) {
_char_type_idx.emplace_back(i);
if (i != 0) {
_char_type_idx_no_0.emplace_back(i);
// The additional deleted filter condition will be in the materialized column at the end of the block.
// After _output_column_by_sel_idx, it will be erased, so we do not need to shrink it.
if (i < block->columns()) {
if (_has_char_type(*column_desc)) {
_char_type_idx.emplace_back(i);
if (i != 0) {
_char_type_idx_no_0.emplace_back(i);
}
}
if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) {
_is_char_type[cid] = true;
}
}
if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_CHAR) {
_is_char_type[cid] = true;
}
}
}
Expand Down Expand Up @@ -2292,6 +2292,10 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
}
_current_return_columns.resize(_schema->columns().size());
_converted_column_ids.resize(_schema->columns().size(), 0);
if (_char_type_idx.empty() && _char_type_idx_no_0.empty()) {
_is_char_type.resize(_schema->columns().size(), false);
_vec_init_char_column_id(block);
}
for (size_t i = 0; i < _schema->num_column_ids(); i++) {
auto cid = _schema->column_id(i);
auto column_desc = _schema->column(cid);
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/rowset/segment_v2/segment_iterator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ class SegmentIterator : public RowwiseIterator {
// TODO: Fix Me
// CHAR type in storage layer padding the 0 in length. But query engine need ignore the padding 0.
// so segment iterator need to shrink char column before output it. only use in vec query engine.
void _vec_init_char_column_id();
void _vec_init_char_column_id(vectorized::Block* block);
bool _has_char_type(const Field& column_desc);

uint32_t segment_id() const { return _segment->id(); }
Expand Down
4 changes: 4 additions & 0 deletions regression-test/data/inverted_index_p0/test_delete.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
2

60 changes: 60 additions & 0 deletions regression-test/suites/inverted_index_p0/test_delete.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_delete"){
// prepare test table

def timeout = 60000
def delta_time = 1000
def alter_res = "null"
def useTime = 0

def indexTblName = "test_delete"

sql "DROP TABLE IF EXISTS ${indexTblName}"
// create 1 replica table
sql """
CREATE TABLE `${indexTblName}` (
`a` int NULL COMMENT '',
`b` varchar(60) NOT NULL COMMENT '',
`c` char(10) NULL COMMENT '',
INDEX index_b(b) USING INVERTED COMMENT '',
INDEX index_c(c) USING INVERTED COMMENT ''
) ENGINE=OLAP
DUPLICATE KEY(`a`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`a`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"is_being_synced" = "false",
"storage_format" = "V2",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false"
);
"""

sql """ INSERT INTO `${indexTblName}`(`a`, `b`, `c`) VALUES ('1', '6afef581285b6608bf80d5a4e46cf839', 'aaa'), ('2', '48a33ec3453a28bce84b8f96fe161956', 'bbb'),
('3', '021603e7dcfe65d44af0efd0e5aee154', 'ccc'), ('4', 'ee27ee1da291e46403c408e220bed6e1', 'ddd'),
('5', 'a648a447b8f71522f11632eba4b4adde', 'eee'), ('6', 'a9fb5c985c90bf05f3bee5ca3ae95260', 'fff'),
('7', '0974e7a82e30d1af83205e474fadd0a2', 'ggg'); """


sql """ DELETE FROM ${indexTblName} WHERE c IN ('aaa','ccc'); """

qt_sql """ SELECT count(1) as cnt FROM ${indexTblName} WHERE a BETWEEN 1 AND 6 AND b IN ('48a33ec3453a28bce84b8f96fe161956', 'a9fb5c985c90bf05f3bee5ca3ae95260'); """
}

0 comments on commit 98fea6a

Please sign in to comment.