Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Storage: row ID generation for MVCC bitmap filter #6458

Merged
merged 66 commits into from
Jan 31, 2023
Merged
Changes from 1 commit
Commits
Show all changes
66 commits
Select commit Hold shift + click to select a range
0ff2ac8
bitmap filter
JinheLin Jul 29, 2022
0e9d3cf
test
JinheLin Nov 21, 2022
aab511e
Fix delta row id.
JinheLin Nov 21, 2022
032f18c
test
JinheLin Nov 22, 2022
3d64091
for test
JinheLin Nov 23, 2022
cac5cd8
format
JinheLin Nov 25, 2022
6115761
format
JinheLin Nov 29, 2022
69e90bc
ci
JinheLin Nov 29, 2022
3496ef4
snapshot
JinheLin Nov 30, 2022
d59c7a8
Clean read.
JinheLin Nov 30, 2022
7e39641
fix
JinheLin Dec 2, 2022
bda0352
fix test
JinheLin Dec 2, 2022
ad315e4
ci
JinheLin Dec 5, 2022
aeff8ae
ci
JinheLin Dec 5, 2022
241c467
ci
JinheLin Dec 5, 2022
47bf40f
test
JinheLin Dec 6, 2022
f323376
test
JinheLin Dec 6, 2022
4c52024
test
JinheLin Dec 7, 2022
517b941
test
JinheLin Dec 7, 2022
d6d5613
ci
JinheLin Dec 7, 2022
9dfc9c1
test
JinheLin Dec 7, 2022
41ac4ca
test
JinheLin Dec 7, 2022
6131946
test
JinheLin Dec 7, 2022
0a47978
ci
JinheLin Dec 14, 2022
e1e4ab6
fix
JinheLin Dec 15, 2022
b911d05
ci
JinheLin Dec 15, 2022
605ff44
optimization
JinheLin Dec 15, 2022
687077b
remove useless code
JinheLin Dec 16, 2022
6184f2d
ci
JinheLin Dec 16, 2022
079ac77
ci
JinheLin Dec 16, 2022
29dd419
ci
JinheLin Dec 16, 2022
920b404
fix
JinheLin Dec 16, 2022
0acfa16
format
JinheLin Dec 16, 2022
97f9e26
filter
JinheLin Dec 19, 2022
4ce09e7
Fix scheduling.
JinheLin Dec 16, 2022
71623ea
ci
JinheLin Dec 20, 2022
e087c64
stable only
JinheLin Dec 21, 2022
9d5d698
ci
JinheLin Dec 30, 2022
7f5e415
ci
JinheLin Dec 30, 2022
4859f34
format
JinheLin Dec 30, 2022
84f44bc
fix
JinheLin Jan 4, 2023
dbf95ba
fix
JinheLin Jan 4, 2023
7d4be45
fix
JinheLin Jan 4, 2023
7c69fd9
test
Jan 9, 2023
a606558
test
JinheLin Jan 11, 2023
5d894bc
ci
JinheLin Jan 11, 2023
cc7a730
Fix test.
JinheLin Jan 12, 2023
91279c6
test
JinheLin Jan 12, 2023
7f7fbf1
ci
JinheLin Jan 12, 2023
090d8a1
fix lint.
JinheLin Jan 12, 2023
dbc1c1a
Resolve conflict.
JinheLin Jan 12, 2023
3b59088
remove useless file.
JinheLin Jan 13, 2023
71375b6
test
JinheLin Jan 13, 2023
322d83e
Remove useless.
JinheLin Jan 17, 2023
d60293c
set res_filter to null when all rows match.
JinheLin Jan 17, 2023
c0124bc
fix
JinheLin Jan 17, 2023
c04fb30
Update dbms/src/Storages/DeltaMerge/RowKeyFilter.h
JinheLin Jan 17, 2023
8483878
fix
JinheLin Jan 17, 2023
7cc7101
remove __func__
JinheLin Jan 17, 2023
bd11cc1
remove useless
JinheLin Jan 29, 2023
525650f
remove snapshot from BitmapFilter.
JinheLin Jan 29, 2023
6546115
add comment.
JinheLin Jan 29, 2023
332bcfe
fix comment.
JinheLin Jan 29, 2023
097d928
add comment.
JinheLin Jan 30, 2023
0ce7c8d
Merge branch 'master' into bitmap-filter-seg-row-id
ti-chi-bot Jan 31, 2023
3dd1d7f
Merge branch 'master' into bitmap-filter-seg-row-id
ti-chi-bot Jan 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test
JinheLin committed Jan 12, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 0e9d3cfdeb5d44c3e1a09846174c62a393dd959e
70 changes: 70 additions & 0 deletions dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
Original file line number Diff line number Diff line change
@@ -38,6 +38,11 @@
#include <iterator>
#include <random>

#include <TestUtils/InputStreamTestUtils.h>
#include <fmt/format.h>

using namespace std::chrono_literals;

namespace DB
{
namespace FailPoints
@@ -567,6 +572,71 @@ try
CATCH


TEST_P(DeltaMergeStoreRWTest, ReadWithRange)
try
{
const ColumnDefine col_i8_define(2, "i8", std::make_shared<DataTypeInt8>());
{
auto table_column_defines = DMTestEnv::getDefaultColumns();
table_column_defines->emplace_back(col_i8_define);
store = reload(table_column_defines);
}

auto create_block = [&](Int64 a, Int8 b) {
auto block = DMTestEnv::prepareSimpleWriteBlock(a, a + 1, false);
block.insert(DB::tests::createColumn<Int8>(
createSignedNumbers(b, b + 1),
col_i8_define.name,
col_i8_define.id));
return block;
};

auto b1 = create_block(std::numeric_limits<Int64>::min(), 1);
store->write(*db_context, db_context->getSettingsRef(), b1);

auto b3 = create_block(0L, 3);
store->write(*db_context, db_context->getSettingsRef(), b3);

auto b2 = create_block(std::numeric_limits<Int64>::max()-1, 2);
store->write(*db_context, db_context->getSettingsRef(), b2);

//while (!store->mergeDeltaAll(*db_context))
{
std::this_thread::sleep_for(10ms);
}
auto getIntHandleKey = [](Int64 i) {
WriteBufferFromOwnString ss;
DB::EncodeInt64(i, ss);
return std::make_shared<String>(ss.releaseStr());
};

{
Int64 start_key = 0;
RowKeyValue start(false, getIntHandleKey(start_key), start_key);
RowKeyValue end = RowKeyValue::INT_HANDLE_MAX_KEY;
RowKeyRange range(start, end, false, 1);
// read all columns from store
const auto & columns = store->getTableColumns();
BlockInputStreamPtr in = store->read(*db_context,
db_context->getSettingsRef(),
columns,
{range},
/* num_streams= */ 1,
/* max_version= */ std::numeric_limits<UInt64>::max(),
EMPTY_FILTER,
TRACING_NAME,
/* keep_order= */ false,
/* is_fast_scan= */ false,
/* expected_block_size= */ 1024)[0];
auto block = in->read();
const auto & col = block.getByPosition(0);
auto * ids = toColumnVectorDataPtr<Int64>(col.column);
std::vector<Int64> v(ids->data(), ids->data() + col.column->size());
std::cout << fmt::format("{}", v) << std::endl;
}
}
CATCH

TEST_P(DeltaMergeStoreRWTest, WriteCrashBeforeWalWithoutCache)
try
{