Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "Storage: Refactor DMFileReader" #8909

Merged
merged 2 commits into from
Apr 8, 2024
Merged

Revert "Storage: Refactor DMFileReader" #8909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3609b75
revert
Lloyd-Pottiger Apr 8, 2024
795c958
Merge branch 'master' into revert-8854-refactor-dmfile
JaySon-Huang Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion dbms/src/Storages/DeltaMerge/BitmapFilter/BitmapFilter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Common/Exception.h>
#include <Storages/DeltaMerge/BitmapFilter/BitmapFilter.h>
#include <Storages/DeltaMerge/DeltaMergeHelpers.h>

Expand Down
61 changes: 8 additions & 53 deletions dbms/src/Storages/DeltaMerge/File/ColumnCache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ namespace DB
{
namespace DM
{
RangeWithStrategys ColumnCache::getReadStrategy(size_t start_pack_idx, size_t pack_count, ColId column_id)
RangeWithStrategys ColumnCache::getReadStrategy(size_t pack_id, size_t pack_count, ColId column_id)
{
PackRange target_range{start_pack_idx, start_pack_idx + pack_count};
PackRange target_range{pack_id, pack_id + pack_count};

RangeWithStrategys range_and_strategys;
range_and_strategys.reserve(pack_count);
auto strategy = Strategy::Unknown;

Strategy strategy = Strategy::Unknown;
size_t range_start = 0;
for (size_t cursor = target_range.first; cursor < target_range.second; ++cursor)
for (size_t cursor = target_range.first; cursor < target_range.second; cursor++)
{
if (isPackInCache(cursor, column_id))
{
Expand All @@ -36,7 +36,7 @@ RangeWithStrategys ColumnCache::getReadStrategy(size_t start_pack_idx, size_t pa
}
else if (strategy == Strategy::Disk)
{
range_and_strategys.emplace_back(PackRange{range_start, cursor}, Strategy::Disk);
range_and_strategys.emplace_back(std::make_pair(PackRange{range_start, cursor}, Strategy::Disk));
}
range_start = cursor;
strategy = Strategy::Memory;
Expand All @@ -45,7 +45,7 @@ RangeWithStrategys ColumnCache::getReadStrategy(size_t start_pack_idx, size_t pa
{
if (strategy == Strategy::Memory)
{
range_and_strategys.emplace_back(PackRange{range_start, cursor}, Strategy::Memory);
range_and_strategys.emplace_back(std::make_pair(PackRange{range_start, cursor}, Strategy::Memory));
}
else if (strategy == Strategy::Disk)
{
Expand All @@ -55,53 +55,8 @@ RangeWithStrategys ColumnCache::getReadStrategy(size_t start_pack_idx, size_t pa
strategy = Strategy::Disk;
}
}
range_and_strategys.emplace_back(PackRange{range_start, target_range.second}, strategy);
range_and_strategys.shrink_to_fit();
return range_and_strategys;
}

RangeWithStrategys ColumnCache::getReadStrategy(
size_t start_pack_idx,
size_t pack_count,
const std::vector<size_t> & clean_read_pack_idx)
{
PackRange target_range{start_pack_idx, start_pack_idx + pack_count};
range_and_strategys.emplace_back(std::make_pair(PackRange{range_start, target_range.second}, strategy));

RangeWithStrategys range_and_strategys;
range_and_strategys.reserve(pack_count);
auto strategy = Strategy::Unknown;
size_t range_start = 0;
for (size_t cursor = target_range.first; cursor < target_range.second; ++cursor)
{
if (std::find(clean_read_pack_idx.cbegin(), clean_read_pack_idx.cend(), cursor) != clean_read_pack_idx.cend())
{
if (strategy == Strategy::Memory)
{
continue;
}
else if (strategy == Strategy::Disk)
{
range_and_strategys.emplace_back(PackRange{range_start, cursor}, Strategy::Disk);
}
range_start = cursor;
strategy = Strategy::Memory;
}
else
{
if (strategy == Strategy::Memory)
{
range_and_strategys.emplace_back(PackRange{range_start, cursor}, Strategy::Memory);
}
else if (strategy == Strategy::Disk)
{
continue;
}
range_start = cursor;
strategy = Strategy::Disk;
}
}
range_and_strategys.emplace_back(PackRange{range_start, target_range.second}, strategy);
range_and_strategys.shrink_to_fit();
return range_and_strategys;
}

Expand Down
6 changes: 1 addition & 5 deletions dbms/src/Storages/DeltaMerge/File/ColumnCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,7 @@ class ColumnCache

using RangeWithStrategy = std::pair<PackRange, ColumnCache::Strategy>;
using RangeWithStrategys = std::vector<RangeWithStrategy>;
RangeWithStrategys getReadStrategy(size_t start_pack_idx, size_t pack_count, ColId column_id);
static RangeWithStrategys getReadStrategy(
size_t start_pack_idx,
size_t pack_count,
const std::vector<size_t> & clean_read_pack_idx);
RangeWithStrategys getReadStrategy(size_t pack_id, size_t pack_count, ColId column_id);

void tryPutColumn(size_t pack_id, ColId column_id, const ColumnPtr & column, size_t rows_offset, size_t rows_count);

Expand Down
Loading